8000 Make benchfeatures work again by jkeiser · Pull Request #857 · simdjson/simdjson · GitHub
[go: up one dir, main page]
Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Make benchfeatures work again #857

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 5, 2020
Merged

Make benchfeatures work again #857

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8c600ca
Make benchfeatures work again
jkeiser May 2, 2020
aa53d87
Generate test data when tests are built
jkeiser May 5, 2020
1e8a54a
Fix benchfeatures.rb on Ruby 1.9
jkeiser May 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,15 +87,12 @@ objs

# Build outputs
/build*/
/visual_studio/

# Fuzzer outputs generated by instructions in fuzz/Fuzzing.md
/corpus.zip
/ossfuzz-out/
/out/

# Don't check in generated API docs
# Generated docs
/doc/api

# Don't check in generated examples
/jsonexamples/generated
/visual_studio
19 changes: 2 additions & 17 deletions benchmark/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
include_directories( . linux )
link_libraries(simdjson simdjson-windows-headers)
# add_executable(benchfeatures benchfeatures.cpp) # doesn't presently compile at all
link_libraries(simdjson simdjson-flags simdjson-windows-headers test-data)
add_executable(benchfeatures benchfeatures.cpp)
add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
add_executable(perfdiff perfdiff.cpp)
add_executable(parse parse.cpp)
Expand All @@ -18,8 +18,6 @@ if (SIMDJSON_GOOGLE_BENCHMARKS)
link_libraries(benchmark::benchmark)
add_executable(bench_parse_call bench_parse_call.cpp)
add_executable(bench_dom_api bench_dom_api.cpp)
target_link_libraries(bench_dom_api test-data)
target_link_libraries(bench_parse_call test-data)
endif()

if (SIMDJSON_COMPETITION)
Expand All @@ -37,16 +35,3 @@ if (SIMDJSON_COMPETITION)
endif()

include(checkperf.cmake)

# IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
# add_test(NAME checkperf
# COMMAND ${CMAKE_COMMAND} -E env
# CHECKPERF_REPOSITORY=${SIMDJSON_GITHUB_REPOSITORY}
# CHECKPERF_BRANCH=master
# CHECKPERF_DIR=${CMAKE_CURRENT_BINARY_DIR}/simdjson-master
# CHECKPERF_CMAKECACHE=${SIMDJSON_USER_CMAKECACHE}
# bash ${CMAKE_CURRENT_SOURCE_DIR}/checkperf.sh ${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json)
# set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
# set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
# set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
# ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
110 changes: 50 additions & 60 deletions benchmark/benchfeatures.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ void exit_usage(string message) {
}

struct option_struct {
architecture arch = architecture::UNSUPPORTED;
bool stage1_only = false;

int32_t iterations = 400;
Expand All @@ -98,10 +97,7 @@ struct option_struct {
verbose = true;
break;
case 'a':
arch = parse_architecture(optarg);
if (arch == architecture::UNSUPPORTED) {
exit_usage(string("Unsupported option value -a ") + optarg + ": expected -a HASWELL, WESTMERE or ARM64");
}
simdjson::active_implementation = simdjson::available_implementations[optarg];
break;
case 's':
if (!strcmp(optarg, "stage1")) {
Expand All @@ -113,15 +109,9 @@ struct option_struct {
}
break;
default:
exit_error("Unexpected argument " + c);
exit_error(string("Unexpected argument ") + std::string(1,static_cast<char>(c)));
}
}

// If architecture is not specified, pick the best supported architecture by default
if (arch == architecture::UNSUPPORTED) {
arch = find_best_supported_architecture();
}
dom::parser::use_implementation(arch);
}

template<typename F>
Expand Down Expand Up @@ -150,20 +140,20 @@ struct feature_benchmarker {
benchmarker struct23;
benchmarker struct23_miss;

feature_benchmarker(const simdjson::implementation &parser, event_collector& collector) :
utf8 ("jsonexamples/generated/utf-8.json", parser, collector),
utf8_miss ("jsonexamples/generated/utf-8-miss.json", parser, collector),
escape ("jsonexamples/generated/escape.json", parser, collector),
escape_miss ("jsonexamples/generated/escape-miss.json", parser, collector),
empty ("jsonexamples/generated/0-structurals.json", parser, collector),
empty_miss ("jsonexamples/generated/0-structurals-miss.json", parser, collector),
struct7 ("jsonexamples/generated/7-structurals.json", parser, collector),
struct7_miss ("jsonexamples/generated/7-structurals-miss.json", parser, collector),
struct7_full ("jsonexamples/generated/7-structurals-full.json", parser, collector),
struct15 ("jsonexamples/generated/15-structurals.json", parser, collector),
struct15_miss("jsonexamples/generated/15-structurals-miss.json", parser, collector),
struct23 ("jsonexamples/generated/23-structurals.json", parser, collector),
struct23_miss("jsonexamples/generated/23-structurals-miss.json", parser, collector)
feature_benchmarker(event_collector& collector) :
utf8 (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8.json", collector),
utf8_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8-miss.json", collector),
escape (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape.json", collector),
escape_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape-miss.json", collector),
empty (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals.json", collector),
empty_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals-miss.json", collector),
struct7 (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals.json", collector),
struct7_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-miss.json", collector),
struct7_full (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-full.json", collector),
struct15 (SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals.json", collector),
struct15_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals-miss.json", collector),
struct23 (SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals.json", collector),
struct23_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals-miss.json", collector)
{

}
Expand All @@ -185,7 +175,7 @@ struct feature_benchmarker {
}

double cost_per_block(BenchmarkStage stage, const benchmarker& feature, size_t feature_blocks, const benchmarker& base) const {
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / feature_blocks;
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / double(feature_blocks);
}

// Whether we're recording cache miss and branch miss events
Expand All @@ -195,7 +185,7 @@ struct feature_benchmarker {

// Base cost of any block (including empty ones)
double base_cost(BenchmarkStage stage) const {
return (empty[stage].best.elapsed_ns() / empty.stats->blocks);
return (empty[stage].best.elapsed_ns() / double(empty.stats->blocks));
}

// Extra cost of a 1-7 structural block over an empty block
Expand All @@ -209,7 +199,7 @@ struct feature_benchmarker {
// Rate of 1-7-structural misses per 8-structural flip
double struct1_7_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses()) / struct7_miss.stats->blocks_with_1_structural_flipped;
return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
}

// Extra cost of an 8-15 structural block over a 1-7 structural block
Expand All @@ -223,7 +213,7 @@ struct feature_benchmarker {
// Rate of 8-15-structural misses per 8-structural flip
double struct8_15_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / struct15_miss.stats->blocks_with_8_structurals_flipped;
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
}

// Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
Expand All @@ -237,7 +227,7 @@ struct feature_benchmarker {
// Rate of 16-structural misses per 16-structural flip
double struct16_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / struct23_miss.stats->blocks_with_16_structurals_flipped;
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
}

// Extra cost of having UTF-8 in a block
Expand All @@ -251,7 +241,7 @@ struct feature_benchmarker {
// Rate of UTF-8 misses per UTF-8 flip
double utf8_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / utf8_miss.stats->blocks_with_utf8_flipped;
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
}

// Extra cost of having escapes in a block
Expand All @@ -265,39 +255,39 @@ struct feature_benchmarker {
// Rate of escape misses per escape flip
double escape_miss_rate(BenchmarkStage stage) const {
if (!has_events()) { return 1; }
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / escape_miss.stats->blocks_with_escapes_flipped;
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
}

double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
// Expected base ns/block (empty)
json_stats& stats = *file.stats;
double expected = base_cost(stage) * stats.blocks;
expected += struct1_7_cost(stage) * stats.blocks_with_1_structural;
expected += utf8_cost(stage) * stats.blocks_with_utf8;
expected += escape_cost(stage) * stats.blocks_with_escapes;
expected += struct8_15_cost(stage) * stats.blocks_with_8_structurals;
expected += struct16_cost(stage) * stats.blocks_with_16_structurals;
return expected / stats.blocks;
double expected = base_cost(stage) * double(stats.blocks);
expected += struct1_7_cost(stage) * double(stats.blocks_with_1_structural);
expected += utf8_cost(stage) * double(stats.blocks_with_utf8);
expected += escape_cost(stage) * double(stats.blocks_with_escapes);
expected += struct8_15_cost(stage) * double(stats.blocks_with_8_structurals);
expected += struct16_cost(stage) * double(stats.blocks_with_16_structurals);
return expected / double(stats.blocks);
}

double calc_expected_miss_cost(BenchmarkStage stage, const benchmarker& file) const {
// Expected base ns/block (empty)
json_stats& stats = *file.stats;
double expected = struct1_7_miss_cost(stage) * stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
expected += utf8_miss_cost(stage) * stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
expected += escape_miss_cost(stage) * stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
expected += struct8_15_miss_cost(stage) * stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
expected += struct16_miss_cost(stage) * stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
return expected / stats.blocks;
double expected = struct1_7_miss_cost(stage) * double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
expected += utf8_miss_cost(stage) * double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
expected += escape_miss_cost(stage) * double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
expected += struct8_15_miss_cost(stage) * double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
expected += struct16_miss_cost(stage) * double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
return expected / double(stats.blocks);
}

double calc_expected_misses(BenchmarkStage stage, const benchmarker& file) const {
json_stats& stats = *file.stats;
double expected = stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
expected += stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
expected += stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
expected += stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
expected += stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
double expected = double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
expected += double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
expected += double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
expected += double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
expected += double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
return expected;
}

Expand Down Expand Up @@ -364,10 +354,10 @@ struct feature_benchmarker {
};

void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
double actual = results[stage].best.elapsed_ns() / results.stats->blocks;
double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
double calc = features.calc_expected(stage, results);
uint64_t actual_misses = results[stage].best.branch_misses();
uint64_t calc_misses = uint64_t(features.calc_expected_misses(stage, results));
double actual_misses = results[stage].best.branch_misses();
double calc_misses = features.calc_expected_misses(stage, results);
double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
printf(" | %-8s ", benchmark_stage_name(stage));
printf("| %-15s ", filename);
Expand All @@ -376,10 +366,10 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
printf("| %8.3g ", calc);
printf("| %8.3g ", actual);
printf("| %+8.3g ", actual - calc);
printf("| %13lu ", calc_misses);
printf("| %13llu ", (long long unsigned)(calc_misses));
if (features.has_events()) {
printf("| %13lu ", actual_misses);
printf("| %+13ld ", int64_t(actual_misses - calc_misses));
printf("| %13llu ", (long long unsigned)(actual_misses));
printf("| %+13lld ", (long long int)(actual_misses - calc_misses));
double miss_adjustment = calc_miss_cost * (double(int64_t(actual_misses - calc_misses)) / calc_misses);
printf("| %8.3g ", calc_miss_cost + miss_adjustment);
printf("| %+8.3g ", actual - (calc + miss_adjustment));
Expand All @@ -401,9 +391,9 @@ int main(int argc, char *argv[]) {

// Set up benchmarkers by reading all files
feature_benchmarker features(collector);
benchmarker gsoc_2018("jsonexamples/gsoc-2018.json", collector);
benchmarker twitter("jsonexamples/twitter.json", collector);
benchmarker random("jsonexamples/random.json", collector);
benchmarker gsoc_2018(SIMDJSON_BENCHMARK_DATA_DIR "gsoc-2018.json", collector);
benchmarker twitter(SIMDJSON_BENCHMARK_DATA_DIR "twitter.json", collector);
benchmarker random(SIMDJSON_BENCHMARK_DATA_DIR "random.json", collector);

// Run the benchmarks
progress_bar progress(options.iterations, 100);
Expand Down
19 changes: 15 additions & 4 deletions jsonexamples/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
set(EXAMPLE_JSON ${CMAKE_CURRENT_BINARY_DIR}/twitter.json PARENT_SCOPE)
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_BINARY_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)

# Copy static files to the build dir so they live alongside the generated ones
file(GLOB_RECURSE example_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.json *.ndjson)
foreach(example_file ${example_files})
configure_file(${example_file} ${example_file} COPYONLY)
endforeach(example_file)

add_subdirectory(generated)

add_library(jsonexamples-data INTERFACE)
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_BINARY_DIR}/")
add_dependencies(jsonexamples-data ${example_files})
add_dependencies(jsonexamples-data generated-data)
17 changes: 17 additions & 0 deletions jsonexamples/generated/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
set(generated_files
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know this works, obviously, but I find it confusing. Nothing else in the project seems to depend on generated_files. But benchmark/benchfeatures.cpp requires these files... Gosh. How does this work? And what happens when ruby is missing... how can benchfeatures work?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When ruby is missing, benchfeatures will fail :) This is all manual right now: you have to make benchfeatures generated-data and run it yourself. Ultimately, I'd like to write the generator in something other than Ruby (probably C++, just because that's what we have available) and make it a full target that builds and runs.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When ruby is missing, benchfeatures will fail :)

This is not a request on my part... but wouldn't the nice thing be to make it so that the benchfeatures is disabled if ruby is missing? (I am not asking for changes, just inquiring.)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about that, but I kinda want to keep it compiling in CI if we can. I'd rather it not rot again.

utf-8.json escape.json
0-structurals.json 7-structurals.json 15-structurals.json 23-structurals.json
)
find_package(Ruby QUIET)
if (RUBY_EXECUTABLE)
add_custom_command(
OUTPUT ${generated_files}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/miss-templates/*.txt
COMMAND ${RUBY_EXECUTABLE} genfeaturejson.rb ${CMAKE_CURRENT_BINARY_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)
add_custom_target(generated-data DEPENDS ${generated_files})
else (RUBY_EXECUTABLE)
# TODO make this work without Ruby and make it part of the normal build
add_custom_target(generated-data)
endif (RUBY_EXECUTABLE)
Loading
0