8000 Make benchfeatures work again · simdjson/simdjson@eac8e61 · GitHub
[go: up one dir, main page]
Skip to content

Commit eac8e61

Browse files
jkeiserjkeiser
committed
Make benchfeatures work again
1 parent 73e27bd commit eac8e61

File tree

9 files changed

+84
-93
lines changed

9 files changed

+84
-93
lines changed

‎.gitignore

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,12 @@ objs
8787

8888
# Build outputs
8989
/build*/
90+
/visual_studio/
9091

9192
# Fuzzer outputs generated by instructions in fuzz/Fuzzing.md
9293
/corpus.zip
9394
/ossfuzz-out/
9495
/out/
9596

96-
# Don't check in generated API docs
97+
# Generated docs
9798
/doc/api
98-
99-
# Don't check in generated examples
100-
/jsonexamples/generated
101-
/visual_studio

‎benchmark/CMakeLists.txt

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
include_directories( . linux )
2-
link_libraries(simdjson simdjson-windows-headers)
3-
# add_executable(benchfeatures benchfeatures.cpp) # doesn't presently compile at all
2+
link_libraries(simdjson simdjson-flags simdjson-windows-headers test-data)
3+
add_executable(benchfeatures benchfeatures.cpp)
44
add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
55
add_executable(perfdiff perfdiff.cpp)
66
add_executable(parse parse.cpp)
@@ -18,8 +18,6 @@ if (SIMDJSON_GOOGLE_BENCHMARKS)
1818
link_libraries(benchmark::benchmark)
1919
add_executable(bench_parse_call bench_parse_call.cpp)
2020
add_executable(bench_dom_api bench_dom_api.cpp)
21-
target_link_libraries(bench_dom_api test-data)
22-
target_link_libraries(bench_parse_call test-data)
2321
endif()
2422

2523
if (SIMDJSON_COMPETITION)
@@ -37,16 +35,3 @@ if (SIMDJSON_COMPETITION)
3735
endif()
3836

3937
include(checkperf.cmake)
40-
41-
# IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
42-
# add_test(NAME checkperf
43-
# COMMAND ${CMAKE_COMMAND} -E env
44-
# CHECKPERF_REPOSITORY=${SIMDJSON_GITHUB_REPOSITORY}
45-
# CHECKPERF_BRANCH=master
46-
# CHECKPERF_DIR=${CMAKE_CURRENT_BINARY_DIR}/simdjson-master
47-
# CHECKPERF_CMAKECACHE=${SIMDJSON_USER_CMAKECACHE}
48-
# bash ${CMAKE_CURRENT_SOURCE_DIR}/checkperf.sh ${PROJECT_SOURCE_DIR}/jsonexamples/twitter.json)
49-
# set_property(TEST checkperf APPEND PROPERTY LABELS per_implementation)
50-
# set_property(TEST checkperf APPEND PROPERTY DEPENDS parse perfdiff ${SIMDJSON_USER_CMAKECACHE})
51-
# set_property(TEST checkperf PROPERTY RUN_SERIAL TRUE)
52-
# ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux")

‎benchmark/benchfeatures.cpp

Lines changed: 50 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ void exit_usage(string message) {
7575
}
7676

7777
struct option_struct {
78-
architecture arch = architecture::UNSUPPORTED;
7978
bool stage1_only = false;
8079

8180
int32_t iterations = 400;
@@ -86,7 +85,7 @@ struct option_struct {
8685
option_struct(int argc, char **argv) {
8786
int c;
8887

89-
while ((c = getopt(argc, argv, "vtn:i:a:s:")) != -1) {
88+
while ((c = getopt(argc, argv, "vtn:i:s:")) != -1) {
9089
switch (c) {
9190
case 'n':
9291
iterations = atoi(optarg);
@@ -97,12 +96,6 @@ struct option_struct {
9796
case 'v':
9897
verbose = true;
9998
break;
100-
case 'a':
101-
arch = parse_architecture(optarg);
102-
if (arch == architecture::UNSUPPORTED) {
103-
exit_usage(string("Unsupported option value -a ") + optarg + ": expected -a HASWELL, WESTMERE or ARM64");
104-
}
105-
break;
10699
case 's':
107100
if (!strcmp(optarg, "stage1")) {
108101
stage1_only = true;
@@ -113,15 +106,9 @@ struct option_struct {
113106
}
114107
break;
115108
default:
116-
exit_error("Unexpected argument " + c);
109+
exit_error(string("Unexpected argument ") + std::string(1,static_cast<char>(c)));
117110
}
118111
}
119-
120-
// If architecture is not specified, pick the best supported architecture by default
121-
if (arch == architecture::UNSUPPORTED) {
122-
arch = find_best_supported_architecture();
123-
}
124-
dom::parser::use_implementation(arch);
125112
}
126113

127114
template<typename F>
@@ -150,20 +137,20 @@ struct feature_benchmarker {
150137
benchmarker struct23;
151138
benchmarker struct23_miss;
152139

153-
feature_benchmarker(const simdjson::implementation &parser, event_collector& collector) :
154-
utf8 ("jsonexamples/generated/utf-8.json", parser, collector),
155-
utf8_miss ("jsonexamples/generated/utf-8-miss.json", parser, collector),
156-
escape ("jsonexamples/generated/escape.json", parser, collector),
157-
escape_miss ("jsonexamples/generated/escape-miss.json", parser, collector),
158-
empty ("jsonexamples/generated/0-structurals.json", parser, collector),
159-
empty_miss ("jsonexamples/generated/0-structurals-miss.json", parser, collector),
160-
struct7 ("jsonexamples/generated/7-structurals.json", parser, collector),
161-
struct7_miss ("jsonexamples/generated/7-structurals-miss.json", parser, collector),
162-
struct7_full ("jsonexamples/generated/7-structurals-full.json", parser, collector),
163-
struct15 ("jsonexamples/generated/15-structurals.json", parser, collector),
164-
struct15_miss("jsonexamples/generated/15-structurals-miss.json", parser, collector),
165-
struct23 ("jsonexamples/generated/23-structurals.json", parser, collector),
166-
struct23_miss("jsonexamples/generated/23-structurals-miss.json", parser, collector)
140+
feature_benchmarker(event_collector& collector) :
141+
utf8 (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8.json", collector),
142+
utf8_mis F438 s (SIMDJSON_BENCHMARK_DATA_DIR "generated/utf-8-miss.json", collector),
143+
escape (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape.json", collector),
144+
escape_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/escape-miss.json", collector),
145+
empty (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals.json", collector),
146+
empty_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/0-structurals-miss.json", collector),
147+
struct7 (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals.json", collector),
148+
struct7_miss (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-miss.json", collector),
149+
struct7_full (SIMDJSON_BENCHMARK_DATA_DIR "generated/7-structurals-full.json", collector),
150+
struct15 (SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals.json", collector),
151+
struct15_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/15-structurals-miss.json", collector),
152+
struct23 (SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals.json", collector),
153+
struct23_miss(SIMDJSON_BENCHMARK_DATA_DIR "generated/23-structurals-miss.json", collector)
167154
{
168155

169156
}
@@ -185,7 +172,7 @@ struct feature_benchmarker {
185172
}
186173

187174
double cost_per_block(BenchmarkStage stage, const benchmarker& feature, size_t feature_blocks, const benchmarker& base) const {
188-
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / feature_blocks;
175+
return (feature[stage].best.elapsed_ns() - base[stage].best.elapsed_ns()) / double(feature_blocks);
189176
}
190177

191178
// Whether we're recording cache miss and branch miss events
@@ -195,7 +182,7 @@ struct feature_benchmarker {
195182

196183
// Base cost of any block (including empty ones)
197184
double base_cost(BenchmarkStage stage) const {
198-
return (empty[stage].best.elapsed_ns() / empty.stats->blocks);
185+
return (empty[stage].best.elapsed_ns() / double(empty.stats->blocks));
199186
}
200187

201188
// Extra cost of a 1-7 structural block over an empty block
@@ -209,7 +196,7 @@ struct feature_benchmarker {
209196
// Rate of 1-7-structural misses per 8-structural flip
210197
double struct1_7_miss_rate(BenchmarkStage stage) const {
211198
if (!has_events()) { return 1; }
212-
return double(struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses()) / struct7_miss.stats->blocks_with_1_structural_flipped;
199+
return struct7_miss[stage].best.branch_misses() - struct7[stage].best.branch_misses() / double(struct7_miss.stats->blocks_with_1_structural_flipped);
213200
}
214201

215202
// Extra cost of an 8-15 structural block over a 1-7 structural block
@@ -223,7 +210,7 @@ struct feature_benchmarker {
223210
// Rate of 8-15-structural misses per 8-structural flip
224211
double struct8_15_miss_rate(BenchmarkStage stage) const {
225212
if (!has_events()) { return 1; }
226-
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / struct15_miss.stats->blocks_with_8_structurals_flipped;
213+
return double(struct15_miss[stage].best.branch_misses() - struct15[stage].best.branch_misses()) / double(struct15_miss.stats->blocks_with_8_structurals_flipped);
227214
}
228215

229216
// Extra cost of a 16+-structural block over an 8-15 structural block (actual varies based on # of structurals!)
@@ -237,7 +224,7 @@ struct feature_benchmarker {
237224
// Rate of 16-structural misses per 16-structural flip
238225
double struct16_miss_rate(BenchmarkStage stage) const {
239226
if (!has_events()) { return 1; }
240-
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / struct23_miss.stats->blocks_with_16_structurals_flipped;
227+
return double(struct23_miss[stage].best.branch_misses() - struct23[stage].best.branch_misses()) / double(struct23_miss.stats->blocks_with_16_structurals_flipped);
241228
}
242229

243230
// Extra cost of having UTF-8 in a block
@@ -251,7 +238,7 @@ struct feature_benchmarker {
251238
// Rate of UTF-8 misses per UTF-8 flip
252239
double utf8_miss_rate(BenchmarkStage stage) const {
253240
if (!has_events()) { return 1; }
254-
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / utf8_miss.stats->blocks_with_utf8_flipped;
241+
return double(utf8_miss[stage].best.branch_misses() - utf8[stage].best.branch_misses()) / double(utf8_miss.stats->blocks_with_utf8_flipped);
255242
}
256243

257244
// Extra cost of having escapes in a block
@@ -265,39 +252,39 @@ struct feature_benchmarker {
265252
// Rate of escape misses per escape flip
266253
double escape_miss_rate(BenchmarkStage stage) const {
267254
if (!has_events()) { return 1; }
268-
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / escape_miss.stats->blocks_with_escapes_flipped;
255+
return double(escape_miss[stage].best.branch_misses() - escape[stage].best.branch_misses()) / double(escape_miss.stats->blocks_with_escapes_flipped);
269256
}
270257

271258
double calc_expected_feature_cost(BenchmarkStage stage, const benchmarker& file) const {
272259
// Expected base ns/block (empty)
273260
json_stats& stats = *file.stats;
274-
double expected = base_cost(stage) * stats.blocks;
275-
expected += struct1_7_cost(stage) * stats.blocks_with_1_structural;
276-
expected += utf8_cost(stage) * stats.blocks_with_utf8;
277-
expected += escape_cost(stage) * stats.blocks_with_escapes;
278-
expected += struct8_15_cost(stage) * stats.blocks_with_8_structurals;
279-
expected += struct16_cost(stage) * stats.blocks_with_16_structurals;
280-
return expected / stats.blocks;
261+
double expected = base_cost(stage) * double(stats.blocks);
262+
expected += struct1_7_cost(stage) * double(stats.blocks_with_1_structural);
263+
expected += utf8_cost(stage) * double(stats.blocks_with_utf8);
264+
expected += escape_cost(stage) * double(stats.blocks_with_escapes);
265+
expected += struct8_15_cost(stage) * double(stats.blocks_with_8_structurals);
266+
expected += struct16_cost(stage) * double(stats.blocks_with_16_structurals);
267+
return expected / double(stats.blocks);
281268
}
282269

283270
double calc_expected_miss_cost(BenchmarkStage stage, const benchmarker& file) const {
284271
// Expected base ns/block (empty)
285272
json_stats& stats = *file.stats;
286-
double expected = struct1_7_miss_cost(stage) * stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
287-
expected += utf8_miss_cost(stage) * stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
288-
expected += escape_miss_cost(stage) * stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
289-
expected += struct8_15_miss_cost(stage) * stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
290-
expected += struct16_miss_cost(stage) * stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
291-
return expected / stats.blocks;
273+
double expected = struct1_7_miss_cost(stage) * double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
274+
expected += utf8_miss_cost(stage) * double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
275+
expected += escape_miss_cost(stage) * double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
276+
expected += struct8_15_miss_cost(stage) * double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
277+
expected += struct16_miss_cost(stage) * double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
278+
return expected / double(stats.blocks);
292279
}
293280

294281
double calc_expected_misses(BenchmarkStage stage, const benchmarker& file) const {
295282
json_stats& stats = *file.stats;
296-
double expected = stats.blocks_with_1_structural_flipped * struct1_7_miss_rate(stage);
297-
expected += stats.blocks_with_utf8_flipped * utf8_miss_rate(stage);
298-
expected += stats.blocks_with_escapes_flipped * escape_miss_rate(stage);
299-
expected += stats.blocks_with_8_structurals_flipped * struct8_15_miss_rate(stage);
300-
expected += stats.blocks_with_16_structurals_flipped * struct16_miss_rate(stage);
283+
double expected = double(stats.blocks_with_1_structural_flipped) * struct1_7_miss_rate(stage);
284+
expected += double(stats.blocks_with_utf8_flipped) * utf8_miss_rate(stage);
285+
expected += double(stats.blocks_with_escapes_flipped) * escape_miss_rate(stage);
286+
expected += double(stats.blocks_with_8_structurals_flipped) * struct8_15_miss_rate(stage);
287+
expected += double(stats.blocks_with_16_structurals_flipped) * struct16_miss_rate(stage);
301288
return expected;
302289
}
303290

@@ -364,10 +351,10 @@ struct feature_benchmarker {
364351
};
365352

366353
void print_file_effectiveness(BenchmarkStage stage, const char* filename, const benchmarker& results, const feature_benchmarker& features) {
367-
double actual = results[stage].best.elapsed_ns() / results.stats->blocks;
354+
double actual = results[stage].best.elapsed_ns() / double(results.stats->blocks);
368355
double calc = features.calc_expected(stage, results);
369-
uint64_t actual_misses = results[stage].best.branch_misses();
370-
uint64_t calc_misses = uint64_t(features.calc_expected_misses(stage, results));
356+
double actual_misses = results[stage].best.branch_misses();
357+
double calc_misses = features.calc_expected_misses(stage, results);
371358
double calc_miss_cost = features.calc_expected_miss_cost(stage, results);
372359
printf(" | %-8s ", benchmark_stage_name(stage));
373360
printf("| %-15s ", filename);
@@ -376,10 +363,10 @@ void print_file_effectiveness(BenchmarkStage stage, const char* filename, const
376363
printf("| %8.3g ", calc);
377364
printf("| %8.3g ", actual);
378365
printf("| %+8.3g ", actual - calc);
379-
printf("| %13lu ", calc_misses);
366+
printf("| %13llu ", (long long unsigned)(calc_misses));
380367
if (features.has_events()) {
381-
printf("| %13lu ", actual_misses);
382-
printf("| %+13ld ", int64_t(actual_misses - calc_misses));
368+
printf("| %13llu ", (long long unsigned)(actual_misses));
369+
printf("| %+13lld ", (long long int)(actual_misses - calc_misses));
383370
double miss_adjustment = calc_miss_cost * (double(int64_t(actual_misses - calc_misses)) / calc_misses);
384371
printf("| %8.3g ", calc_miss_cost + miss_adjustment);
385372
printf("| %+8.3g ", actual - (calc + miss_adjustment));
@@ -401,9 +388,9 @@ int main(int argc, char *argv[]) {
401388

402389
// Set up benchmarkers by reading all files
403390
feature_benchmarker features(collector);
404-
benchmarker gsoc_2018("jsonexamples/gsoc-2018.json", collector);
405-
benchmarker twitter("jsonexamples/twitter.json", collector);
406-
benchmarker random("jsonexamples/random.json", collector);
391+
benchmarker gsoc_2018(SIMDJSON_BENCHMARK_DATA_DIR "gsoc-2018.json", collector);
392+
benchmarker twitter(SIMDJSON_BENCHMARK_DATA_DIR "twitter.json", collector);
393+
benchmarker random(SIMDJSON_BENCHMARK_DATA_DIR "random.json", collector);
407394

408395
// Run the benchmarks
409396
progress_bar progress(options.iterations, 100);

‎jsonexamples/CMakeLists.txt

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
1-
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
2-
set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
3-
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
1+
set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
2+
set(EXAMPLE_JSON ${CMAKE_CURRENT_BINARY_DIR}/twitter.json PARENT_SCOPE)
3+
set(EXAMPLE_NDJSON ${CMAKE_CURRENT_BINARY_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
44
add_library(jsonexamples-data INTERFACE)
5-
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
5+
target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_BINARY_DIR}/")
6+
7+
# Copy static files to the build dir so they live alongside the generated ones
8+
file(GLOB_RECURSE example_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.json *.ndjson)
9+
foreach(example_file ${example_files})
10+
configure_file(${example_file} ${example_file} COPYONLY)
11+
endforeach(example_file)
12+
13+
add_subdirectory(generated)

‎jsonexamples/generated/CMakeLists.txt

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
set(generated_files
2+
utf-8.json escape.json
3+
0-structurals.json 7-structurals.json 15-structurals.json 23-structurals.json
4+
)
5+
find_package(Ruby QUIET)
6+
if (RUBY_EXECUTABLE)
7+
add_custom_command(
8+
OUTPUT ${generated_files}
9+
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/miss-templates/*.txt
10+
COMMAND ${RUBY_EXECUTABLE} genfeaturejson.rb ${CMAKE_CURRENT_BINARY_DIR}
11+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
12+
)
13+
add_custom_target(generated-data DEPENDS ${generated_files})
14+
endif (RUBY_EXECUTABLE)

‎benchmark/genfeaturejson.rb renamed to ‎jsonexamples/generated/genfeaturejson.rb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def write_files(filename, start1, repeat1, end1, repeat2: '', include_newline: t
2626

2727
def write_full(filename, start1, repeat1, end1)
2828
puts "Writing #{filename} ..."
29-
File.open(filename, "w") do |file|
29+
File.open(filename, "wb") do |file|
3030
write_chunks(file, start1, repeat1, end1, @@file_size)
3131
end
3232
raise "OMG wrong file size #{File.size(filename)} (should be #{@@file_size})" if File.size(filename) != @@file_size
@@ -39,7 +39,7 @@ def write_half(filename, start1, repeat1, end1, repeat2)
3939
halfway_point = start1.bytesize + repeat1_len + repeat2.bytesize
4040

4141
puts "Writing #{filename} ..."
42-
File.open(filename, "w") do |file|
42+
File.open(filename, "wb") do |file|
4343
write_chunks(file, start1, repeat1, repeat2, halfway_point)
4444
write_chunks(file, repeat2, repeat2, end1, @@file_size-halfway_point)
4545
end
@@ -55,7 +55,7 @@ def write_half_miss(filename, start1, repeat1, end1, repeat2)
5555
repeat_template = (repeat_chunks - 1).step(repeat_template.size - 1, repeat_chunks).map { |i| repeat_template[i] }
5656

5757
puts "Writing #{filename} ..."
58-
File.open(filename, "w") do |file|
58+
File.open(filename, "wb") do |file|
5959
file.write(start1)
6060
repeat_template.each do |should_repeat|
6161
file.write(should_repeat == "1" ? repeat1 : repeat2)
@@ -83,7 +83,7 @@ def write_chunks(file, start1, repeat1, end1, size)
8383
end
8484
end
8585

86-
output_dir = File.expand_path("../jsonexamples/generated", File.dirname(__FILE__))
86+
output_dir = ARGV[0] || File.expand_path(".", File.dirname(__FILE__))
8787
miss_templates = File.expand_path("miss-templates", File.dirname(__FILE__))
8888
Dir.mkdir(output_dir) unless File.directory?(output_dir)
8989
w = ChunkWriter.new(output_dir, miss_templates)

‎benchmark/genfeaturejsonseed.rb renamed to ‎jsonexamples/generated/genfeaturejsonseed.rb

File renamed without changes.

‎benchmark/miss-templates/128.txt renamed to ‎jsonexamples/generated/miss-templates/128.txt

File renamed without changes.

‎benchmark/miss-templates/64.txt renamed to ‎jsonexamples/generated/miss-templates/64.txt

File renamed without changes.

0 commit comments

Comments
 (0)
0