From 3553148174ceb9c01eec1ca660aa77d5f0bd0b0e Mon Sep 17 00:00:00 2001 From: goksu <25721443+goeksu@users.noreply.github.com> Date: Fri, 18 Aug 2023 02:04:38 +0300 Subject: [PATCH] detecting outliers --- builds/testing.mk | 2 +- src/tools/ftbench/ftbench.c | 75 +++++++++++++++++-------- src/tools/ftbench/src/tohtml.py | 99 ++++++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 27 deletions(-) diff --git a/builds/testing.mk b/builds/testing.mk index b5e79b07a..55288e4af 100644 --- a/builds/testing.mk +++ b/builds/testing.mk @@ -3,7 +3,7 @@ FTBENCH_DIR = $(TOP_DIR)/src/tools/ftbench FTBENCH_SRC = $(FTBENCH_DIR)/ftbench.c FTBENCH_OBJ = $(OBJ_DIR)/bench.$(SO) FTBENCH_BIN = $(OBJ_DIR)/bench$E -FTBENCH_FLAG ?= -c 1000 -w 100 +FTBENCH_FLAG ?= -c 550 -w 50 INCLUDES = $(TOP_DIR)/include FONTS = $(wildcard $(FTBENCH_DIR)/fonts/*.ttf) BASELINE_DIR = $(OBJ_DIR)/baseline/ diff --git a/src/tools/ftbench/ftbench.c b/src/tools/ftbench/ftbench.c index c7126ae9f..8b2188131 100644 --- a/src/tools/ftbench/ftbench.c +++ b/src/tools/ftbench/ftbench.c @@ -260,11 +260,18 @@ #define TIMER_GET( timer ) ( timer )->total #define TIMER_RESET( timer ) ( timer )->total = 0 +#define CHUNK_SIZE 100 +int compare(const void* a, const void* b) { + if (*(double*)a > *(double*)b) return 1; + if (*(double*)a < *(double*)b) return -1; + return 0; +} /* * Bench code */ + static void benchmark( FT_Face face, btest_t* test, @@ -276,6 +283,8 @@ int n, done; btimer_t timer, elapsed; + int NUM_CHUNKS = max_iter / CHUNK_SIZE; + double results[NUM_CHUNKS]; if ( test->cache_first ) { @@ -283,39 +292,59 @@ test->bench( &timer, face, test->user_data ); } + // Initial warm-up + for (n = 0; n < warmup; n++) { + test->bench(&timer, face, test->user_data); + } + printf( " %-25s ", test->title ); fflush( stdout ); - TIMER_RESET( &timer ); - TIMER_RESET( &elapsed ); - int is_warmup = 1; - - for ( n = 0, done = 0; !max_iter || n < max_iter; n++ ) - { - if ( is_warmup && n == warmup ){ - is_warmup = 0; + for (int chunk = 0; chunk < NUM_CHUNKS; chunk++) { TIMER_RESET( &timer ); TIMER_RESET( &elapsed ); - } - - TIMER_START( &elapsed ); + // Execute a chunk of iterations + for (n = 0, done = 0; n < CHUNK_SIZE; n++) { + TIMER_START( &elapsed ); + done += test->bench( &timer, face, test->user_data ); + TIMER_STOP( &elapsed ); - done += test->bench( &timer, face, test->user_data ); - - TIMER_STOP( &elapsed ); - - - if (!is_warmup && TIMER_GET( &elapsed ) > 1E6 * max_time ) - break; + + } + if (TIMER_GET( &elapsed ) > 1E6 * max_time) { + //break; + } + results[chunk] = TIMER_GET( &timer ); } - if ( done ) - printf( "%10.1f microseconds %10d done\n", - TIMER_GET( &timer ), done ); - else - printf( "no error-free calls\n" ); + // Sort results for IQR calculation + qsort(results, NUM_CHUNKS, sizeof(double), compare); + + double q1 = results[NUM_CHUNKS / 4]; + double q3 = results[3 * NUM_CHUNKS / 4]; + double iqr = q3 - q1; + double lower_bound = q1 - 1.5 * iqr; + double upper_bound = q3 + 1.5 * iqr; + + double total_time = 0.0; + int valid_chunks = 0; + + for (int chunk = 0; chunk < NUM_CHUNKS; chunk++) { + if (results[chunk] >= lower_bound && results[chunk] <= upper_bound) { + total_time += results[chunk]; + valid_chunks++; + } + } + + double average_time = total_time / valid_chunks; + + + + printf( "%10.1f microseconds %10d done\n", + average_time, done ); + } diff --git a/src/tools/ftbench/src/tohtml.py b/src/tools/ftbench/src/tohtml.py index 3fc01db60..61205a671 100644 --- a/src/tools/ftbench/src/tohtml.py +++ b/src/tools/ftbench/src/tohtml.py @@ -55,6 +55,9 @@ def main(): generate_info_table(html_file, baseline_info, benchmark_info) + # Generate total results table + generate_total_results_table(html_file, BASELINE_DIR, BENCHMARK_DIR) + # Generate results tables for filename in os.listdir(BASELINE_DIR): if filename.endswith(".txt") and not filename == "info.txt": @@ -64,6 +67,8 @@ def main(): generate_results_table( html_file, baseline_results, benchmark_results, filename ) + + write_to_html(html_file, "
Freetype Benchmark
\n") write_to_html(html_file, "\n\n") @@ -103,7 +108,95 @@ def generate_info_table(html_file, baseline_info, benchmark_info): ), ) write_to_html(html_file, "
") - write_to_html(html_file, "* Cumulative time for iterations which is better in smaller values
\n") + write_to_html(html_file, "

* Average time for all iterations. Smaller values are better.

") + write_to_html(html_file, "

** N count in (x | y) format is for showing baseline and benchmark N counts seperately when they differs.

") + + +def generate_total_results_table(html_file, baseline_dir, benchmark_dir): + """Prepare total results table for html""" + + # This dictionary will store aggregated results. + test_results = {test: {"baseline": 0, "benchmark": 0, "n_baseline": 0, "n_benchmark": 0} for test in [ + "Load", "Load_Advances (Normal)", "Load_Advances (Fast)", "Load_Advances (Unscaled)", "Render", + "Get_Glyph", "Get_Char_Index", "Iterate CMap", "New_Face", "Embolden", "Stroke", "Get_BBox", + "Get_CBox", "New_Face & load glyph(s)" + ]} + + for filename in os.listdir(baseline_dir): + if filename.endswith(".txt") and not filename == "info.txt": + + baseline_results = read_file(os.path.join(baseline_dir, filename)) + benchmark_results = read_file(os.path.join(benchmark_dir, filename)) + + for baseline_line, benchmark_line in zip(baseline_results, benchmark_results): + if baseline_line.startswith(" "): + baseline_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", baseline_line) + benchmark_match = re.match(r"\s+(.*?)\s+(\d+\.\d+)\s+microseconds\s+(\d+)\s", benchmark_line) + + if baseline_match and benchmark_match: + test = baseline_match.group(1).strip() + baseline_value = float(baseline_match.group(2)) + benchmark_value = float(benchmark_match.group(2)) + baseline_n = int(baseline_match.group(3)) + benchmark_n = int(benchmark_match.group(3)) + + # Aggregate the results + if test in test_results: + test_results[test]["baseline"] += baseline_value + test_results[test]["benchmark"] += benchmark_value + test_results[test]["n_baseline"] += baseline_n + test_results[test]["n_benchmark"] += benchmark_n + + # Writing to HTML + write_to_html(html_file, "

Total Results

\n") + write_to_html(html_file, '\n') + write_to_html( + html_file, + '\ + \n' + ) + + total_baseline = total_benchmark = total_diff = total_n_baseline = total_n_benchmark = 0 + + for test, values in test_results.items(): + baseline = values["baseline"] + benchmark = values["benchmark"] + n_baseline = values["n_baseline"] + n_benchmark = values["n_benchmark"] + + n_display = f"{n_baseline} | {n_benchmark}" if n_baseline != n_benchmark else str(n_baseline) + + diff = ((baseline - benchmark) / baseline) * 100 + + # Calculate for total row + total_baseline += baseline + total_benchmark += benchmark + total_n_baseline += n_baseline + total_n_benchmark += n_benchmark + + # Check which value is smaller for color highlighting + baseline_color = "highlight" if baseline <= benchmark else "" + benchmark_color = "highlight" if benchmark <= baseline else "" + + write_to_html( + html_file, + f'\ + \ + \n' + ) + + total_diff = ((total_baseline - total_benchmark) / total_baseline) * 100 + total_n_display = f"{total_n_baseline} | {total_n_benchmark}" if total_n_baseline != total_n_benchmark else str(total_n_baseline) + + write_to_html( + html_file, + f'\ + \ + \n' + ) + + write_to_html(html_file, "
TestNBaseline (µs)Benchmark (µs)Difference (%)
{test}{n_display}{baseline:.0f}{benchmark:.0f}{diff:.1f}
TOTAL{total_n_display}{total_baseline:.0f}{total_benchmark:.0f}{total_diff:.1f}

\n") + def generate_results_table(html_file, baseline_results, benchmark_results, filename): @@ -114,7 +207,7 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen if line.startswith("ftbench results for font") ][0] - write_to_html(html_file, "

Results for {}

\n".format(fontname)) + write_to_html(html_file, "

Results for {}

\n".format(fontname)) write_to_html(html_file, '\n') write_to_html( html_file, @@ -185,7 +278,7 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen percentage_diff, ), ) - + write_to_html( html_file, '\
TOTAL{}