From c3254394f3048aadbfdc2d3723a2fbc8baec6f6d Mon Sep 17 00:00:00 2001 From: goksu <25721443+goeksu@users.noreply.github.com> Date: Mon, 7 Aug 2023 15:07:19 +0300 Subject: [PATCH] warmup iteration instead time --- builds/testing.mk | 2 +- src/tools/ftbench/ftbench.c | 74 ++++++++++++++++----------------- src/tools/ftbench/src/tohtml.py | 16 +++---- 3 files changed, 45 insertions(+), 47 deletions(-) diff --git a/builds/testing.mk b/builds/testing.mk index ace79129f..b5e79b07a 100644 --- a/builds/testing.mk +++ b/builds/testing.mk @@ -3,7 +3,7 @@ FTBENCH_DIR = $(TOP_DIR)/src/tools/ftbench FTBENCH_SRC = $(FTBENCH_DIR)/ftbench.c FTBENCH_OBJ = $(OBJ_DIR)/bench.$(SO) FTBENCH_BIN = $(OBJ_DIR)/bench$E -FTBENCH_FLAG ?= -c 500 -w 1 +FTBENCH_FLAG ?= -c 1000 -w 100 INCLUDES = $(TOP_DIR)/include FONTS = $(wildcard $(FTBENCH_DIR)/fonts/*.ttf) BASELINE_DIR = $(OBJ_DIR)/baseline/ diff --git a/src/tools/ftbench/ftbench.c b/src/tools/ftbench/ftbench.c index ed2eea916..3b5e602ec 100644 --- a/src/tools/ftbench/ftbench.c +++ b/src/tools/ftbench/ftbench.c @@ -98,7 +98,7 @@ #define CACHE_SIZE 1024 #define BENCH_TIME 2.0 -#define WARMUP_TIME 1.0 +#define WARMUP_ITER 0 #define FACE_SIZE 10 @@ -275,7 +275,7 @@ int n, done; btimer_t timer, elapsed; - double warmup_time = warmup * 1E6; // Convert to microseconds + if ( test->cache_first ) { @@ -293,8 +293,12 @@ for ( n = 0, done = 0; !max_iter || n < max_iter; n++ ) { - if ( is_warmup ) - n = 0; + if ( is_warmup && n == warmup ){ + is_warmup = 0; + TIMER_RESET( &timer ); + TIMER_RESET( &elapsed ); + } + TIMER_START( &elapsed ); @@ -302,21 +306,14 @@ TIMER_STOP( &elapsed ); - if ( is_warmup && TIMER_GET( &elapsed ) > warmup_time ) - { - is_warmup = 0; - done = 0; - TIMER_RESET( &timer ); - TIMER_RESET( &elapsed ); - } - - else if ( !is_warmup && TIMER_GET( &elapsed ) > 1E6 * max_time ) + + if (!is_warmup && TIMER_GET( &elapsed ) > 1E6 * max_time ) break; } if ( done ) - printf( "%10.3f ms %10d done\n", - TIMER_GET( &timer ) / 1000000, done ); + printf( "%10.1f ms %10d done\n", + TIMER_GET( &timer ), done ); else printf( "no error-free calls\n" ); } @@ -1117,7 +1114,7 @@ unsigned int size = FACE_SIZE; int max_iter = 0; double max_time = BENCH_TIME; - double warmup_time = WARMUP_TIME; + int warmup_iter = WARMUP_ITER; int j; unsigned int versions[2] = { TT_INTERPRETER_VERSION_35, @@ -1360,10 +1357,11 @@ break; case 'w': - warmup_time = atof( optarg ); - if ( warmup_time < 0 ) - warmup_time = -warmup_time; + warmup_iter = atoi( optarg ); + if ( warmup_iter < 0 ) + warmup_iter = -warmup_iter; break; + /* break; */ @@ -1454,8 +1452,8 @@ if ( max_iter ) printf( "number of iterations for each test: at most %d\n", max_iter ); - printf( "number of seconds as warmup time before all tests: %.0f\n", - warmup_time ); + printf( "number of iteration as warmup in all tests: %d\n", + warmup_iter ); printf( "\n" "executing tests:\n" ); @@ -1479,7 +1477,7 @@ case FT_BENCH_LOAD_GLYPH: test.title = "Load"; test.bench = test_load; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); if ( cache_man ) { @@ -1489,7 +1487,7 @@ { test.title = "Load (image cached)"; test.bench = test_image_cache; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); } if ( !FTC_SBitCache_New( cache_man, &sbit_cache ) ) @@ -1497,7 +1495,7 @@ test.title = "Load (sbit cached)"; test.bench = test_sbit_cache; if ( size ) - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); else printf( " %-25s disabled (size = 0)\n", test.title ); } @@ -1510,24 +1508,24 @@ test.title = "Load_Advances (Normal)"; test.bench = test_load_advances; flags = FT_LOAD_DEFAULT; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); test.title = "Load_Advances (Fast)"; test.bench = test_load_advances; flags = FT_LOAD_TARGET_LIGHT; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); test.title = "Load_Advances (Unscaled)"; test.bench = test_load_advances; flags = FT_LOAD_NO_SCALE; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); break; case FT_BENCH_RENDER: test.title = "Render"; test.bench = test_render; if ( size ) - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); else printf( " %-25s disabled (size = 0)\n", test.title ); break; @@ -1535,13 +1533,13 @@ case FT_BENCH_GET_GLYPH: test.title = "Get_Glyph"; test.bench = test_get_glyph; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); break; case FT_BENCH_GET_CBOX: test.title = "Get_CBox"; test.bench = test_get_cbox; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); break; case FT_BENCH_GET_BBOX: @@ -1553,7 +1551,7 @@ /* rotate outlines by 30 degrees so that CBox and BBox differ */ FT_Set_Transform( face, &rot30, NULL ); - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); FT_Set_Transform( face, NULL, NULL ); } break; @@ -1572,7 +1570,7 @@ test.title = "Get_Char_Index"; test.bench = test_get_char_index; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); if ( cache_man && !FTC_CMapCache_New( cache_man, &cmap_cache ) ) @@ -1581,7 +1579,7 @@ test.title = "Get_Char_Index (cached)"; test.bench = test_cmap_cache; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); } free( charset.code ); @@ -1592,20 +1590,20 @@ case FT_BENCH_CMAP_ITER: test.title = "Iterate CMap"; test.bench = test_cmap_iter; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); break; case FT_BENCH_NEW_FACE: test.title = "New_Face"; test.bench = test_new_face; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); break; case FT_BENCH_EMBOLDEN: test.title = "Embolden"; test.bench = test_embolden; if ( size ) - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); else printf( " %-25s disabled (size = 0)\n", test.title ); break; @@ -1614,7 +1612,7 @@ test.title = "Stroke"; test.bench = test_stroke; if ( size ) - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); else printf( " %-25s disabled (size = 0)\n", test.title ); break; @@ -1622,7 +1620,7 @@ case FT_BENCH_NEW_FACE_AND_LOAD_GLYPH: test.title = "New_Face & load glyph(s)"; test.bench = test_new_face_and_load_glyph; - benchmark( face, &test, max_iter, max_time, warmup_time ); + benchmark( face, &test, max_iter, max_time, warmup_iter ); break; } } diff --git a/src/tools/ftbench/src/tohtml.py b/src/tools/ftbench/src/tohtml.py index 85e221a24..c92af37be 100644 --- a/src/tools/ftbench/src/tohtml.py +++ b/src/tools/ftbench/src/tohtml.py @@ -103,7 +103,7 @@ def generate_info_table(html_file, baseline_info, benchmark_info): ), ) write_to_html(html_file, "
") - write_to_html(html_file, "* Smaller values mean faster operation
\n") + write_to_html(html_file, "* Cumulative time for iterations which is better in smaller values
\n") def generate_results_table(html_file, baseline_results, benchmark_results, filename): @@ -119,8 +119,8 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen write_to_html( html_file, 'TestN\ - * Baseline (ms)\ - * Benchmark (ms)\ + * Baseline (µs)\ + * Benchmark (µs)\ Difference (%)\n'.format( os.path.join(BASELINE_DIR, filename[:-4]), os.path.join(BENCHMARK_DIR, filename[:-4]), @@ -159,13 +159,13 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen total_n += int(benchmark_n) total_time_baseline += baseline_value total_time_benchmark += benchmark_value - total_difference += percentage_diff + if baseline_value > benchmark_value: write_to_html( html_file, '{}{}\ - {:.3f}{:.3f}{:.1f}\n'.format( + {:.0f}{:.0f}{:.1f}\n'.format( baseline_match.group(1), n, baseline_value, @@ -177,7 +177,7 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen write_to_html( html_file, '{}{}\ - {:.3f}{:.3f}{:.1f}\n'.format( + {:.0f}{:.0f}{:.1f}\n'.format( baseline_match.group(1), n, baseline_value, @@ -189,8 +189,8 @@ def generate_results_table(html_file, baseline_results, benchmark_results, filen write_to_html( html_file, 'TOTAL{}\ - {:.3f}{:.3f}{:.1f}\n'.format( - total_n, total_time_baseline, total_time_benchmark, total_difference / 14 + {:.0f}{:.0f}{:.1f}\n'.format( + total_n, total_time_baseline, total_time_benchmark, (total_time_baseline - total_time_benchmark) / total_time_baseline * -100 ), ) write_to_html(html_file, "
\n")