From e3d8aee979d549c19ef15cce468522518b1c2842 Mon Sep 17 00:00:00 2001 From: goksu <25721443+goeksu@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:48:48 +0300 Subject: [PATCH] testing page redesigned --- builds/testing.mk | 21 ++++--- src/tools/ftbench/src/tohtml.py | 108 +++++++++++++++++--------------- 2 files changed, 70 insertions(+), 59 deletions(-) diff --git a/builds/testing.mk b/builds/testing.mk index e1e780264..1745ecefe 100644 --- a/builds/testing.mk +++ b/builds/testing.mk @@ -8,6 +8,8 @@ BASELINE = $(addprefix $(FTBENCH_DIR)/baseline/, $(notdir $(FONTS:.ttf=.txt))) BENCHMARK = $(addprefix $(FTBENCH_DIR)/benchmark/, $(notdir $(FONTS:.ttf=.txt))) BASELINE_DIR = $(FTBENCH_DIR)/baseline/ BENCHMARK_DIR = $(FTBENCH_DIR)/benchmark/ +BASELINE_INFO = $(BASELINE_DIR)info.txt +BENCHMARK_INFO = $(BENCHMARK_DIR)info.txt HTMLCREATOR = $(FTBENCH_DIR)/src/tohtml.py HTMLFILE = $(TOP_DIR)/benchmark.html @@ -25,11 +27,11 @@ $(FTBENCH_BIN): $(FTBENCH_SRC) | $(OBJ_DIR) .PHONY: baseline baseline: $(FTBENCH_BIN) $(BASELINE_DIR) @echo "Creating baseline..." + @echo "$(FTBENCH_FLAG)" > $(BASELINE_INFO) + @echo "`git -C $(TOP_DIR) rev-parse HEAD`" >> $(BASELINE_INFO) + @echo "`git -C $(TOP_DIR) show -s --format=%ci HEAD`" >> $(BASELINE_INFO) + @echo "`git -C $(TOP_DIR) rev-parse --abbrev-ref HEAD`" >> $(BASELINE_INFO) @$(foreach font, $(FONTS), \ - echo "$(FTBENCH_FLAG)" > $(BASELINE_DIR)$(notdir $(font:.ttf=.txt)); \ - echo "`git rev-parse HEAD`" >> $(BASELINE_DIR)$(notdir $(font:.ttf=.txt)); \ - echo "`git show -s --format=%ci HEAD`" >> $(BASELINE_DIR)$(notdir $(font:.ttf=.txt)); \ - echo "`git rev-parse --abbrev-ref HEAD`" >> $(BASELINE_DIR)$(notdir $(font:.ttf=.txt)); \ $(FTBENCH_BIN) $(FTBENCH_FLAG) $(font) >> $(BASELINE_DIR)$(notdir $(font:.ttf=.txt)); \ ) @echo "Baseline created." @@ -38,20 +40,19 @@ baseline: $(FTBENCH_BIN) $(BASELINE_DIR) .PHONY: benchmark benchmark: $(FTBENCH_BIN) $(BENCHMARK_DIR) @echo "Creating benchmark..." + @echo "$(FTBENCH_FLAG)" > $(BENCHMARK_INFO) + @echo "`git -C $(TOP_DIR) rev-parse HEAD`" >> $(BENCHMARK_INFO) + @echo "`git -C $(TOP_DIR) show -s --format=%ci HEAD`" >> $(BENCHMARK_INFO) + @echo "`git -C $(TOP_DIR) rev-parse --abbrev-ref HEAD`" >> $(BENCHMARK_INFO) @$(foreach font, $(FONTS), \ - echo "$(FTBENCH_FLAG)" > $(BENCHMARK_DIR)$(notdir $(font:.ttf=.txt)); \ - echo "`git rev-parse HEAD`" >> $(BENCHMARK_DIR)$(notdir $(font:.ttf=.txt)); \ - echo "`git show -s --format=%ci HEAD`" >> $(BENCHMARK_DIR)$(notdir $(font:.ttf=.txt)); \ - echo "`git rev-parse --abbrev-ref HEAD`" >> $(BENCHMARK_DIR)$(notdir $(font:.ttf=.txt)); \ $(FTBENCH_BIN) $(FTBENCH_FLAG) $(font) >> $(BENCHMARK_DIR)$(notdir $(font:.ttf=.txt)); \ ) @$(PYTHON) $(HTMLCREATOR) @echo "Benchmark created." - .PHONY: clean-benchmark clean-benchmark: @echo "Cleaning..." @$(RM) $(FTBENCH_BIN) - @$(RM) -rf $(BASELINE_DIR) $(BENCHMARK_DIR) $(HTMLFILE) + @$(RM) -rf $(BASELINE_DIR) $(BENCHMARK_DIR) $(HTMLFILE) @echo "Cleaned." diff --git a/src/tools/ftbench/src/tohtml.py b/src/tools/ftbench/src/tohtml.py index eee707947..149265fc5 100644 --- a/src/tools/ftbench/src/tohtml.py +++ b/src/tools/ftbench/src/tohtml.py @@ -3,65 +3,75 @@ import re # Create the HTML file current_dir = os.path.dirname(os.path.abspath(__file__)) - -# Get the project root directory (assuming the script is inside nested directories in the project root) project_root = os.path.abspath(os.path.join(current_dir, '../../../../')) +benchmark_html = os.path.join(project_root, 'benchmark.html') -# Construct the absolute path to the benchmark file -benchmark_file = os.path.join(project_root, 'benchmark.html') +# GitLab URL +gitlab_url = 'https://gitlab.freedesktop.org/freetype/freetype/-/commit/' -with open(benchmark_file, 'w') as f: - f.write('\n') - f.write('\n') - f.write('Benchmark Results\n') - f.write('\n') - f.write('\n') - f.write('

Benchmark Results

\n') +# Directories +baseline_dir = os.path.join(project_root, 'src/tools/ftbench/baseline') +benchmark_dir = os.path.join(project_root, 'src/tools/ftbench/benchmark') - # Traverse through the 'baseline directory - for filename in os.listdir(os.path.join(project_root, 'src/tools/ftbench/baseline')): - baseline_filepath = os.path.join(os.path.join(project_root, 'src/tools/ftbench/baseline'), filename) - benchmark_filepath = os.path.join(os.path.join(project_root, 'src/tools/ftbench/benchmark'), filename) +# Open HTML file for writing +with open(benchmark_html, 'w') as html_file: + html_file.write('\n\nBenchmark Results\n\n\n

Benchmark Results

\n') - # Process the baseline file - with open(baseline_filepath, 'r') as baseline_file: - baseline_lines = baseline_file.readlines() + # If it's the info file, we want to handle it differently + with open(os.path.join(baseline_dir, "info.txt"), 'r') as f: + baseline_info = f.readlines() + with open(os.path.join(benchmark_dir, "info.txt"), 'r') as f: + benchmark_info = f.readlines() - # Process the benchmark file - with open(benchmark_filepath, 'r') as benchmark_file: - benchmark_lines = benchmark_file.readlines() + # Check if commit ids are the same + if baseline_info[1].strip() == benchmark_info[1].strip(): + html_file.write('

Warning: Baseline and Benchmark have the same commit ID

\n') + + baseline_info[1] = '{}\n'.format(gitlab_url, baseline_info[1].strip(), baseline_info[1][:8]) + + benchmark_info[1] = '{}\n'.format(gitlab_url, benchmark_info[1].strip(), benchmark_info[1][:8]) - f.write(f'

Results for {filename}

\n') - f.write('\n') - f.write('\n') + # Write info to HTML + html_file.write('

Info

\n') + html_file.write('
TestBaselineBenchmark
\n') + html_file.write('\n') + info_list = ['Parameters', 'Commit ID', 'Commit Date', 'Branch'] + for info, baseline_line, benchmark_line in zip(info_list, baseline_info, benchmark_info): + html_file.write('\n'.format(info, baseline_line.strip(), benchmark_line.strip())) + html_file.write('
InfoBaselineBenchmark
{}{}{}

\n') - # Write the meta-data to the HTML file - f.write(f'Parameters{baseline_lines[0]}{benchmark_lines[0]}\n') - f.write(f'Commit ID{baseline_lines[1]}{benchmark_lines[1]}\n') - f.write(f'Commit Date{baseline_lines[2]}{benchmark_lines[2]}\n') - f.write(f'Branch{baseline_lines[3]}{benchmark_lines[3]}\n') + # Traverse through the 'baseline' directory + for filename in os.listdir(baseline_dir): + if filename != 'info.txt': + # If it's not the info file, it's a font file + fontname = filename.split('.')[0] + with open(os.path.join(baseline_dir, filename), 'r') as f: + baseline_results = f.readlines() + with open(os.path.join(benchmark_dir, filename), 'r') as f: + benchmark_results = f.readlines() - # For each line in the baseline and benchmark files - for baseline_line, benchmark_line in zip(baseline_lines[4:], benchmark_lines[4:]): - # If the line starts with a space, it's a test result line - if baseline_line.startswith(' '): - # Extract the test name, the time per operation, and the number of operations done - baseline_match = re.match(r' (\w+(\s\(\w+\))?)\s+(\d+\.\d+)\s', baseline_line) - benchmark_match = re.match(r' (\w+(\s\(\w+\))?)\s+(\d+\.\d+)\s', benchmark_line) + # Write results to HTML + html_file.write('

Results for {}

\n'.format(fontname)) + html_file.write('\n') + html_file.write('\n') - # If the line could be parsed - if baseline_match and benchmark_match: - # Check which value is higher - baseline_value = float(baseline_match.group(3)) - benchmark_value = float(benchmark_match.group(3)) + for baseline_line, benchmark_line in zip(baseline_results, benchmark_results): + if baseline_line.startswith(' '): + baseline_match = re.match(r' (\w+(\s\(\w+\))?)\s+(\d+\.\d+)\s', baseline_line) + benchmark_match = re.match(r' (\w+(\s\(\w+\))?)\s+(\d+\.\d+)\s', benchmark_line) - # Write the test result to the HTML file - if baseline_value > benchmark_value: - f.write(f'\n') - else: - f.write(f'\n') + if baseline_match and benchmark_match: + baseline_value = float(baseline_match.group(3)) + benchmark_value = float(benchmark_match.group(3)) - f.write('
Test Baseline Benchmark Difference
{baseline_match.group(1)}{baseline_match.group(3)} µs/op{benchmark_match.group(3)} µs/op
{baseline_match.group(1)}{baseline_match.group(3)} µs/op{benchmark_match.group(3)} µs/op
\n') + # Calculate the percentage difference + percentage_diff = ((baseline_value - benchmark_value) / baseline_value) * 100 - f.write('\n') - f.write('\n') + if baseline_value > benchmark_value: + html_file.write('{}{:.2f}\tus/op{:.2f}\tus/op{:.2f}%\n'.format(baseline_match.group(1), baseline_value, benchmark_value, percentage_diff)) + else: + html_file.write('{}{:.2f}\tus/op{:.2f}\tus/op{:.2f}%\n'.format(baseline_match.group(1), baseline_value, benchmark_value, percentage_diff)) + + html_file.write('
\n') + + html_file.write('
Freetype Benchmark\n\n')