ci: Migrate tracie tests done in shell script to pytest

v2: Verbatim translation from the original shell script
    Make the corrections visible in explicit commits (Andres)
    Remove redundant code (Alexandros)
    Code style nitpick (Rohan)

Reimplementation of the tracie's self-tests using a pythonic test suit
(pytest).

The new tracie/test.py module is almost a direct translation of the
tests defined in the tracie/test.sh. This new implementation of the
test provides a more common framework where define the tests.
Also allows a better introspection for the tests results and/or
resulting errors.

This patch also adds python3-pytest as dependency for the built images
and adapts the tracie-runner scripts to run the self-test using pytest.

Signed-off-by: Pablo Saavedra <psaavedra@igalia.com>
Reviewed-by: Alexandros Frantzis <alexandros.frantzis@collabora.com> [v1]
Reviewed-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Rohan Garg <rohan.garg@collabora.com> [v1]
(cherry picked from commit 550a4f7764)

Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/5548>

.gitlab-ci.yml

@@ -136,7 +136,7 @@ x86_build:
     - .fdo.container-build@debian
     - .container
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_build "2020-05-08"
+    FDO_DISTRIBUTION_TAG: &x86_build "2020-05-18"
 
 .use-x86_build:
   variables:
@@ -149,13 +149,13 @@ x86_build:
 x86_test-gl:
   extends: x86_build
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-08"
+    FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-18"
 
 # Debian 10 based x86 test image for VK
 x86_test-vk:
   extends: x86_build
   variables:
-    FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-08"
+    FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-18"
 
 # Debian 9 based x86 build image (old LLVM)
 x86_build_old:

.gitlab-ci/container/x86_test-gl.sh

@@ -64,6 +64,7 @@ apt-get install -y --no-remove \
       python3-mako \
       python3-numpy \
       python3-pil \
+      python3-pytest \
       python3-requests \
       python3-six \
       python3-yaml \

.gitlab-ci/container/x86_test-vk.sh

@@ -64,6 +64,7 @@ apt-get install -y --no-remove \
       python \
       python3-distutils \
       python3-pil \
+      python3-pytest \
       python3-requests \
       python3-yaml \
       vulkan-tools \

.gitlab-ci/tracie-runner-gl.sh

@@ -26,7 +26,7 @@ RESULTS=`pwd`/results
 mkdir -p $RESULTS
 
 # Perform a self-test to ensure tracie is working properly.
-"$INSTALL/tracie/tests/test.sh"
+python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
 
 if [ "$GALLIUM_DRIVER" = "virpipe" ]; then
     # tracie is to use virpipe, and virgl_test_server llvmpipe

.gitlab-ci/tracie-runner-vk.sh

@@ -21,7 +21,7 @@ export DXVK_LOG_LEVEL="none"
 export DXVK_STATE_CACHE=0
 
 # Perform a self-test to ensure tracie is working properly.
-"$INSTALL/tracie/tests/test.sh"
+python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
 
 # Sanity check to ensure that our environment is sufficient to make our tests
 # run against the Mesa built by CI, rather than any installed distro version.

.gitlab-ci/tracie/tests/test.py

@@ -0,0 +1,186 @@
+import logging
+import pytest
+import re
+import shutil
+
+from os import environ, chdir
+from os.path import dirname, exists, realpath
+
+import tracie
+
+
+RESULTS_YAML = "results/results.yml"
+TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
+TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
+TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
+TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png"
+TRACIE_DIR = dirname(realpath(__file__)) + "/.."
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def write_to(content, filename):
+    with open(filename, 'w') as f:
+        f.write(content)
+
+
+def read_from(filename):
+    with open(filename) as f:
+        content = f.read()
+    return content
+
+
+def run_tracie():
+    '''
+    Run tests for the .testtrace types, using the "gl-test-device" and
+    "vk-test-device" device names.
+    '''
+    result = tracie.main(["--device-name", "gl-test-device",
+                          "--file", "./tests/traces.yml"])
+    if not result:
+        return False
+    result = tracie.main(["--device-name", "vk-test-device",
+                          "--file", "./tests/traces.yml"])
+    return result
+
+
+def prepare_for_run(tmp_path):
+    '''
+    Copy all the tracie scripts to the test dir for the unit tests.
+    This avoids polluting the normal working dir with test result artifacts.
+    '''
+    test_dir = str(tmp_path) + "/run"
+    shutil.copytree(TRACIE_DIR, test_dir)
+    # Change the working dir to the test_dir
+    chdir(test_dir)
+    # Set the traces-db
+    shutil.move("./tests/test-data", "./traces-db")
+    # Disable trace storing
+    environ["TRACIE_STORE_IMAGES"] = "0"
+
+
+def cleanup(tmp_path):
+    '''
+    Performs the clean up of the test dir.
+    '''
+    if exists(tmp_path):
+        shutil.rmtree(tmp_path)
+
+
+@pytest.fixture(autouse=True)
+def run_test(tmp_path):
+    '''
+    Wraps the execution of each test as follows:
+
+      prepare_for_run()
+      test()
+      cleanup()
+    '''
+    logger.debug("Working dir: %s", tmp_path)
+    prepare_for_run(tmp_path)
+    yield
+    cleanup(tmp_path)
+
+
+def check_results_yaml_content(filename, expectations):
+    '''
+    Checks the content of the filename with the list of expectations
+    passed as parameter.
+
+    Arguments:
+        filename (str): The path of the file to check
+        expectations (list): A list with the content to find in the file
+
+    Returns:
+        bool: The return value. True if the content of the filename satisfies
+              the expectations, False otherwise.
+    '''
+    content = read_from(filename)
+    for e in expectations:
+        ocurrencies = re.findall(e, content)
+        if not len(ocurrencies):
+            logger.error("Expectation not found in %s: %s", filename, e)
+            return False
+    return True
+
+
+def test_tracie_succeeds_if_all_images_match():
+    assert run_tracie()
+    expectations = [
+        "actual: 5efda83854befe0155ff8517a58d5b51",
+        "expected: 5efda83854befe0155ff8517a58d5b51",
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_fails_on_image_mismatch():
+    filename = "./tests/traces.yml"
+    content = read_from(filename)
+    content = content.replace("5efda83854befe0155ff8517a58d5b51",
+                              "8e0a801367e1714463475a824dab363b")
+    write_to(content, filename)
+    assert not run_tracie()
+    expectations = [
+        "actual: 5efda83854befe0155ff8517a58d5b51",
+        "expected: 8e0a801367e1714463475a824dab363b",
+        "trace2/test/vk-test-device/olive.testtrace-0.png"
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_skips_traces_without_checksum():
+    filename = "./tests/traces.yml"
+    content = '''traces:
+  - path: trace1/red.testtrace
+    expectations:
+    - device: bla
+      checksum: 000000000000000'''
+    write_to(content, filename)
+
+    # red.testtrace should be skipped, since it doesn't
+    # have any checksums for our device
+    filename = "./traces-db/trace1/red.testtrace"
+    content = "ff0000ff"
+    write_to(content, filename)
+    assert run_tracie()
+
+
+def test_tracie_fails_on_dump_image_error():
+    # "invalid" should fail to parse as rgba and
+    # cause an error
+    filename = "./traces-db/trace1/magenta.testtrace"
+    write_to("invalid\n", filename)
+    run_tracie()
+    expectations = [
+        "actual: error",
+        "expected: 8e0a801367e1714463475a824dab363b",
+        "trace1/magenta.testtrace",
+    ]
+    assert check_results_yaml_content(RESULTS_YAML, expectations)
+
+
+def test_tracie_stores_only_logs_on_checksum_match():
+    assert run_tracie()
+    assert exists(TRACE_LOG_TEST1)
+    assert exists(TRACE_LOG_TEST2)
+    assert not exists(TRACE_PNG_TEST1)
+    assert not exists(TRACE_PNG_TEST2)
+
+
+def test_tracie_stores_images_on_checksum_mismatch():
+    filename = "./tests/traces.yml"
+    content = read_from(filename)
+    content = content.replace("5efda83854befe0155ff8517a58d5b51",
+                              "8e0a801367e1714463475a824dab363b")
+    write_to(content, filename)
+    assert not run_tracie()
+    assert not exists(TRACE_PNG_TEST1)
+    assert exists(TRACE_PNG_TEST2)
+
+
+def test_tracie_stores_images_on_request():
+    environ["TRACIE_STORE_IMAGES"] = "1"
+    assert run_tracie()
+    assert exists(TRACE_PNG_TEST1)
+    assert exists(TRACE_PNG_TEST2)

.gitlab-ci/tracie/tests/test.sh

@@ -1,154 +0,0 @@
-#!/bin/sh
-
-TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.."
-TEST_DIR=""
-TEST_EXIT=0
-
-assert() {
-    if ! $1; then
-        echo "Assertion failed:  \"$1\""
-        exit 1
-    fi
-}
-
-run_tracie() {
-    # Run tests for the .testtrace types, using the "gl-test-device" and "vk-test-device" device names.
-    python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name gl-test-device && \
-    python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name vk-test-device
-}
-
-cleanup() {
-    [ "$TEST_DIR" = "/tmp/*" ] && rm -rf "$TEST_DIR"
-}
-
-prepare_for_run() {
-    TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)"
-    # Copy all the tracie scripts to the test dir for the run-tests.sh script.
-    # This avoids polluting the normal working dir with test result artifacts.
-    cp -R "$TRACIE_DIR"/. "$TEST_DIR"
-    cd "$TEST_DIR"
-    mkdir traces-db
-    mv tests/test-data/* traces-db/.
-    trap cleanup EXIT
-    # Ensure we have a clean environment.
-    unset TRACIE_STORE_IMAGES
-}
-
-run_test() {
-    prepare_for_run
-    log=$(mktemp)
-    if ($1 > "$log" 2>&1 ;); then
-        if [ -t 1 ]; then
-            echo "$1: \e[0;32mSuccess\e[0m"
-        else
-            echo "$1: Success"
-        fi
-    else
-        if [ -t 1 ]; then
-            echo "$1: \e[0;31mFail\e[0m"
-        else
-            echo "$1: Fail"
-        fi
-        cat "$log"
-        TEST_EXIT=1
-    fi
-    rm "$log"
-    cleanup
-}
-
-assert_results_yaml_contains() {
-    grep -q "actual: $1" $4
-    assert "[ $? = 0 ]"
-
-    grep -q "expected: $2" $4
-    assert "[ $? = 0 ]"
-
-    if [ $3 != "" ]; then
-        grep -q $3 $4
-    fi
-
-    assert "[ $? = 0 ]"
-}
-
-tracie_succeeds_if_all_images_match() {
-    run_tracie
-    assert "[ $? = 0 ]"
-    assert_results_yaml_contains 5efda83854befe0155ff8517a58d5b51 5efda83854befe0155ff8517a58d5b51 "" "$PWD/results/results.yml"
-}
-
-tracie_fails_on_image_mismatch() {
-    sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
-        "$TEST_DIR/tests/traces.yml"
-
-    run_tracie
-    assert "[ $? != 0 ]"
-    assert_results_yaml_contains 5efda83854befe0155ff8517a58d5b51 8e0a801367e1714463475a824dab363b "trace2/test/vk-test-device/olive.testtrace-0.png" "$PWD/results/results.yml"
-}
-
-tracie_skips_traces_without_checksum() {
-    echo "traces:" > "$TEST_DIR/tests/traces.yml"
-    echo "  - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml"
-    echo "    expectations:" >> "$TEST_DIR/tests/traces.yml"
-    echo "    - device: bla" >> "$TEST_DIR/tests/traces.yml"
-    echo "      checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml"
-    # red.testtrace should be skipped, since it doesn't
-    # have any checksums for our device
-    echo "ff0000ff" > traces-db/trace1/red.testtrace
-
-    run_tracie
-    assert "[ $? = 0 ]"
-}
-
-tracie_fails_on_dump_image_error() {
-    # "invalid" should fail to parse as rgba and
-    # cause an error
-    echo "invalid" > traces-db/trace1/magenta.testtrace
-
-    run_tracie
-    assert "[ $? != 0 ]"
-    assert_results_yaml_contains "error" "8e0a801367e1714463475a824dab363b" "trace1/magenta.testtrace" "$PWD/results/results.yml"
-}
-
-tracie_stores_only_logs_on_checksum_match() {
-    run_tracie
-    assert "[ $? = 0 ]"
-
-    assert "[ -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace.log" ]"
-    assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace.log" ]"
-
-    assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
-    assert "[ ! -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
-
-    ls -lR "$TEST_DIR"
-}
-
-tracie_stores_images_on_checksum_mismatch() {
-    sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
-        "$TEST_DIR/tests/traces.yml"
-
-    run_tracie
-    assert "[ $? != 0 ]"
-
-    assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
-    assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
-}
-
-tracie_stores_images_on_request() {
-    (export TRACIE_STORE_IMAGES=1; run_tracie)
-    assert "[ $? = 0 ]"
-
-    assert "[ -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
-    assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
-
-    ls -lR "$TEST_DIR"
-}
-
-run_test tracie_succeeds_if_all_images_match
-run_test tracie_fails_on_image_mismatch
-run_test tracie_skips_traces_without_checksum
-run_test tracie_fails_on_dump_image_error
-run_test tracie_stores_only_logs_on_checksum_match
-run_test tracie_stores_images_on_checksum_mismatch
-run_test tracie_stores_images_on_request
-
-exit $TEST_EXIT