ci: add piglit replay jobs and remove tracie ones

Along the way, modify the piglit run script and refactor the way
piglit jobs are generated.

v2:
  - Squashed the commit to remove tracie jobs (Eric).

v3:
  - Extend information in the comments about the need to use a
    running X server for replaying with Vulkan (Tomeu).
  - Do actually fail if the upload doesn't work (Tomeu).

v4:
  - Rename *-piglit-traces jobs with *-traces.

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Tomeu Vizoso <tomeu.vizoso@collabora.com> [v3]
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6388>

.gitlab-ci.yml

@@ -1008,36 +1008,65 @@ meson-mingw32-x86_64:
     - meson-clover-testing
 
 .piglit-test:
-  extends:
-    - .test-gl
-    - .llvmpipe-rules
   artifacts:
     when: on_failure
     name: "mesa_${CI_JOB_NAME}"
     paths:
       - summary/
   variables:
-    LIBGL_ALWAYS_SOFTWARE: 1
     PIGLIT_NO_WINDOW: 1
   script:
     - install/piglit/run.sh
 
-llvmpipe-piglit-cl:
+.piglit-traces-test:
   extends:
     - .piglit-test
+  cache:
+    key: ${CI_JOB_NAME}
+    paths:
+      - replayer-db/
+  artifacts:
+    when: on_failure
+    name: "mesa_${CI_JOB_NAME}"
+    reports:
+      junit: results/junit.xml
+  variables:
+    PIGLIT_PROFILES: replay
+    PIGLIT_REPLAY_UPLOAD_TO_MINIO: 1
+    PIGLIT_REPLAY_EXTRA_ARGS:  --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/
+    PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL: "/mesa-tracie-results/$CI_PROJECT_PATH"
+    PIGLIT_REPLAY_ARTIFACTS_BASE_URL: "/artifacts/$CI_PROJECT_PATH/$CI_PIPELINE_ID/$CI_JOB_ID"
+    PIGLIT_HTML_SUMMARY: 0
+    PIGLIT_JUNIT_RESULTS: 1
+
+.llvmpipe-test:
+  extends:
+    - .llvmpipe-rules
+  variables:
+    LIBGL_ALWAYS_SOFTWARE: "true"
+    GALLIUM_DRIVER: "llvmpipe"
+
+llvmpipe-piglit-cl:
+  extends:
     - .test-cl
+    - .piglit-test
+    - .llvmpipe-test
     - .llvmpipe-cl-rules
   variables:
     LP_CL: 1
     LP_NUM_THREADS: 1
     PIGLIT_PROFILES: cl
+    PIGLIT_RESULTS: "llvmpipe-cl"
     PIGLIT_OPTIONS: >
       -x bswap -x phatk -x clz-optimizations
   script:
     - install/piglit/run_cl.sh
 
 llvmpipe-piglit-quick_gl:
-  extends: .piglit-test
+  extends:
+    - .test-gl
+    - .piglit-test
+    - .llvmpipe-test
   variables:
     LP_NUM_THREADS: 0
     PIGLIT_OPTIONS: >
@@ -1050,18 +1079,53 @@ llvmpipe-piglit-quick_gl:
       -x max-texture-size
       -x maxsize
     PIGLIT_PROFILES: quick_gl
+    PIGLIT_RESULTS: "llvmpipe-quick_gl"
 
 llvmpipe-piglit-glslparser:
-  extends: .piglit-test
+  extends:
+    - .test-gl
+    - .piglit-test
+    - .llvmpipe-test
   variables:
     LP_NUM_THREADS: 0
     PIGLIT_PROFILES: glslparser
+    PIGLIT_RESULTS: "llvmpipe-glslparser"
 
 llvmpipe-piglit-quick_shader:
-  extends: .piglit-test
+  extends:
+    - .test-gl
+    - .piglit-test
+    - .llvmpipe-test
   variables:
     LP_NUM_THREADS: 1
     PIGLIT_PROFILES: quick_shader
+    PIGLIT_RESULTS: "llvmpipe-quick_shader"
+
+llvmpipe-traces:
+  extends:
+    - .test-gl
+    - .piglit-traces-test
+    - .llvmpipe-test
+  variables:
+    EGL_PLATFORM: "surfaceless"
+    PIGLIT_REPLAY_DESCRIPTION_FILE: "${CI_PROJECT_DIR}/install/traces-llvmpipe.yml"
+    PIGLIT_REPLAY_DEVICE_NAME: "gl-vmware-llvmpipe"
+    PIGLIT_RESULTS: "llvmpipe-replay"
+
+virgl-traces:
+  extends:
+    - .test-gl
+    - .piglit-traces-test
+    - .virgl-rules
+  variables:
+    LIBGL_ALWAYS_SOFTWARE: "true"
+    GALLIUM_DRIVER: "virpipe"
+    EGL_PLATFORM: "surfaceless"
+    PIGLIT_REPLAY_DESCRIPTION_FILE: "${CI_PROJECT_DIR}/install/traces-virgl.yml"
+    PIGLIT_REPLAY_DEVICE_NAME: "gl-virgl"
+    PIGLIT_RESULTS: "virgl-replay"
+    MESA_GLES_VERSION_OVERRIDE: "3.1"
+    MESA_GLSL_VERSION_OVERRIDE: "310"
 
 .deqp-test:
   script:
@@ -1474,13 +1538,6 @@ radv-fossils:
     reports:
       junit: results/junit.xml
 
-.traces-test-gl:
-  extends:
-    - .test-gl
-    - .traces-test
-  script:
-    - ./install/tracie-runner-gl.sh
-
 .traces-test-vk:
   extends:
     - .test-vk
@@ -1488,16 +1545,6 @@ radv-fossils:
   script:
     - ./install/tracie-runner-vk.sh
 
-llvmpipe-traces:
-  extends:
-    - .traces-test-gl
-    - .llvmpipe-rules
-  variables:
-    LIBGL_ALWAYS_SOFTWARE: "true"
-    GALLIUM_DRIVER: "llvmpipe"
-    DEVICE_NAME: "gl-vmware-llvmpipe"
-    DRIVER_NAME: "llvmpipe"
-
 radv-polaris10-traces:
   extends:
     - .traces-test-vk
@@ -1519,15 +1566,3 @@ radv-raven-traces:
     DRIVER_NAME: "radv"
   tags:
     - raven
-
-virgl-traces:
-  extends:
-    - .traces-test-gl
-    - .virgl-rules
-  variables:
-    LIBGL_ALWAYS_SOFTWARE: "true"
-    GALLIUM_DRIVER: "virpipe"
-    DEVICE_NAME: "gl-virgl"
-    DRIVER_NAME: "virgl"
-    MESA_GLES_VERSION_OVERRIDE: "3.1"
-    MESA_GLSL_VERSION_OVERRIDE: "310"

.gitlab-ci/piglit/llvmpipe-quick_gl.txt

@@ -464,8 +464,7 @@ spec/arb_depth_buffer_float/depthstencil-render-miplevels 585 d=z32f_s=z24_s8: s
 spec/arb_depth_buffer_float/depthstencil-render-miplevels 585 s=z24_s8_d=z32f: skip
 spec/arb_depth_buffer_float/depthstencil-render-miplevels 585 s=z24_s8_d=z32f_s8: skip
 spec/arb_direct_state_access/getcompressedtextureimage: skip
-spec/arb_direct_state_access/gettextureimage-formats: skip
+spec/arb_direct_state_access/gettextureimage-formats: crash
-spec/arb_direct_state_access/gettextureimage-formats init-by-rendering: skip
 spec/arb_fragment_program/fp-indirections: skip
 spec/arb_fragment_shader_interlock/arb_fragment_shader_interlock-image-load-store: skip
 spec/arb_framebuffer_no_attachments/arb_framebuffer_no_attachments-params/dsa: skip
@@ -915,6 +914,7 @@ spec/ext_direct_state_access/named-buffers 15/namedcopybuffersubdataext: skip
 spec/ext_external_objects/memory-object-api-errors: skip
 spec/ext_external_objects/semaphore-api-errors: skip
 spec/ext_external_objects/vk-buf-exchange: skip
+spec/ext_external_objects/vk-depth-display: skip
 spec/ext_external_objects/vk-image-display: skip
 spec/ext_external_objects/vk-image-display-overwrite: skip
 spec/ext_external_objects/vk-image-overwrite: skip
@@ -1661,10 +1661,10 @@ wgl/wgl-sanity: skip
 summary:
        name:  results
        ----  --------
-       pass:    23082
+       pass:    23099
        fail:      197
-      crash:        0
+      crash:        1
-       skip:     1439
+       skip:     1438
     timeout:        0
        warn:        6
  incomplete:        0
@@ -1673,4 +1673,4 @@ summary:
     changes:        0
       fixes:        0
 regressions:        0
-      total:    24742
+      total:    24759

.gitlab-ci/piglit/llvmpipe-quick_shader.txt

@@ -379,6 +379,8 @@ spec/glsl-1.50/execution/compatibility/gs-texcoord-array-2: skip
 spec/glsl-1.50/execution/compatibility/vs-gs-ff-frag: skip
 spec/glsl-1.50/execution/compatibility/vs-gs-texcoord-array: skip
 spec/glsl-1.50/execution/compatibility/vs-gs-texcoord-array-2: skip
+spec/glsl-1.50/execution/primitive-id-no-gs-quad-strip: skip
+spec/glsl-1.50/execution/primitive-id-no-gs-quads: skip
 spec/glsl-4.00/execution/conversion/frag-conversion-explicit-dmat2-mat2: fail
 spec/glsl-4.00/execution/conversion/frag-conversion-explicit-dmat2x3-mat2x3: fail
 spec/glsl-4.00/execution/conversion/frag-conversion-explicit-dmat2x4-mat2x4: fail
@@ -596,10 +598,10 @@ spec/nv_viewport_swizzle/viewport_swizzle: skip
 summary:
        name:  results
        ----  --------
-       pass:    15808
+       pass:    15819
        fail:       83
       crash:      170
-       skip:      342
+       skip:      344
     timeout:        0
        warn:        0
  incomplete:        0
@@ -608,4 +610,4 @@ summary:
     changes:        0
       fixes:        0
 regressions:        0
-      total:    16403
+      total:    16416

.gitlab-ci/piglit/llvmpipe-replay.txt

@@ -0,0 +1,16 @@
+summary:
+       name:  results
+       ----  --------
+       pass:       43
+       fail:        0
+      crash:        0
+       skip:        0
+    timeout:        0
+       warn:        0
+ incomplete:        0
+ dmesg-warn:        0
+ dmesg-fail:        0
+    changes:        0
+      fixes:        0
+regressions:        0
+      total:       43

.gitlab-ci/piglit/run.sh

@@ -1,36 +1,200 @@
-#!/bin/bash
+#!/bin/sh
 
-set -e
+set -ex
-set -o xtrace
 
-VERSION=`cat install/VERSION`
+INSTALL="$(pwd)/install"
+
+RESULTS="$(pwd)/results"
+mkdir -p "$RESULTS"
+
+# Set up the driver environment.
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$INSTALL/lib/"
+
+# Sanity check to ensure that our environment is sufficient to make our tests
+# run against the Mesa built by CI, rather than any installed distro version.
+MESA_VERSION=$(cat "$INSTALL/VERSION" | sed 's/\./\\./g')
+
+if [ "$VK_DRIVER" ]; then
+
+    ### VULKAN ###
+
+    # Set the Vulkan driver to use.
+    export VK_ICD_FILENAMES="$INSTALL/share/vulkan/icd.d/${VK_DRIVER}_icd.x86_64.json"
+
+    if [ "x$PIGLIT_PROFILES" = "xreplay" ]; then
+        # Set environment for VulkanTools' VK_LAYER_LUNARG_screenshot layer.
+        export VK_LAYER_PATH="$VK_LAYER_PATH:/VulkanTools/build/etc/vulkan/explicit_layer.d"
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/VulkanTools/build/lib"
+
+        # Set environment for Wine.
+        export WINEDEBUG="-all"
+        export WINEPREFIX="/dxvk-wine64"
+        export WINEESYNC=1
+
+        # Set environment for DXVK.
+        export DXVK_LOG_LEVEL="none"
+        export DXVK_STATE_CACHE=0
+
+        # Set environment for gfxreconstruct executables.
+        export PATH="/gfxreconstruct/build/bin:$PATH"
+    fi
+
+    SANITY_MESA_VERSION_CMD="vulkaninfo"
+
+
+    # Set up the Window System Interface (WSI)
+
+    # IMPORTANT:
+    #
+    # Nothing to do here.
+    #
+    # Run vulkan against the host's running X server (xvfb doesn't
+    # have DRI3 support).
+    # Set the DISPLAY env variable in each gitlab-runner's
+    # configuration file:
+    # https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section
+else
+
+    ### GL/ES ###
+
+    if [ "x$PIGLIT_PROFILES" = "xreplay" ]; then
+        # Set environment for renderdoc libraries.
+        export PYTHONPATH="$PYTHONPATH:/renderdoc/build/lib"
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/renderdoc/build/lib"
+
+        # Set environment for apitrace executable.
+        export PATH="/apitrace/build:$PATH"
+
+        # Our rootfs may not have "less", which apitrace uses during
+        # apitrace dump
+        export PAGER=cat
+    fi
+
+    SANITY_MESA_VERSION_CMD="wflinfo"
+
+
+    # Set up the platform windowing system.
+
+    if [ "x$EGL_PLATFORM" = "xsurfaceless" ]; then
+
+        # Set environment for the waffle library.
+        export LD_LIBRARY_PATH="/waffle/build/lib:$LD_LIBRARY_PATH"
+
+        # Set environment for wflinfo executable.
+        export PATH="/waffle/build/bin:$PATH"
+
+        # Use the surfaceless EGL platform.
+        export DISPLAY=
+        export WAFFLE_PLATFORM="surfaceless_egl"
+
+        SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD --platform surfaceless_egl --api gles2"
+
+        if [ "x$GALLIUM_DRIVER" = "xvirpipe" ]; then
+            # piglit is to use virpipe, and virgl_test_server llvmpipe
+            export GALLIUM_DRIVER="$GALLIUM_DRIVER"
+
+            GALLIUM_DRIVER=llvmpipe \
+            GALLIVM_PERF="nopt,no_filter_hacks" \
+            VTEST_USE_EGL_SURFACELESS=1 \
+            VTEST_USE_GLES=1 \
+            virgl_test_server >"$RESULTS"/vtest-log.txt 2>&1 &
+
+            sleep 1
+        fi
+    else
+        SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD --platform glx --api gl --profile core"
+        RUN_CMD_WRAPPER="xvfb-run --server-args=\"-noreset\" sh -c"
+    fi
+fi
+
+SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD | tee /tmp/version.txt | grep \"Mesa $MESA_VERSION\(\s\|$\)\""
 
 rm -rf results
 cd /piglit
 
-PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1)
+PIGLIT_OPTIONS=$(printf "%s" "$PIGLIT_OPTIONS")
-set +e
+
-xvfb-run --server-args="-noreset" sh -c \
+PIGLIT_CMD="./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES "$(/usr/bin/printf "%q" "$RESULTS")
-         "export LD_LIBRARY_PATH=$OLDPWD/install/lib;
+
-         wflinfo --platform glx --api gl --profile core | tee /tmp/version.txt | grep \"Mesa $VERSION\\\$\" &&
+RUN_CMD="$SANITY_MESA_VERSION_CMD && $PIGLIT_CMD"
-         ./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results"
+
-retVal=$?
+if [ "$RUN_CMD_WRAPPER" ]; then
-if [ $retVal -ne 0 ]; then
+    RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
-    echo "Found $(cat /tmp/version.txt), expected $VERSION"
 fi
-set -e
 
-PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES}
+eval $RUN_CMD
+
+if [ $? -ne 0 ]; then
+    printf "%s\n" "Found $(cat /tmp/version.txt), expected $MESA_VERSION"
+fi
+
+if [ ${PIGLIT_JUNIT_RESULTS:-0} -eq 1 ]; then
+    ./piglit summary aggregate "$RESULTS" -o junit.xml
+fi
+
+PIGLIT_RESULTS="${PIGLIT_RESULTS:-$PIGLIT_PROFILES}"
 mkdir -p .gitlab-ci/piglit
-cp $OLDPWD/install/piglit/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline
+./piglit summary console "$RESULTS"/results.json.bz2 \
-./piglit summary console $OLDPWD/results | head -n -1 | grep -v ": pass" >.gitlab-ci/piglit/$PIGLIT_RESULTS.txt
+    | tee ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig" \
+    | head -n -1 | grep -v ": pass" >".gitlab-ci/piglit/$PIGLIT_RESULTS.txt"
 
-if diff -q .gitlab-ci/piglit/$PIGLIT_RESULTS.txt{.baseline,}; then
+if [ "x$PIGLIT_PROFILES" = "xreplay" ] \
+       && [ ${PIGLIT_REPLAY_UPLOAD_TO_MINIO:-0} -eq 1 ]; then
+
+    ci-fairy minio login $CI_JOB_JWT
+
+    __PREFIX="trace/$PIGLIT_REPLAY_DEVICE_NAME"
+    __MINIO_PATH="$PIGLIT_REPLAY_ARTIFACTS_BASE_URL"
+    __MINIO_TRACES_PREFIX="traces"
+
+    ci-fairy minio cp "$RESULTS"/junit.xml \
+        "minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/junit.xml"
+    ci-fairy minio cp "$RESULTS"/results.json.bz2 \
+        "minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/results.json.bz2"
+
+    find "$RESULTS/$__PREFIX" -type f -name "*.png" -printf "%P\n" \
+        | while read -r line; do
+
+        __TRACE="${line%-*-*}"
+        if grep -q "^$__PREFIX/$__TRACE: pass$" ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig"; then
+            if [ "x$CI_PROJECT_PATH" != "x$FDO_UPSTREAM_REPO" ]; then
+                continue
+            fi
+            __MINIO_PATH="$PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL"
+            __DESTINATION_FILE_PATH="${line##*-}"
+            if ci-fairy minio ls "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}" 2>/dev/null; then
+                continue
+            fi
+        else
+            __MINIO_PATH="$PIGLIT_REPLAY_ARTIFACTS_BASE_URL"
+            __DESTINATION_FILE_PATH="$__MINIO_TRACES_PREFIX/${line##*-}"
+        fi
+
+        ci-fairy minio cp "$RESULTS/$__PREFIX/$line" \
+            "minio://${MINIO_HOST}${__MINIO_PATH}/${__DESTINATION_FILE_PATH}"
+    done
+fi
+
+cp "$INSTALL/piglit/$PIGLIT_RESULTS.txt" \
+   ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline"
+if diff -q ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" \
+        ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt"; then
     exit 0
 fi
 
-./piglit summary html --exclude-details=pass $OLDPWD/summary $OLDPWD/results
+if [ ${PIGLIT_HTML_SUMMARY:-1} -eq 1 ]; then
+    ./piglit summary html --exclude-details=pass \
+        "$OLDPWD"/summary "$RESULTS"/results.json.bz2
 
-echo Unexpected change in results:
+    if [ "x$PIGLIT_PROFILES" = "xreplay" ]; then
-diff -u .gitlab-ci/piglit/$PIGLIT_RESULTS.txt{.baseline,}
+        find "$OLDPWD"/summary -type f -name "*.html" -print0 \
+            | xargs -0 sed -i 's@<img src="file://'"${RESULTS}"'@<img src="https://'"${MINIO_HOST}${PIGLIT_REPLAY_ARTIFACTS_BASE_URL}"'@g'
+        find "$OLDPWD"/summary -type f -name "*.html" -print0 \
+            | xargs -0 sed -i 's@<img src="file://@<img src="https://'"${MINIO_HOST}${PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL}"'/@g'
+    fi
+fi
+
+printf "%s\n" "Unexpected change in results:"
+diff -u ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" \
+     ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt"
 exit 1

.gitlab-ci/piglit/virgl-replay.txt

@@ -0,0 +1,16 @@
+summary:
+       name:  results
+       ----  --------
+       pass:        7
+       fail:        0
+      crash:        0
+       skip:        0
+    timeout:        0
+       warn:        0
+ incomplete:        0
+ dmesg-warn:        0
+ dmesg-fail:        0
+    changes:        0
+      fixes:        0
+regressions:        0
+      total:        7