ci/piglit: allow parallel piglit jobs

This allows to split a piglit job in several parallel jobs, to speed up
the execution.

Due piglit restrictions, this only works for single profiles. Otherwise
an error will be shown in the runner.

Also, a new gitlab job variable `PIGLIT_TESTS` is introduced that
contains the excluded/included tests with `-x` or `-n`. The rest of the
piglit options go to `PIGLIT_OPTIONS` (like `--timeout n`).

v2 (Andres):
 - Replay profile is supported in parallel jobs.
 - Bail out inmediately if parallel jobs is tried with multiple
profiles.
 - Use testlist only when doing parallel jobs.
 - Do not drop pass tests when filtering executed tests.
 - Get rid of PIGLIT_FRACTION.

v4:
 - uncommit unrelated change (Andres).

Signed-off-by: Juan A. Suarez Romero <jasuarez@igalia.com>
Reviewed-by: Andres Gomez <agomez@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/9022>

.gitlab-ci.yml

@@ -905,7 +905,7 @@ test-d3d12-windows:
     GIT_STRATEGY: none # testing doesn't build anything from source
     GALLIUM_DRIVER: d3d12
     PIGLIT_PROFILE: quick_gl
-    PIGLIT_OPTIONS: -x nv_copy_depth_to_color -x repeat-wait -x arb_timer_query@timestamp-get
+    PIGLIT_TESTS: -x nv_copy_depth_to_color -x repeat-wait -x arb_timer_query@timestamp-get
   script:
     - . _install/piglit_run.ps1
   artifacts:

.gitlab-ci/bare-metal/rootfs-setup.sh

@@ -71,6 +71,7 @@ for var in \
     PIGLIT_REPLAY_REFERENCE_IMAGES_BASE_URL \
     PIGLIT_REPLAY_UPLOAD_TO_MINIO \
     PIGLIT_RESULTS \
+    PIGLIT_TESTS \
     TEST_LD_PRELOAD \
     TU_DEBUG \
     VK_CPU \

.gitlab-ci/piglit/run.sh

@@ -113,23 +113,21 @@ else
     fi
 fi
 
-SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD | tee /tmp/version.txt | grep \"Mesa $MESA_VERSION\(\s\|$\)\""
-
 if [ "$ZINK_USE_LAVAPIPE" ]; then
     export VK_ICD_FILENAMES="$INSTALL/share/vulkan/icd.d/lvp_icd.x86_64.json"
 fi
 
-rm -rf results
+# If the job is parallel at the  gitlab job level, will take the corresponding
-cd /piglit
+# fraction of the caselist.
+if [ -n "$CI_NODE_INDEX" ]; then
 
-PIGLIT_OPTIONS=$(printf "%s" "$PIGLIT_OPTIONS")
+    if [ "$PIGLIT_PROFILES" != "${PIGLIT_PROFILES% *}" ]; then
+        FAILURE_MESSAGE=$(printf "%s" "Can't parallelize piglit with multiple profiles")
+        quiet print_red printf "%s\n" "$FAILURE_MESSAGE"
+        exit 1
+    fi
 
-PIGLIT_CMD="./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES "$(/usr/bin/printf "%q" "$RESULTS")
+    USE_CASELIST=1
-
-RUN_CMD="export LD_LIBRARY_PATH=$__LD_LIBRARY_PATH; $SANITY_MESA_VERSION_CMD && $PIGLIT_CMD"
-
-if [ "$RUN_CMD_WRAPPER" ]; then
-    RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
 fi
 
 print_red() {
@@ -183,6 +181,35 @@ replay_minio_upload_images() {
     done
 }
 
+SANITY_MESA_VERSION_CMD="$SANITY_MESA_VERSION_CMD | tee /tmp/version.txt | grep \"Mesa $MESA_VERSION\(\s\|$\)\""
+
+rm -rf results
+cd /piglit
+
+if [ -n "$USE_CASELIST" ]; then
+    PIGLIT_TESTS=$(printf "%s" "$PIGLIT_TESTS")
+    PIGLIT_GENTESTS="./piglit print-cmd $PIGLIT_TESTS $PIGLIT_PROFILES --format \"{name}\" > /tmp/case-list.txt"
+    RUN_GENTESTS="export LD_LIBRARY_PATH=$__LD_LIBRARY_PATH; $PIGLIT_GENTESTS"
+
+    eval $RUN_GENTESTS
+
+    sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
+
+    PIGLIT_TESTS="--test-list /tmp/case-list.txt"
+fi
+
+PIGLIT_OPTIONS=$(printf "%s" "$PIGLIT_OPTIONS")
+
+PIGLIT_TESTS=$(printf "%s" "$PIGLIT_TESTS")
+
+PIGLIT_CMD="./piglit run -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_TESTS $PIGLIT_PROFILES "$(/usr/bin/printf "%q" "$RESULTS")
+
+RUN_CMD="export LD_LIBRARY_PATH=$__LD_LIBRARY_PATH; $SANITY_MESA_VERSION_CMD && $PIGLIT_CMD"
+
+if [ "$RUN_CMD_WRAPPER" ]; then
+    RUN_CMD="set +e; $RUN_CMD_WRAPPER "$(/usr/bin/printf "%q" "$RUN_CMD")"; set -e"
+fi
+
 FAILURE_MESSAGE=$(printf "%s" "Unexpected change in results:")
 
 eval $RUN_CMD
@@ -223,8 +250,21 @@ if [ "x$PIGLIT_PROFILES" = "xreplay" ] \
         "minio://${MINIO_HOST}${__MINIO_PATH}/${__MINIO_TRACES_PREFIX}/junit.xml"
 fi
 
-cp "$INSTALL/$PIGLIT_RESULTS.txt" \
+if [ -n "$USE_CASELIST" ]; then
-   ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline"
+    # Just filter the expected results based on the tests that were actually
+    # executed, and switch to the version with no summary
+    cat $RESULTSFILE | head -n -16 > ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new"
+    RESULTSFILE=".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new"
+    cat ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.orig" | head -n -17 | rev \
+        | cut -f2- -d: | rev | sed "s/$/:/g" > /tmp/executed.txt
+
+    grep -F -f /tmp/executed.txt "$INSTALL/$PIGLIT_RESULTS.txt" \
+       > ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" || true
+else
+    cp "$INSTALL/$PIGLIT_RESULTS.txt" \
+       ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline"
+fi
+
 if diff -q ".gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline" $RESULTSFILE; then
     exit 0
 fi

.gitlab-ci/piglit/run_cl.sh

@@ -10,12 +10,31 @@ cd /piglit
 
 export OCL_ICD_VENDORS=$OLDPWD/install/etc/OpenCL/vendors/
 
-PIGLIT_OPTIONS=$(echo $PIGLIT_OPTIONS | head -n 1)
 set +e
 unset DISPLAY
 export LD_LIBRARY_PATH=$OLDPWD/install/lib
 clinfo
-./piglit run -c -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_PROFILES $OLDPWD/results
+
+# If the job is parallel at the gitlab job level, will take the corresponding
+# fraction of the caselist.
+if [ -n "$CI_NODE_INDEX" ]; then
+
+    if [ "$PIGLIT_PROFILES" != "${PIGLIT_PROFILES% *}" ]; then
+        echo "Can't parallelize piglit with multiple profiles"
+        exit 1
+    fi
+    USE_CASELIST=1
+fi
+
+if [ -n "$USE_CASELIST" ]; then
+    ./piglit print-cmd $PIGLIT_TESTS $PIGLIT_PROFILES --format "{name}" > /tmp/case-list.txt
+
+    sed -ni $CI_NODE_INDEX~$CI_NODE_TOTAL"p" /tmp/case-list.txt
+
+    PIGLIT_TESTS="--test-list /tmp/case-list.txt"
+fi
+
+./piglit run -c -j${FDO_CI_CONCURRENT:-4} $PIGLIT_OPTIONS $PIGLIT_TESTS $PIGLIT_PROFILES $OLDPWD/results
 retVal=$?
 if [ $retVal -ne 0 ]; then
     echo "Found $(cat /tmp/version.txt), expected $VERSION"
@@ -24,15 +43,26 @@ set -e
 
 PIGLIT_RESULTS=${PIGLIT_RESULTS:-$PIGLIT_PROFILES}
 mkdir -p .gitlab-ci/piglit
-cp $OLDPWD/install/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline
+./piglit summary console $OLDPWD/results | head -n -1 | grep -v ": pass" > .gitlab-ci/piglit/$PIGLIT_RESULTS.txt
-./piglit summary console $OLDPWD/results | head -n -1 | grep -v ": pass" >.gitlab-ci/piglit/$PIGLIT_RESULTS.txt
 
-if diff -q .gitlab-ci/piglit/$PIGLIT_RESULTS.txt{.baseline,}; then
+if [ -n "$USE_CASELIST" ]; then
+    # Just filter the expected results based on the tests that were actually
+    # executed, and switch to the version with no summary
+    cat .gitlab-ci/piglit/$PIGLIT_RESULTS.txt | head -n -16 | tee .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new \
+         | rev | cut -f2- -d: | rev | sed "s/$/:/g" > /tmp/executed.txt
+    grep -F -f /tmp/executed.txt $OLDPWD/install/$PIGLIT_RESULTS.txt \
+         > .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline || true
+else
+    cp $OLDPWD/install/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.baseline
+    cp .gitlab-ci/piglit/$PIGLIT_RESULTS.txt .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.new
+fi
+
+if diff -q .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.{baseline,new}; then
     exit 0
 fi
 
 ./piglit summary html --exclude-details=pass $OLDPWD/summary $OLDPWD/results
 
 echo Unexpected change in results:
-diff -u .gitlab-ci/piglit/$PIGLIT_RESULTS.txt{.baseline,}
+diff -u .gitlab-ci/piglit/$PIGLIT_RESULTS.txt.{baseline,new}
 exit 1

.gitlab-ci/windows/piglit_run.ps1

@@ -3,7 +3,7 @@ $env:PIGLIT_NO_FAST_SKIP = 1
 Copy-Item -Path _install\bin\opengl32.dll -Destination C:\Piglit\lib\piglit\bin\opengl32.dll
 
 # Run this using VsDevCmd.bat to ensure DXIL.dll is in %PATH%
-cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && py -3 C:\Piglit\bin\piglit.py run `"$env:PIGLIT_PROFILE`" $env:PIGLIT_OPTIONS .\results"
+cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=amd64 && py -3 C:\Piglit\bin\piglit.py run `"$env:PIGLIT_PROFILE`" $env:PIGLIT_OPTIONS $env:PIGLIT_TESTS .\results"
 
 py -3 C:\Piglit\bin\piglit.py summary console .\results | Select -SkipLast 1 | Select-String -NotMatch -Pattern ': pass' | Set-Content -Path .\result.txt
 

src/broadcom/ci/gitlab-ci.yml

@@ -40,6 +40,8 @@ vc4-rpi3-gles2:armhf:
     BM_POE_TIMEOUT: 180
     PIGLIT_OPTIONS: >-
       --timeout 60
+    PIGLIT_PLATFORM: gbm
+    PIGLIT_TESTS: >-
       -x .tesc
       -x .tese
       -x arb_gpu_shader_fp64
@@ -83,7 +85,6 @@ vc4-rpi3-gles2:armhf:
       -x ppgtt_memory_alignment
       -x streaming-texture-leak
       -x texturesize
-    PIGLIT_PLATFORM: gbm
 
 vc4-rpi3-piglit-quick_gl:armhf:
   extends:
@@ -154,6 +155,8 @@ v3d-rpi4-gles2:armhf:
     BARE_METAL_TEST_SCRIPT: "/install/piglit/run.sh"
     PIGLIT_OPTIONS: >-
       --timeout 60
+    PIGLIT_PLATFORM: gbm
+    PIGLIT_TESTS: >-
       -x .tesc
       -x .tese
       -x arb_gpu_shader_fp64
@@ -182,7 +185,6 @@ v3d-rpi4-gles2:armhf:
       -x maxsize
       -x streaming-texture-leak
       -x tex-miplevel-selection
-    PIGLIT_PLATFORM: gbm
 
 v3d-rpi4-piglit-quick_gl:armhf:
   extends:

src/freedreno/ci/gitlab-ci.yml

@@ -112,7 +112,7 @@ a530_gles31:
     # Also excluding some particularly slow tests (gl-1.3-texture-env)
     #
     # Check fastboot.sh for more piglit tests being excluded!
-    PIGLIT_OPTIONS: >-
+    PIGLIT_TESTS: >-
       -x atomic_counters
       -x gl-1.0-blend-func
       -x glsl-1.30@execution@clipping
@@ -277,7 +277,7 @@ a630_vk_sysmem:
     #
     # vs-clip-distance/vertex-enables skipped because they seem to be flaky in
     # parallel with other tests (reliably fails on its own)
-    PIGLIT_OPTIONS: >-
+    PIGLIT_TESTS: >-
       -x vs-output-array-vec2-index-wr-before-gs
       -x fixed-clip-enables
       -x vs-clip-distance-enables

src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml

@@ -16,7 +16,7 @@ llvmpipe-piglit-cl:
     LP_NUM_THREADS: 1
     PIGLIT_PROFILES: cl
     PIGLIT_RESULTS: "llvmpipe-cl"
-    PIGLIT_OPTIONS: >
+    PIGLIT_TESTS: >
       -x bswap -x phatk -x clz-optimizations
   script:
     - install/piglit/run_cl.sh
@@ -31,6 +31,7 @@ llvmpipe-piglit-quick_gl:
     LP_NUM_THREADS: 0
     PIGLIT_OPTIONS: >
       --process-isolation false
+    PIGLIT_TESTS: >
       -x egl_ext_device_
       -x egl_ext_platform_device
       -x ext_timer_query@time-elapsed

src/gallium/drivers/softpipe/ci/gitlab-ci.yml

@@ -39,7 +39,7 @@ softpipe-piglit-quick:
     # rasterpos skipped until we uprev piglit (commit ff2a7650be7349)
     # triangle-rasterization skipped due to flakes thanks to it using a time-based random seed.
     # glx-multithread-texture is flaky, undiagnosed
-    PIGLIT_OPTIONS: >
+    PIGLIT_TESTS: >
       -x rasterpos
       -x triangle-rasterization
       -x glx-multithread-texture

src/gallium/drivers/zink/ci/gitlab-ci.yml

@@ -15,7 +15,7 @@ zink-piglit-quick_gl:
     - .piglit-test
     - .zink-lvp-test
   variables:
-    PIGLIT_OPTIONS: >
+    PIGLIT_TESTS: >
       -x arb_map_buffer_alignment@arb_map_buffer_alignment-map-invalidate-range
       -x glx-make-current -x arb_timer_query -x arb_sample_shading@builtin-gl-sample-mask
     PIGLIT_PROFILES: quick_gl