From 58e6d8eee245fdff705a084ad37f9d741dff0bd6 Mon Sep 17 00:00:00 2001 From: Emma Anholt Date: Tue, 29 Nov 2022 14:35:19 -0800 Subject: [PATCH] ci: Update the skqp testing docs and retire the old runner script. Reviewed-by: Guilherme Gallo Part-of: --- .gitlab-ci/skqp-runner.sh | 371 ---------------------------------- .gitlab-ci/test/gitlab-ci.yml | 5 - docs/ci/skqp.rst | 100 ++------- 3 files changed, 16 insertions(+), 460 deletions(-) delete mode 100755 .gitlab-ci/skqp-runner.sh diff --git a/.gitlab-ci/skqp-runner.sh b/.gitlab-ci/skqp-runner.sh deleted file mode 100755 index 0a64da574d0..00000000000 --- a/.gitlab-ci/skqp-runner.sh +++ /dev/null @@ -1,371 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2022 Collabora Limited -# Author: Guilherme Gallo -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice (including the next -# paragraph) shall be included in all copies or substantial portions of the -# Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# Args: -# $1: section id -# $2: section header -gitlab_section_start() { - echo -e "\e[0Ksection_start:$(date +%s):$1[collapsed=${GL_COLLAPSED:-false}]\r\e[0K\e[32;1m$2\e[0m" -} - -# Args: -# $1: section id -gitlab_section_end() { - echo -e "\e[0Ksection_end:$(date +%s):$1\r\e[0K" -} - - -# sponge allows piping to files that are being used as input. -# E.g.: sort file.txt | sponge file.txt -# In order to avoid installing moreutils just to have the sponge binary, we can -# use a bash function for it -# Source https://unix.stackexchange.com/a/561346/310927 -sponge () ( - set +x - append=false - - while getopts 'a' opt; do - case $opt in - a) append=true ;; - *) echo error; exit 1 - esac - done - shift "$(( OPTIND - 1 ))" - - outfile=$1 - - tmpfile=$(mktemp "$(dirname "$outfile")/tmp-sponge.XXXXXXXX") && - cat >"$tmpfile" && - if "$append"; then - cat "$tmpfile" >>"$outfile" - else - if [ -f "$outfile" ]; then - chmod --reference="$outfile" "$tmpfile" - fi - if [ -f "$outfile" ]; then - mv "$tmpfile" "$outfile" - elif [ -n "$outfile" ] && [ ! -e "$outfile" ]; then - cat "$tmpfile" >"$outfile" - else - cat "$tmpfile" - fi - fi && - rm -f "$tmpfile" -) - -remove_comments_from_files() ( - INPUT_FILES="$*" - for INPUT_FILE in ${INPUT_FILES} - do - [ -f "${INPUT_FILE}" ] || continue - sed -i '/#/d' "${INPUT_FILE}" - sed -i '/^\s*$/d' "${INPUT_FILE}" - done -) - -subtract_test_lists() ( - MINUEND=$1 - sort "${MINUEND}" | sponge "${MINUEND}" - shift - for SUBTRAHEND in "$@" - do - sort "${SUBTRAHEND}" | sponge "${SUBTRAHEND}" - join -v 1 "${MINUEND}" "${SUBTRAHEND}" | - sponge "${MINUEND}" - done -) - -merge_rendertests_files() { - BASE_FILE=$1 - shift - FILES="$*" - # shellcheck disable=SC2086 - cat $FILES "$BASE_FILE" | - sort --unique --stable --field-separator=, --key=1,1 | - sponge "$BASE_FILE" -} - -assure_files() ( - for CASELIST_FILE in $* - do - >&2 echo "Looking for ${CASELIST_FILE}..." - [ -f ${CASELIST_FILE} ] || ( - >&2 echo "Not found. Creating empty." - touch ${CASELIST_FILE} - ) - done -) - -# Generate rendertests from scratch, customizing with fails/flakes/crashes files -generate_rendertests() ( - set -e - GENERATED_FILE=$(mktemp) - TESTS_FILE_PREFIX="${SKQP_FILE_PREFIX}-${SKQP_BACKEND}_rendertests" - FLAKES_FILE="${TESTS_FILE_PREFIX}-flakes.txt" - FAILS_FILE="${TESTS_FILE_PREFIX}-fails.txt" - CRASHES_FILE="${TESTS_FILE_PREFIX}-crashes.txt" - RENDER_TESTS_FILE="${TESTS_FILE_PREFIX}.txt" - - # Default to an empty known flakes file if it doesn't exist. - assure_files ${FLAKES_FILE} ${FAILS_FILE} ${CRASHES_FILE} - - # skqp does not support comments in rendertests.txt file - remove_comments_from_files "${FLAKES_FILE}" "${FAILS_FILE}" "${CRASHES_FILE}" - - # create an exhaustive rendertest list - "${SKQP_BIN_DIR}"/list_gms | sort > "$GENERATED_FILE" - - # Remove undesirable tests from the list - subtract_test_lists "${GENERATED_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}" - - # Add ",0" to each test to set the expected diff sum to zero - sed -i 's/$/,0/g' "$GENERATED_FILE" - - merge_rendertests_files "$GENERATED_FILE" "${FAILS_FILE}" - - mv "${GENERATED_FILE}" "${RENDER_TESTS_FILE}" - - echo "${RENDER_TESTS_FILE}" -) - -generate_unittests() ( - set -e - GENERATED_FILE=$(mktemp) - TESTS_FILE_PREFIX="${SKQP_FILE_PREFIX}_unittests" - FLAKES_FILE="${TESTS_FILE_PREFIX}-flakes.txt" - FAILS_FILE="${TESTS_FILE_PREFIX}-fails.txt" - CRASHES_FILE="${TESTS_FILE_PREFIX}-crashes.txt" - UNIT_TESTS_FILE="${TESTS_FILE_PREFIX}.txt" - - # Default to an empty known flakes file if it doesn't exist. - assure_files ${FLAKES_FILE} ${FAILS_FILE} ${CRASHES_FILE} - - # Remove unitTest_ prefix - for UT_FILE in "${FAILS_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}"; do - sed -i 's/^unitTest_//g' "${UT_FILE}" - done - - # create an exhaustive unittests list - "${SKQP_BIN_DIR}"/list_gpu_unit_tests > "${GENERATED_FILE}" - - # Remove undesirable tests from the list - subtract_test_lists "${GENERATED_FILE}" "${CRASHES_FILE}" "${FLAKES_FILE}" "${FAILS_FILE}" - - remove_comments_from_files "${GENERATED_FILE}" - mv "${GENERATED_FILE}" "${UNIT_TESTS_FILE}" - - echo "${UNIT_TESTS_FILE}" -) - -run_all_tests() { - rm -f "${SKQP_ASSETS_DIR}"/skqp/*.txt -} - -copy_tests_files() ( - # Copy either unit test or render test files from a specific driver given by - # GPU VERSION variable. - # If there is no test file at the expected location, this function will - # return error_code 1 - SKQP_BACKEND="${1}" - SKQP_FILE_PREFIX="${INSTALL}/${GPU_VERSION}-skqp" - - if echo "${SKQP_BACKEND}" | grep -qE 'vk|gl(es)?' - then - echo "Generating rendertests.txt file" - GENERATED_RENDERTESTS=$(generate_rendertests) - cp "${GENERATED_RENDERTESTS}" "${SKQP_ASSETS_DIR}"/skqp/rendertests.txt - mkdir -p "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}" - cp "${GENERATED_RENDERTESTS}" "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}/generated_rendertests.txt" - return 0 - fi - - # The unittests.txt path is hardcoded inside assets directory, - # that is why it needs to be a special case. - if echo "${SKQP_BACKEND}" | grep -qE "unitTest" - then - echo "Generating unittests.txt file" - GENERATED_UNITTESTS=$(generate_unittests) - cp "${GENERATED_UNITTESTS}" "${SKQP_ASSETS_DIR}"/skqp/unittests.txt - mkdir -p "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}" - cp "${GENERATED_UNITTESTS}" "${SKQP_RESULTS_DIR}/${SKQP_BACKEND}/generated_unittests.txt" - fi -) - -resolve_tests_files() { - if [ -n "${RUN_ALL_TESTS}" ] - then - run_all_tests - return - fi - - SKQP_BACKEND=${1} - if ! copy_tests_files "${SKQP_BACKEND}" - then - echo "No override test file found for ${SKQP_BACKEND}. Using the default one." - fi -} - -test_vk_backend() { - if echo "${SKQP_BACKENDS:?}" | grep -qE 'vk' - then - if [ -n "$VK_DRIVER" ]; then - return 0 - fi - - echo "VK_DRIVER environment variable is missing." - # shellcheck disable=SC2012 - VK_DRIVERS=$(ls "$INSTALL"/share/vulkan/icd.d/ | cut -f 1 -d '_') - if [ -n "${VK_DRIVERS}" ] - then - echo "Please set VK_DRIVER to the correct driver from the list:" - echo "${VK_DRIVERS}" - fi - echo "No Vulkan tests will be executed, but it was requested in SKQP_BACKENDS variable. Exiting." - exit 2 - fi - - # Vulkan environment is not configured, but it was not requested by the job - return 1 -} - -setup_backends() { - if test_vk_backend - then - export VK_ICD_FILENAMES="$INSTALL"/share/vulkan/icd.d/"$VK_DRIVER"_icd."${VK_CPU:-$(uname -m)}".json - fi -} - -show_reports() ( - set +xe - - # Unit tests produce empty HTML reports, guide the user to check the TXT file. - if echo "${SKQP_BACKENDS}" | grep -qE "unitTest" - then - # Remove the empty HTML report to avoid confusion - rm -f "${SKQP_RESULTS_DIR}"/unitTest/report.html - - echo "See skqp unit test results at:" - echo "https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${SKQP_RESULTS_DIR}/unitTest/unit_tests.txt" - fi - - REPORT_FILES=$(mktemp) - find "${SKQP_RESULTS_DIR}"/**/report.html -type f > "${REPORT_FILES}" - while read -r REPORT - do - # shellcheck disable=SC2001 - BACKEND_NAME=$(echo "${REPORT}" | sed 's@.*/\([^/]*\)/report.html@\1@') - echo "See skqp ${BACKEND_NAME} render tests report at:" - echo "https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${REPORT}" - done < "${REPORT_FILES}" - - # If there is no report available, tell the user that something is wrong. - if [ ! -s "${REPORT_FILES}" ] - then - echo "No skqp report available. Probably some fatal error has occured during the skqp execution." - fi -) - -usage() { - cat < unittests.txt` - -2. Run SkQP job - -3. If there is a failing or crashing unit test, remove it from the corresponding ``unittests.txt`` - -4. If there is a crashing render test, remove it from the corresponding ``rendertests.txt`` - -5. If there is a failing render test, visually inspect the result from the HTML report - - If the render result is OK, update the max error count for that test - - Otherwise, or put ``-1`` in the same threshold, as seen in :ref:`rendertests-design` - -6. Remember to put the new tests files to the locations cited in :ref:`test-location` +In SkQP itself, testcases can have increased failing pixel thresholds added to +them to keep CI green when the rendering is "correct" but out of normal range. +However, we don't support changing the thresholds in our testing. Because any +driver rendering not meeting the normal thresholds will trigger Android CTS +failures, we treat them as failures and track them as expected failures the +```*-fails.txt`` file.`