fdo-mirrors/mesa
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/mesa/mesa.git synced 2026-05-09 04:38:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ci: add support for structural tagging

Browse source 
Make structural tagging functions available for both test and build
scripts.

Introduces the update_tag.sh helper for listing, checking, and updating
deterministic tags.

Also adds the ci_tag_build_time_check and ci_tag_test_time_check
functions to validate tags during build and test phases, ensuring
consistent component versioning.

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/33421>
This commit is contained in:
Guilherme Gallo 2025-02-04 17:36:22 -03:00 • committed by Marge Bot
parent 775c2c3254
commit f13b95ad5c
6 changed files with 959 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitlab-ci/conditional-build-image-tags.yml Normal file
View file

@ -0,0 +1 @@
variables:

23
.gitlab-ci/container/container_pre_build.sh
View file

@ -1,4 +1,7 @@
#!/bin/sh
# When changing this file, you need to bump the following
# .gitlab-ci/image-tags.yml tags:
# DEBIAN_BUILD_TAG
if test -x /usr/bin/ccache; then
if test -f /etc/debian_version; then
@ -49,3 +52,23 @@ if [ -f "$CARGO_ENV_FILE" ]; then
# shellcheck disable=SC1090
source "$CARGO_ENV_FILE"
fi
ci_tag_early_checks() {
# Runs the first part of the build script to perform the tag check only
uncollapsed_section_switch "ci_tag_early_checks" "Ensuring component versions match declared tags in CI builds"
echo "[Structured Tagging] Checking components: ${CI_BUILD_COMPONENTS}"
# shellcheck disable=SC2086
for component in ${CI_BUILD_COMPONENTS}; do
bin/ci/update_tag.py --check ${component} || exit 1
done
echo "[Structured Tagging] Components check done"
section_end "ci_tag_early_checks"
}
# Check if each declared tag component is up to date before building
if [ -n "${CI_BUILD_COMPONENTS:-}" ]; then
# Remove any duplicates by splitting on whitespace, sorting, then joining back
CI_BUILD_COMPONENTS="$(echo "${CI_BUILD_COMPONENTS}" | xargs -n1 | sort -u | xargs)"
ci_tag_early_checks
fi

7
.gitlab-ci/image-tags.yml
View file

@ -11,6 +11,13 @@
# cannot parse input: "$image:$tag": invalid reference format
# check the length of $tag; if it's > 128 chars you need to shorten your tag.
include:
# Include the conditional build image tags used for structured tagging runtime checks that happens
# during both build and test jobs
# It can be auto-generated by bin/ci/update_tag.py script, so to keep the script's serde simple,
# let's keep the component tags in a separate file
- .gitlab-ci/conditional-build-image-tags.yml
variables:
DEBIAN_X86_64_BUILD_BASE_IMAGE: "debian/x86_64_build-base"
DEBIAN_BASE_TAG: "20250223-way-prot"

141
.gitlab-ci/setup-test-env.sh
View file

@ -187,6 +187,147 @@ function trap_err {
error ${CURRENT_SECTION:-'unknown-section'}: ret code: $*
}
# ------ Structured tagging
export _CI_TAG_CHECK_DIR="/mesa-ci-build-tag"
_ci_tag_from_name_to_var() {
# Transforms MY_COMPONENT_TAG to my-component
echo "${1%_TAG}" | tr '[:upper:]' '[:lower:]' | tr '_' '-'
}
_ci_tag_check() (
x_off
_declared_name="${1}"
declare -n _declared="${_declared_name}"
_calculated="${2}"
local component_lower=$(_ci_tag_from_name_to_var "${_declared_name}")
if [ -z "${_declared:-}" ]; then
# Close the section
error "Fatal error"
_error_msg "Structured tag is not set: ${_declared_name}"
_error_msg ""
echo "If you are adding a new component, please run:"
echo "bin/ci/update_tag.py --include ${component_lower}"
echo "This will automatically update the YAML file for you."
echo "Or manually edit .gitlab-ci/conditional-build-image-tags.yml to add the new"
echo "component."
error ""
exit 2
fi
if [ "${_declared}" != "${_calculated}" ]; then
# Close the section
error "Fatal error"
_error_msg "Mismatch in declared and calculated tags:"
_error_msg " ${_declared_name} from the YAML is \"${_declared}\""
_error_msg " ... but I think it should be \"${_calculated}\""
_error_msg ""
echo "Usually this happens when you change what you want to be built without also"
echo "changing the YAML declaration. For example, you've bumped SKQP to version"
echo "1.2.3, but you still have 'SKQP_VER: 1.2.2' in"
echo ".gitlab-ci/conditional-build-image-tags.yml."
echo ""
echo "If you meant to change the component I'm talking about, please change the"
echo "tag and resubmit. You can also run:"
echo "bin/ci/update_tag.py --include ${component_lower}"
echo "to update the tag automatically."
echo ""
echo "If you didn't mean to change the component, please ping @mesa/ci-helpers and we"
echo "can help you figure out what's gone wrong."
echo ""
echo "But for now, I've got to fail this build. Sorry."
exit 2
fi
x_restore
)
_ci_tag_check_build() {
x_off
if [ -n "${NEW_TAG_DRY_RUN:-}" ]; then
echo "${2}"
exit 0
fi
_ci_tag_check "${1}" "${2}"
if [ -n "${CI_NOT_BUILDING_ANYTHING:-}" ]; then
exit 0
fi
x_restore
}
get_tag_file() {
x_off
# If no tag name is provided, return the directory
echo "${_CI_TAG_CHECK_DIR}/${1:-}"
x_restore
}
_ci_tag_write() (
set +x
local tag_name="${1}"
local tag_value="${2}"
mkdir -p "${_CI_TAG_CHECK_DIR}"
echo -n "${tag_value}" > "$(get_tag_file "${tag_name}")"
)
_ci_calculate_tag() {
x_off
# the args are files that can affect the build output
mapfile -t extra_files < <(printf '%s\n' "$@")
(
for extra_file in "${extra_files[@]}"; do
if [ ! -f "${extra_file}" ]; then
error "File '${extra_file}' does not exist"
exit 1
fi
cat "${extra_file}"
done
) | md5sum | cut -d' ' -f1
x_restore
}
ci_tag_build_time_check() {
# Get the caller script and hash its contents plus the extra files
x_off
local tag_name="${1}"
local build_script_file="build-$(_ci_tag_from_name_to_var "${tag_name}").sh"
local build_script=".gitlab-ci/container/${build_script_file}"
shift
# now $@ has the extra files
local calculated_tag=$(_ci_calculate_tag "${build_script}" "$@")
_ci_tag_check_build "${tag_name}" "${calculated_tag}"
_ci_tag_write "${tag_name}" "${calculated_tag}"
x_restore
}
ci_tag_test_time_check() {
x_off
local tag_file=$(get_tag_file "${1}")
if [ ! -f "${tag_file}" ]; then
_error_msg "Structured tag file ${tag_file} does not exist"
_error_msg "Please run the ci_tag_build_time_check first and rebuild the image/rootfs"
exit 2
fi
_ci_tag_check "${1}" "$(cat "${tag_file}")"
x_restore
}
# Export all functions
export -f _ci_calculate_tag
export -f _ci_tag_check
export -f _ci_tag_check_build
export -f _ci_tag_from_name_to_var
export -f _ci_tag_write
export -f ci_tag_build_time_check
export -f ci_tag_test_time_check
export -f get_tag_file
# Structured tagging ------
export -f error
export -f trap_err

368
bin/ci/test/test_update_tag.py Normal file
View file

@ -0,0 +1,368 @@
import os
import sys
import subprocess
import yaml
import pytest
from textwrap import dedent
from unittest.mock import patch
from bin.ci.update_tag import (
from_component_to_build_tag,
filter_components,
from_component_to_tag_var,
from_script_name_to_component,
from_script_name_to_tag_var,
load_container_yaml,
load_image_tags_yaml,
update_image_tag_in_yaml,
run_build_script,
main,
)
@pytest.fixture
def temp_image_tags_file(tmp_path, monkeypatch):
"""
Fixture that creates a temporary YAML file (to simulate CONDITIONAL_TAGS_FILE)
and updates the global variable accordingly.
"""
temp_file = tmp_path / "conditional-build-image-tags.yml"
# Write initial dummy content.
temp_file.write_text(yaml.dump({"variables": {}}))
monkeypatch.setattr("bin.ci.update_tag.CONDITIONAL_TAGS_FILE", str(temp_file))
return temp_file
@pytest.fixture
def temp_container_ci_file(tmp_path, monkeypatch):
"""
Fixture that creates a temporary container CI file (to simulate CONTAINER_CI_FILE)
and updates the global variable accordingly.
"""
temp_file = tmp_path / "container-ci.yml"
temp_file.touch()
monkeypatch.setattr("bin.ci.update_tag.CONTAINER_CI_FILE", temp_file)
return temp_file
@pytest.fixture
def mock_container_dir(tmp_path, monkeypatch):
"""
Fixture that creates a dummy container directory and build script.
"""
# Create a dummy container directory and build script.
container_dir = tmp_path / "container"
container_dir.mkdir()
# Patch CONTAINER_DIR so that the build script is found.
monkeypatch.setattr("bin.ci.update_tag.CONTAINER_DIR", container_dir)
# Create a dummy setup-test-env.sh file and patch set_dummy_env_vars.
dummy_setup_path = tmp_path / "setup-test-env.sh"
dummy_setup_path.write_text("echo Setup")
monkeypatch.setattr(
"bin.ci.update_tag.prepare_setup_env_script", lambda: dummy_setup_path
)
return container_dir
@pytest.fixture
def mock_build_script(mock_container_dir):
"""
Fixture that creates a dummy build script in the container directory.
"""
build_script = mock_container_dir / "build-foo.sh"
return build_script
###############################################################################
# Tests for argument parsing and helper functions
###############################################################################
@pytest.mark.parametrize(
"component, expected_tag_var",
[
("foo", "FOO_TAG"),
("my-component", "MY_COMPONENT_TAG"),
("foo-bar-baz", "FOO_BAR_BAZ_TAG"),
],
)
def test_from_component_to_tag_var(component, expected_tag_var):
"""
Test that from_component_to_tag_var returns the correct tag variable name.
"""
assert from_component_to_tag_var(component) == expected_tag_var
@pytest.mark.parametrize(
"component, expected_build_tag",
[
("foo", "CONDITIONAL_BUILD_FOO_TAG"),
("my-component", "CONDITIONAL_BUILD_MY_COMPONENT_TAG"),
("foo-bar-baz", "CONDITIONAL_BUILD_FOO_BAR_BAZ_TAG"),
],
)
def test_from_component_to_build_tag(component, expected_build_tag):
"""
Test that from_component_to_build_tag returns the correct build tag name.
"""
assert from_component_to_build_tag(component) == expected_build_tag
@pytest.mark.parametrize(
"script_name, expected_component",
[
("build-foo.sh", "foo"),
("build-my-component.sh", "my-component"),
("build-foo-bar-baz.sh", "foo-bar-baz"),
],
)
def test_from_script_name_to_component(script_name, expected_component):
"""
Test that from_script_name_to_component returns the correct component name.
"""
assert from_script_name_to_component(script_name) == expected_component
@pytest.mark.parametrize(
"script_name, expected_tag_var",
[
("build-foo.sh", "FOO_TAG"),
("build-my-component.sh", "MY_COMPONENT_TAG"),
("build-foo-bar-baz.sh", "FOO_BAR_BAZ_TAG"),
],
)
def test_from_script_name_to_tag_var(script_name, expected_tag_var):
"""
Test that from_script_name_to_tag_var returns the correct tag variable name.
"""
assert from_script_name_to_tag_var(script_name) == expected_tag_var
def test_filter_components():
"""
Test that filter_components returns only components that match an include regex
and do not match any exclude regex. If includes is empty, an empty list is returned.
"""
components = ["alpha", "beta", "gamma", "delta"]
# When includes is empty, should return an empty list.
assert filter_components(components, [], []) == []
# Test includes only.
# Components that start with 'a' or 'b'
expected = ["alpha", "beta"]
result = filter_components(components, ["^[a-b].*$"], [])
assert result == expected
# Test with an exclude that filters out "alpha" (which matches "lph").
result = filter_components(components, ["^[a-b].*$"], ["^.*lph.*$"])
# "alpha" is removed.
assert result == ["beta"]
@pytest.mark.parametrize(
"component, tag",
[
# Basic case
("test", "new_tag"),
# Component with hyphen
("my-component", "v1.2.3"),
# Uppercase component name
("UpperCase", "123"),
# Numbers in component name
("123service", "build-456"),
],
)
def test_update_image_tag_in_yaml(component, tag, temp_image_tags_file):
"""
Test multiple updates with different component names and tags, verifying:
1. Correct variable name generation
2. Proper tag value storage
3. Maintained sorting of variables
"""
# Initial update
update_image_tag_in_yaml(component, tag)
data = load_image_tags_yaml()
expected_var = from_component_to_build_tag(component)
assert data["variables"][expected_var] == tag
# Add second component and verify both exist
update_image_tag_in_yaml("another_component", "secondary_tag")
updated_data = load_image_tags_yaml()
assert from_component_to_build_tag("another_component") in updated_data["variables"]
assert updated_data["variables"][expected_var] == tag # Original value remains
# Verify sorting
variables = list(data["variables"].keys())
assert len(updated_data["variables"]) == len(variables) + 1
assert variables == sorted(variables), "Variables are not alphabetically sorted"
def test_if_run_extracts_the_tag_from_stdout(monkeypatch, mock_build_script):
"""
Test that run_build_script returns the tag (last stdout line) when the build
script executes successfully.
"""
mock_build_script.write_text("#!/bin/bash\necho Build script\necho new_tag")
# Create a fake subprocess.CompletedProcess to simulate a successful build.
fake_result = subprocess.CompletedProcess(
args=[], returncode=0, stdout="line1\nnew_tag", stderr=""
)
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
tag = run_build_script("foo", check_only=False)
assert tag == "new_tag"
def test_running_real_process_works(monkeypatch, mock_build_script):
"""
Test that run_build_script returns the tag (last stdout line) when the build
script executes successfully.
"""
mock_build_script.write_text("echo Build script\necho new_tag")
tag = run_build_script("foo", check_only=False)
assert tag == "new_tag"
###############################################################################
# Tests for main() argument features
###############################################################################
def setup_build_script_and_container_ci_file(container_dir, container_ci_file, comp):
tag_var = from_component_to_build_tag(comp)
build_check_var = from_component_to_tag_var(comp)
build_script_path = container_dir / f"build-{comp}.sh"
build_script_path.write_text(
dedent(
f"""
#!/bin/bash
ci_tag_build_time_check {tag_var}
echo "new_tag"
"""
)
)
container_ci_file.write_text(
container_ci_file.read_text()
+ dedent(
f"""
.container-builds-{comp}:
variables:
{tag_var}: "${{{build_check_var}}}"
"""
)
)
def test_main_list(monkeypatch, capsys, mock_container_dir, temp_container_ci_file):
"""
Test that when the --list argument is provided, main() prints all detected
components
"""
# Monkeypatch find_components to return a known list.
for comp in ["comp-a", "comp-b"]:
setup_build_script_and_container_ci_file(
mock_container_dir, temp_container_ci_file, comp
)
# Set sys.argv to simulate passing --list.
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--list"])
main()
captured = capsys.readouterr().out
assert "Detected components:" in captured
assert "comp-a" in captured
assert "comp-b" in captured
def test_main_check(monkeypatch, temp_image_tags_file, temp_container_ci_file):
"""
Test that when --check is provided, main() calls run_build_script in check mode.
"""
with patch(
"bin.ci.update_tag.run_build_script", return_value="dummy_tag"
) as mock_run_build_script:
# Simulate command line: --check compX
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "comp-x"])
main()
# Verify run_build_script was called with check_only=True.
assert mock_run_build_script.call_count == 1
assert "comp-x" in mock_run_build_script.call_args.args[0]
assert mock_run_build_script.call_args.kwargs["check_only"] is True
EXIT_CODE_SCENARIOS = {
"unbound_variable": (
"line 2: UNDEFINED_VARIABLE: unbound variable",
127,
3,
"Please set the variable UNDEFINED_VARIABLE",
),
"build script error": (
"",
50,
50,
"",
),
"tag_mismatch": (
"Tag mismatch for foo.",
2,
2,
"Tag mismatch for foo.",
),
}
@pytest.mark.parametrize(
"stderr_content, script_returncode, expected_exit_code, expected_error_message",
EXIT_CODE_SCENARIOS.values(),
ids=list(EXIT_CODE_SCENARIOS.keys()),
)
def test_build_script_error_exit_codes(
monkeypatch,
mock_build_script,
capsys,
stderr_content,
script_returncode,
expected_exit_code,
expected_error_message,
):
"""
Test that build script errors generate appropriate exit codes:
1 - Unhandled error in this script
2 - Tag mismatch when using --check
3 - Unbound variable error in build script
x - Build script failed with return code x
"""
# Create a build script that will fail
mock_build_script.write_text("#!/bin/bash\necho 'This will fail'")
# Create a fake subprocess.CompletedProcess to simulate a failed build
fake_result = subprocess.CompletedProcess(
args=[], returncode=script_returncode, stdout="", stderr=stderr_content
)
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "foo"])
monkeypatch.setattr(
"bin.ci.update_tag.get_current_tag_value", lambda *args: "current_tag"
)
# Mock sys.exit to capture the exit code instead of exiting the test
with pytest.raises(SystemExit) as e:
main()
# Check for expected error message if one is specified
if expected_error_message:
captured = capsys.readouterr()
assert expected_error_message in captured.err
# Verify correct exit code
assert e.value.code == expected_exit_code

419
bin/ci/update_tag.py Executable file
View file

@ -0,0 +1,419 @@
#!/usr/bin/env python3
import logging
import os
import re
import sys
import argparse
import subprocess
import yaml
from typing import Optional, Set, Any
from datetime import datetime, timezone
from pathlib import Path
CI_PROJECT_DIR: str | None = os.environ.get("CI_PROJECT_DIR", None)
GIT_REPO_ROOT: str = CI_PROJECT_DIR or str(Path(__file__).resolve().parent.parent.parent)
SETUP_TEST_ENV_PATH: Path = Path(GIT_REPO_ROOT) / ".gitlab-ci" / "setup-test-env.sh"
CONDITIONAL_TAGS_FILE: Path = (
Path(GIT_REPO_ROOT) / ".gitlab-ci" / "conditional-build-image-tags.yml"
)
CONTAINER_DIR: Path = Path(GIT_REPO_ROOT) / ".gitlab-ci" / "container"
CONTAINER_CI_FILE: Path = CONTAINER_DIR / "gitlab-ci.yml"
# Very simple type alias for GitLab YAML data structure
# It is composed by a dictionary of job names, each with a dictionary of fields
# (e.g., script, stage, rules, etc.)
YamlData = dict[str, dict[str, Any]]
# Dummy environment vars to make build scripts happy
# To avoid set -u errors in build scripts
DUMMY_ENV_VARS: dict[str, str] = {
# setup-test-env.sh
"CI_JOB_STARTED_AT": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%z"),
# build-deqp.sh
"DEQP_API": "dummy",
"DEQP_TARGET": "dummy",
}
def from_component_to_build_tag(component: str) -> str:
# e.g., "angle" -> "CONDITIONAL_BUILD_ANGLE_TAG"
return "CONDITIONAL_BUILD_" + re.sub(r"-", "_", component.upper()) + "_TAG"
def from_component_to_tag_var(component: str) -> str:
# e.g., "angle" -> "ANGLE_TAG"
return re.sub(r"-", "_", component.upper()) + "_TAG"
def from_script_name_to_component(script_name: str) -> str:
# e.g., "build-angle.sh" -> "angle"
return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1", script_name)
def from_script_name_to_tag_var(script_name: str) -> str:
# e.g., "build-angle.sh" -> "ANGLE_TAG"
return (
re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1_TAG", script_name)
.replace("-", "_")
.upper()
)
def prepare_setup_env_script() -> Path:
"""
Sets up dummy environment variables to mimic the script in the CI repo.
Returns the path to the setup-test-env.sh script.
"""
if not SETUP_TEST_ENV_PATH.exists():
sys.exit(".gitlab-ci/setup-test-env.sh not found. Exiting.")
# Dummy environment vars to mimic the script
for key, value in DUMMY_ENV_VARS.items():
os.environ[key] = value
os.environ["CI_PROJECT_DIR"] = GIT_REPO_ROOT
return SETUP_TEST_ENV_PATH
def validate_build_script(script_filename: str) -> bool:
"""
Returns True if the build script for the given component uses the structured tag variable.
"""
build_script = CONTAINER_DIR / script_filename
with open(build_script, "r", encoding="utf-8") as f:
script_content = f.read()
tag_var = from_script_name_to_tag_var(script_filename)
if not re.search(tag_var, script_content, re.IGNORECASE):
logging.debug(
f"Skipping {build_script} because it doesn't use {tag_var}",
)
return False
return True
def load_container_yaml() -> YamlData:
if not os.path.isfile(CONTAINER_CI_FILE):
sys.exit(f"File not found: {CONTAINER_CI_FILE}")
# Ignore !reference and other custom GitLab tags, we just want to know the
# job names and fields
yaml.SafeLoader.add_multi_constructor('', lambda loader, suffix, node: None)
with open(CONTAINER_CI_FILE, "r", encoding="utf-8") as f:
data = yaml.load(f, Loader=yaml.SafeLoader)
if not isinstance(data, dict):
return {"variables": {}}
return data
def find_candidate_components() -> list[str]:
"""
1) Reads .gitlab-ci/container/gitlab-ci.yml to find component links:
lines matching '.*.container-builds-<component>'
2) Looks for matching build-<component>.sh in .gitlab-ci/container/
3) Returns a sorted list of components in the intersection of these sets.
"""
container_yaml = load_container_yaml()
# Extract patterns like `container-builds-foo` from job names
candidates: Set[str] = set()
for job_name in container_yaml:
if match := re.search(r"\.container-builds-([a-z0-9_-]+)$", str(job_name)):
candidates.add(match.group(1))
if not candidates:
logging.error(
f"No viable build components found in {CONTAINER_CI_FILE}. "
"Please check the file for valid component names. "
"They should be named like '.container-builds-<component>'."
)
return []
# Find build scripts named build-<component>.sh
build_scripts: list[str] = []
for path in CONTAINER_DIR.glob("build-*.sh"):
if validate_build_script(path.name):
logging.info(f"Found build script: {path.name}")
component = from_script_name_to_component(path.name)
build_scripts.append(component)
# Return sorted intersection of components found in build scripts and candidates
return sorted(candidates.intersection(build_scripts))
def filter_components(
components: list[str], includes: list[str], excludes: list[str]
) -> list[str]:
"""
Returns components that match at least one `includes` regex and none of the `excludes` regex.
If includes is empty, returns an empty list (unless user explicitly does --all or --include).
"""
if not includes:
return []
filtered = []
for comp in components:
# Must match at least one "include"
if not any(re.fullmatch(inc, comp) for inc in includes):
logging.debug(f"Excluding {comp}, no matches in includes.")
continue
# Must not match any "exclude"
if any(re.fullmatch(exc, comp) for exc in excludes):
logging.debug(f"Excluding {comp}, matched exclude pattern.")
continue
filtered.append(comp)
return filtered
def run_build_script(component: str, check_only: bool = False) -> Optional[str]:
"""
Runs .gitlab-ci/container/build-<component>.sh to produce a new tag (last line of stdout).
If check_only=True, we skip updates to the YAML (but do the build to see if it passes).
Returns the extracted tag (string) on success, or None on failure.
"""
# 1) Set up environment
setup_env_script = prepare_setup_env_script()
build_script = os.path.join(CONTAINER_DIR, f"build-{component}.sh")
if not os.path.isfile(build_script):
logging.error(f"Build script not found for {component}: {build_script}")
return None
# Tag var should appear in the script, e.g., ANGLE_TAG for 'angle'
tag_var = from_component_to_tag_var(component)
# We set up environment for the child process
child_env: dict[str, str] = {}
child_env["NEW_TAG_DRY_RUN"] = "1"
if check_only:
# For checking only
child_env.pop("NEW_TAG_DRY_RUN", None)
child_env["CI_NOT_BUILDING_ANYTHING"] = "1"
if tag_value := get_current_tag_value(component):
child_env[tag_var] = tag_value
else:
logging.error(f"No current tag value for {component}")
return None
logging.debug(
f"Running build for {component} with "
f"{tag_var}={child_env.get(tag_var)} "
f"(check_only={check_only})"
)
# Run the build script
result = subprocess.run(
["bash", "-c", f"source {setup_env_script} && bash -x {build_script}"],
env=os.environ | child_env,
capture_output=True,
text=True,
)
logging.debug(f"{' '.join(result.args)}")
# Tag check succeeded
if result.returncode == 0:
# Tag is assumed to be the last line of stdout
lines = result.stdout.strip().splitlines()
return lines[-1] if lines else ""
# Tag check failed, let's dissect the error
if result.returncode == 2:
logging.error(
f"Tag mismatch for {component}."
)
logging.error(result.stdout)
return None
# Check if there's an unbound variable error
err_output = result.stderr
unbound_match = re.search(r"([A-Z_]+)(?=: unbound variable)", err_output)
if unbound_match:
var_name = unbound_match.group(1)
logging.fatal(f"Please set the variable {var_name} in {build_script}.")
sys.exit(3)
# Unexpected error in the build script, propagate the exit code
logging.fatal(
f"Build script for {component} failed with return code {result.returncode}"
)
logging.error(result.stdout)
sys.exit(result.returncode)
def load_image_tags_yaml() -> YamlData:
try:
with open(CONDITIONAL_TAGS_FILE, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
if not isinstance(data, dict):
return {"variables": {}}
if "variables" not in data:
data["variables"] = {}
return data
except FileNotFoundError:
return {"variables": {}}
def get_current_tag_value(component: str) -> Optional[str]:
full_tag_var = from_component_to_build_tag(component)
data = load_image_tags_yaml()
variables = data.get("variables", {})
if not isinstance(variables, dict):
return None
return variables.get(full_tag_var)
def update_image_tag_in_yaml(component: str, tag_value: str) -> None:
"""
Uses PyYAML to edit the YAML file at IMAGE_TAGS_FILE, setting the environment variable
for the given component. Maintains sorted keys.
"""
full_tag_var = from_component_to_build_tag(component)
data = load_image_tags_yaml()
# Ensure we have a variables dictionary
if "variables" not in data:
data["variables"] = {}
elif not isinstance(data["variables"], dict):
data["variables"] = {}
# Update the tag
data["variables"][full_tag_var] = tag_value
# Sort the variables
data["variables"] = dict(sorted(data["variables"].items()))
# Write back to file
with open(CONDITIONAL_TAGS_FILE, "w", encoding="utf-8") as f:
yaml.dump(data, f, sort_keys=False) # Don't sort top-level keys
def parse_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="Manage container image tags for CI builds with regex-based includes/excludes.",
epilog="""
Exit codes:
0 - Success
1 - Unhandled error in this script
2 - Tag mismatch when using --check
3 - Unbound variable error in build script
x - Build script failed with return code x
""",
)
parser.add_argument(
"--include",
"-i",
action="append",
default=[],
help="Full match regex pattern for components to include.",
)
parser.add_argument(
"--exclude",
"-x",
action="append",
default=[],
help="Full match regex pattern for components to exclude.",
)
parser.add_argument(
"--all", action="store_true", help="Equivalent to --include '.*'"
)
parser.add_argument(
"--check",
"-c",
action="append",
default=[],
help="Check matching components instead of updating YAML. "
"If any component fails, exit with a non-zero exit code.",
)
parser.add_argument(
"--list",
"-l",
action="store_true",
help="List all available components and exit.",
)
parser.add_argument(
"-v",
"--verbose",
action="count",
default=0,
help="Increase verbosity level (-v for info, -vv for debug)",
)
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
return parser.parse_args()
def main():
args = parse_args()
# Configure logging based on verbosity level
if args.verbose == 1:
log_level = logging.INFO
elif args.verbose == 2:
log_level = logging.DEBUG
else:
log_level = logging.WARNING
logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s")
# 0) Check if the YAML file exists
if not os.path.isfile(CONDITIONAL_TAGS_FILE):
logging.fatal(
f"Conditional build image tags file not found: {CONDITIONAL_TAGS_FILE}"
)
return
# 1) If checking, just run build scripts in check mode and propagate errors
if args.check:
tag_mismatch = False
for comp in args.check:
try:
if run_build_script(comp, check_only=True) is None:
# The tag is invalid
tag_mismatch = True
except SystemExit as e:
# Let custom exit codes propagate
raise e
except Exception as e:
logging.error(f"Internal error: {e}")
sys.exit(3)
# If any component failed, exit with code 1
if tag_mismatch:
sys.exit(2)
return
# Convert --all into a wildcard include
if args.all:
args.include.append(".*")
# 2) If --list, just show all discovered components
all_components = find_candidate_components()
if args.list:
print("Detected components:", ", ".join(all_components))
return
# 3) Filter components
final_components = filter_components(all_components, args.include, args.exclude)
if args.verbose:
logging.debug(f"Found components: {all_components}")
logging.debug(f"Filtered components: {final_components}")
for comp in final_components:
logging.info(f"Updating {comp}...")
new_tag = run_build_script(comp, check_only=False)
if new_tag is not None:
update_image_tag_in_yaml(comp, new_tag)
if args.verbose:
logging.debug(f"Updated {comp} with tag: {new_tag}")
if __name__ == "__main__":
main()