diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index e42977b5b8b..f33bcd6ca9b 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -30,7 +30,7 @@ variables: ALPINE_X86_64_BUILD_TAG: "20250423-rootfs" ALPINE_X86_64_LAVA_SSH_TAG: "20250423-rootfs" - ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250609-init" + ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250610-eval" FEDORA_X86_64_BUILD_TAG: "20250423-rootfs" diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py index 98a95f4fe4e..1bbd2c06265 100644 --- a/.gitlab-ci/lava/utils/lava_job_definition.py +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -236,12 +236,24 @@ class LAVAJobDefinition: return jwt_steps + def encode_job_env_vars(self) -> list[str]: + steps = [] + with open(self.job_submitter.env_file, "rb") as f: + encoded = base64.b64encode(f.read()).decode() + safe_encoded = shlex.quote(encoded) + + steps += [ + f'echo {safe_encoded} | base64 -d >> /set-job-env-vars.sh', + ] + + return steps + def init_stage1_steps(self) -> list[str]: run_steps = [] # job execution script: # - inline .gitlab-ci/common/init-stage1.sh # - fetch and unpack per-pipeline build artifacts from build job - # - fetch and unpack per-job environment from lava-submit.sh + # - fetch, unpack and encode per-job env from lava-submit.sh # - exec .gitlab-ci/common/init-stage2.sh with open(self.job_submitter.first_stage_init, "r") as init_sh: @@ -264,12 +276,7 @@ class LAVAJobDefinition: # Forward environmental variables to the DUT # base64-encoded to avoid YAML quoting issues - with open(self.job_submitter.env_file, "rb") as f: - encoded = base64.b64encode(f.read()).decode() - safe_encoded = shlex.quote(encoded) - run_steps += [ - f'echo "eval \\\"$(echo {safe_encoded} | base64 -d)\\\"" >> /set-job-env-vars.sh', - ] + run_steps += self.encode_job_env_vars() run_steps.append("export CURRENT_SECTION=dut_boot") diff --git a/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml index 74fcef8e1b9..13f1b21ef44 100644 --- a/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml +++ b/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml @@ -89,7 +89,7 @@ actions: steps: - |- echo test FASTBOOT - echo "eval \"$(echo ZWNobyB0ZXN0IEZBU1RCT09U | base64 -d)\"" >> /set-job-env-vars.sh + echo ZWNobyB0ZXN0IEZBU1RCT09U | base64 -d >> /set-job-env-vars.sh export CURRENT_SECTION=dut_boot - export -p > /dut-env-vars.sh - test: diff --git a/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml index 97a47a263b5..64fc0b16422 100644 --- a/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml +++ b/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml @@ -85,7 +85,7 @@ actions: run: steps: - echo test FASTBOOT - - echo "eval \"$(echo ZWNobyB0ZXN0IEZBU1RCT09U | base64 -d)\"" >> /set-job-env-vars.sh + - echo ZWNobyB0ZXN0IEZBU1RCT09U | base64 -d >> /set-job-env-vars.sh - export CURRENT_SECTION=dut_boot - set -e - echo Could not find jwt file, disabling S3 requests... diff --git a/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml index 36a400e4d7e..4eddf348150 100644 --- a/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml +++ b/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml @@ -60,7 +60,7 @@ actions: steps: - |- echo test UBOOT - echo "eval \"$(echo ZWNobyB0ZXN0IFVCT09U | base64 -d)\"" >> /set-job-env-vars.sh + echo ZWNobyB0ZXN0IFVCT09U | base64 -d >> /set-job-env-vars.sh export CURRENT_SECTION=dut_boot - export -p > /dut-env-vars.sh - test: diff --git a/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml index be20d2db726..7f07b095ae7 100644 --- a/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml +++ b/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml @@ -58,7 +58,7 @@ actions: run: steps: - echo test UBOOT - - echo "eval \"$(echo ZWNobyB0ZXN0IFVCT09U | base64 -d)\"" >> /set-job-env-vars.sh + - echo ZWNobyB0ZXN0IFVCT09U | base64 -d >> /set-job-env-vars.sh - export CURRENT_SECTION=dut_boot - set -e - echo Could not find jwt file, disabling S3 requests... diff --git a/.gitlab-ci/tests/utils/test_lava_job_definition.py b/.gitlab-ci/tests/utils/test_lava_job_definition.py index 53e27a40b40..05f6643e3cc 100644 --- a/.gitlab-ci/tests/utils/test_lava_job_definition.py +++ b/.gitlab-ci/tests/utils/test_lava_job_definition.py @@ -1,6 +1,7 @@ import importlib import os import re +import subprocess from itertools import chain from pathlib import Path from typing import Any, Iterable, Literal @@ -217,3 +218,101 @@ def test_lava_job_definition( # Check that the generated job definition matches the expected one assert job_dict == expected_job_dict + + +@pytest.mark.parametrize( + "directive", + ["declare -x", "export"], +) +@pytest.mark.parametrize( + "original_env_output", + [ + # Test basic environment variables + "FOO=bar\nBAZ=qux", + # Test export statements + "{directive} FOO=bar", + # Test multiple exports + "{directive} FOO=bar\n{directive} BAZ=qux\nNORM=val", + # Test mixed content with export + "{directive} FOO=bar\nBAZ=qux\n{directive} HELLO=world", + # Test empty file + "", + # Test special characters that need shell quoting + "FOO='bar baz'\nQUOTE=\"hello world\"", + # Test variables with spaces and quotes + "{directive} VAR='val spaces'\nQUOTES=\"test\"", + # Test inline scripts with export + "{directive} FOO=bar\nBAZ=qux\n{directive} HELLO=world", + # Test single quote inside double quotes in variable + "{directive} FOO='Revert \"commit's error\"'", + # Test backticks in variable + "{directive} FOO=`echo 'test'`", + ], + ids=[ + "basic_vars", + "single_export", + "multiple_exports", + "mixed_exports", + "empty_file", + "special_chars", + "spaces_and_quotes", + "inline_scripts_with_export", + "single_quote_in_var", + "backticks", + ] +) +def test_encode_job_env_vars(directive, original_env_output, shell_file, clear_env_vars): + """Test the encode_job_env_vars function with various environment file contents.""" + import base64 + import shlex + + # Create environment file with test content + original_env_output = original_env_output.format(directive=directive) + env_file = shell_file(original_env_output) + + # Create job submitter with the environment file + job_submitter = mock.MagicMock(spec=LAVAJobSubmitter, env_file=env_file) + job_definition = LAVAJobDefinition(job_submitter) + + # Call the function under test + result = job_definition.encode_job_env_vars() + + # Verify the result is a list with exactly one element + assert isinstance(result, list) + assert len(result) == 1 + + # Extract the command from the result + command = result[0] + assert isinstance(command, str) + + # Extract the base64 encoded part + start_marker = 'echo ' + end_marker = ' | base64 -d' + + start_idx = command.find(start_marker) + len(start_marker) + end_idx = command.find(end_marker) + redirect_idx = command.find(">") + encoded_part = command[start_idx:end_idx] + + # Verify if the script is executed correctly + env_script_process = subprocess.run( + ["bash", "-c", command[:redirect_idx]], capture_output=True, text=True + ) + + if env_script_process.returncode != 0: + pytest.fail(f"Failed to execute script: {env_script_process.stderr}") + + generated_env_output = env_script_process.stdout.strip() + + # The encoded part should be shell-quoted, so we need to parse it + # Use shlex to unquote the encoded content + unquoted_encoded = shlex.split(encoded_part)[0] + + # Decode the base64 content + try: + decoded_content = base64.b64decode(unquoted_encoded).decode() + except Exception as e: + pytest.fail(f"Failed to decode base64 content: {e}. Encoded part: {encoded_part}") + + # Verify the decoded content matches the original file content + assert decoded_content == original_env_output == generated_env_output