From 9c49138b1a36a08d369703e3bc6cb39ead035020 Mon Sep 17 00:00:00 2001 From: Valentine Burley Date: Fri, 1 Aug 2025 11:39:27 +0200 Subject: [PATCH] ci/lava: Use init-stage1 from Mesa build instead of inlining it We can use the init-stage1 script from the Mesa build artifacts instead of keeping a copy in the lava-trigger container and inlining it into the LAVA job definition. v2 (Guilherme) * Source the `init-stage1.sh` file instead of executing it * Avoid hardcoding the run_steps asserts, check only the core parts Signed-off-by: Valentine Burley Co-authored-by: Guilherme Gallo Part-of: --- .gitlab-ci/common/init-stage1.sh | 13 ++++++++----- .../container/alpine/x86_64_lava-trigger.sh | 1 - .gitlab-ci/image-tags.yml | 2 +- .gitlab-ci/lava/lava-submit.sh | 1 - .gitlab-ci/lava/lava_job_submitter.py | 1 - ...TBOOT_force_uart=False_job_definition.yaml | 4 ++-- ...STBOOT_force_uart=True_job_definition.yaml | 5 +++-- ...UBOOT_force_uart=False_job_definition.yaml | 4 ++-- .../UBOOT_force_uart=True_job_definition.yaml | 5 +++-- .../tests/utils/test_lava_job_definition.py | 13 +++++++------ .gitlab-ci/lava/utils/lava_job_definition.py | 19 ++++++++----------- 11 files changed, 34 insertions(+), 34 deletions(-) diff --git a/.gitlab-ci/common/init-stage1.sh b/.gitlab-ci/common/init-stage1.sh index 5fbe638a5b5..26637684937 100755 --- a/.gitlab-ci/common/init-stage1.sh +++ b/.gitlab-ci/common/init-stage1.sh @@ -3,10 +3,6 @@ # Very early init, used to make sure devices and network are set up and # reachable. -# When changing this file, you need to bump the following -# .gitlab-ci/image-tags.yml tags: -# ALPINE_X86_64_LAVA_TRIGGER_TAG - set -ex cd / @@ -21,7 +17,14 @@ mkdir -p /dev/shm findmnt --mountpoint /dev/shm || mount -t tmpfs -o noexec,nodev,nosuid tmpfs /dev/shm findmnt --mountpoint /tmp || mount -t tmpfs tmpfs /tmp -echo "nameserver 8.8.8.8" > /etc/resolv.conf +# For the vmware farm, change the nameserver as 8.8.8.8 is off limit. +# This is temporary and will be reverted once the farm is moved. +if [ "${FARM:-}" = "vmware" ]; then + echo "nameserver 192.19.189.10" > /etc/resolv.conf +else + echo "nameserver 8.8.8.8" > /etc/resolv.conf +fi + [ -z "$NFS_SERVER_IP" ] || echo "$NFS_SERVER_IP caching-proxy" >> /etc/hosts # Set the time so we can validate certificates before we fetch anything; diff --git a/.gitlab-ci/container/alpine/x86_64_lava-trigger.sh b/.gitlab-ci/container/alpine/x86_64_lava-trigger.sh index a9724ee621a..e625e76532e 100644 --- a/.gitlab-ci/container/alpine/x86_64_lava-trigger.sh +++ b/.gitlab-ci/container/alpine/x86_64_lava-trigger.sh @@ -35,7 +35,6 @@ pip3 install --break-system-packages -r bin/ci/requirements-lava.txt cp -Rp .gitlab-ci/lava / cp -Rp .gitlab-ci/bin/structured_logger.py /lava -cp -Rp .gitlab-ci/common/init-stage1.sh /lava . .gitlab-ci/container/container_pre_build.sh diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml index 483960c68bd..997fcb5696b 100644 --- a/.gitlab-ci/image-tags.yml +++ b/.gitlab-ci/image-tags.yml @@ -30,7 +30,7 @@ variables: ALPINE_X86_64_BUILD_TAG: "20250722-libwayland" ALPINE_X86_64_LAVA_SSH_TAG: "20250423-rootfs" - ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250804-logger" + ALPINE_X86_64_LAVA_TRIGGER_TAG: "20250804-init" FEDORA_X86_64_BUILD_TAG: "20250722-libwayland" diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh index 124ecf1ec2f..f445b66aa91 100755 --- a/.gitlab-ci/lava/lava-submit.sh +++ b/.gitlab-ci/lava/lava-submit.sh @@ -126,7 +126,6 @@ PYTHONPATH=/ /lava/lava_job_submitter.py \ --rootfs-url "${ROOTFS_URL}" \ --kernel-url-prefix "${KERNEL_IMAGE_BASE}/${DEBIAN_ARCH}" \ --dtb-filename "${DTB}" \ - --first-stage-init /lava/init-stage1.sh \ --env-file dut-env-vars.sh \ --jwt-file "${S3_JWT_FILE}" \ --kernel-image-name "${KERNEL_IMAGE_NAME}" \ diff --git a/.gitlab-ci/lava/lava_job_submitter.py b/.gitlab-ci/lava/lava_job_submitter.py index 2cb5f9f88b5..8cbf5865c1a 100755 --- a/.gitlab-ci/lava/lava_job_submitter.py +++ b/.gitlab-ci/lava/lava_job_submitter.py @@ -407,7 +407,6 @@ class LAVAJobSubmitter(PathResolver): shell_prompt: str = "lava-shell:" dtb_filename: str = None dump_yaml: bool = False # Whether to dump the YAML payload to stdout - first_stage_init: str = None env_file: pathlib.Path = None jwt_file: pathlib.Path = None kernel_image_name: str = None diff --git a/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=False_job_definition.yaml index 3ee8d4b3e3e..3d7900c9fae 100644 --- a/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=False_job_definition.yaml +++ b/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=False_job_definition.yaml @@ -88,8 +88,8 @@ actions: run: steps: - |- - echo test FASTBOOT - echo ZWNobyB0ZXN0IEZBU1RCT09U | base64 -d >> /set-job-env-vars.sh + FARM=test_farm . /test_dir/install/common/init-stage1.sh + echo RkFSTT10ZXN0X2Zhcm0gL3Rlc3RfZGlyL2luc3RhbGwvY29tbW9uL2luaXQtc3RhZ2UxLnNo | base64 -d >> /set-job-env-vars.sh export CURRENT_SECTION=dut_boot - export -p > /dut-env-vars.sh - test: diff --git a/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=True_job_definition.yaml index 64fc0b16422..f1ca57cbb58 100644 --- a/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=True_job_definition.yaml +++ b/.gitlab-ci/lava/tests/data/FASTBOOT_force_uart=True_job_definition.yaml @@ -84,8 +84,9 @@ actions: format: Lava-Test Test Definition 1.0 run: steps: - - echo test FASTBOOT - - echo ZWNobyB0ZXN0IEZBU1RCT09U | base64 -d >> /set-job-env-vars.sh + - FARM=test_farm . /test_dir/install/common/init-stage1.sh + - echo RkFSTT10ZXN0X2Zhcm0gL3Rlc3RfZGlyL2luc3RhbGwvY29tbW9uL2luaXQtc3RhZ2UxLnNo + | base64 -d >> /set-job-env-vars.sh - export CURRENT_SECTION=dut_boot - set -e - echo Could not find jwt file, disabling S3 requests... diff --git a/.gitlab-ci/lava/tests/data/UBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/lava/tests/data/UBOOT_force_uart=False_job_definition.yaml index 2e7b260d106..6564eefc2f3 100644 --- a/.gitlab-ci/lava/tests/data/UBOOT_force_uart=False_job_definition.yaml +++ b/.gitlab-ci/lava/tests/data/UBOOT_force_uart=False_job_definition.yaml @@ -59,8 +59,8 @@ actions: run: steps: - |- - echo test UBOOT - echo ZWNobyB0ZXN0IFVCT09U | base64 -d >> /set-job-env-vars.sh + FARM=test_farm . /test_dir/install/common/init-stage1.sh + echo RkFSTT10ZXN0X2Zhcm0gL3Rlc3RfZGlyL2luc3RhbGwvY29tbW9uL2luaXQtc3RhZ2UxLnNo | base64 -d >> /set-job-env-vars.sh export CURRENT_SECTION=dut_boot - export -p > /dut-env-vars.sh - test: diff --git a/.gitlab-ci/lava/tests/data/UBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/lava/tests/data/UBOOT_force_uart=True_job_definition.yaml index 7f07b095ae7..f68b12f8909 100644 --- a/.gitlab-ci/lava/tests/data/UBOOT_force_uart=True_job_definition.yaml +++ b/.gitlab-ci/lava/tests/data/UBOOT_force_uart=True_job_definition.yaml @@ -57,8 +57,9 @@ actions: format: Lava-Test Test Definition 1.0 run: steps: - - echo test UBOOT - - echo ZWNobyB0ZXN0IFVCT09U | base64 -d >> /set-job-env-vars.sh + - FARM=test_farm . /test_dir/install/common/init-stage1.sh + - echo RkFSTT10ZXN0X2Zhcm0gL3Rlc3RfZGlyL2luc3RhbGwvY29tbW9uL2luaXQtc3RhZ2UxLnNo + | base64 -d >> /set-job-env-vars.sh - export CURRENT_SECTION=dut_boot - set -e - echo Could not find jwt file, disabling S3 requests... diff --git a/.gitlab-ci/lava/tests/utils/test_lava_job_definition.py b/.gitlab-ci/lava/tests/utils/test_lava_job_definition.py index d24e23bf67d..269bb2cae4c 100644 --- a/.gitlab-ci/lava/tests/utils/test_lava_job_definition.py +++ b/.gitlab-ci/lava/tests/utils/test_lava_job_definition.py @@ -60,7 +60,6 @@ def job_submitter_factory(mode: Literal["UBOOT", "FASTBOOT"], shell_file): device_type=device_type, farm="test_farm", dtb_filename="my_dtb_filename", - first_stage_init=shell_file, env_file=shell_file, job_timeout_min=job_timeout_min, mesa_job_name=mesa_job_name, @@ -108,8 +107,9 @@ def test_generate_lava_job_definition_sanity( # Do not actually connect to the LAVA server mock_lava_proxy.return_value = mock_proxy - init_script_content = f"echo test {mode}" - job_submitter = job_submitter_factory(mode, shell_file(init_script_content)) + farm_env = "FARM=test_farm" + init_script = "/install/common/init-stage1.sh" + job_submitter = job_submitter_factory(mode, shell_file(init_script)) job_definition = LAVAJobDefinition(job_submitter).generate_lava_job_definition() # Load the YAML output and check that it contains the expected keys and values @@ -184,7 +184,8 @@ def test_generate_lava_job_definition_sanity( "test", ] ) - assert init_script_content in run_steps + assert farm_env in run_steps + assert init_script in run_steps # use yaml files from tests/data/ to test the job definition generation @@ -210,8 +211,8 @@ def test_lava_job_definition( # Load the YAML output and check that it contains the expected keys and values expected_job_dict = load_yaml_file(f"{mode}_force_uart={force_uart}_job_definition.yaml") - init_script_content = f"echo test {mode}" - job_submitter = job_submitter_factory(mode, shell_file(init_script_content)) + init_script = f"FARM=test_farm /test_dir/install/common/init-stage1.sh" + job_submitter = job_submitter_factory(mode, shell_file(init_script)) job_definition = LAVAJobDefinition(job_submitter).generate_lava_job_definition() job_dict = yaml.load(job_definition) diff --git a/.gitlab-ci/lava/utils/lava_job_definition.py b/.gitlab-ci/lava/utils/lava_job_definition.py index 2b00550c35b..711d030eac2 100644 --- a/.gitlab-ci/lava/utils/lava_job_definition.py +++ b/.gitlab-ci/lava/utils/lava_job_definition.py @@ -253,20 +253,17 @@ class LAVAJobDefinition: return steps def init_stage1_steps(self) -> list[str]: - run_steps = [] # job execution script: - # - inline .gitlab-ci/common/init-stage1.sh + # - source .gitlab-ci/common/init-stage1.sh # - fetch and unpack per-pipeline build artifacts from build job - # - fetch, unpack and encode per-job env from lava-submit.sh + # - inline and encode per-job env from lava-submit.sh # - exec .gitlab-ci/common/init-stage2.sh - - with open(self.job_submitter.first_stage_init, "r") as init_sh: - # For vmware farm, patch nameserver as 8.8.8.8 is off limit. - # This is temporary and will be reverted once the farm is moved. - if self.job_submitter.mesa_job_name.startswith("vmware-"): - run_steps += [x.rstrip().replace("nameserver 8.8.8.8", "nameserver 192.19.189.10") for x in init_sh if not x.startswith("#") and x.rstrip()] - else: - run_steps += [x.rstrip() for x in init_sh if not x.startswith("#") and x.rstrip()] + run_steps = [ + f"FARM={self.job_submitter.farm} " + # We need to source the init-stage1.sh script, so that the environment + # variables including PWD are set in the current shell. + f". {self.job_submitter.project_dir}/install/common/init-stage1.sh" + ] # We cannot distribute the Adreno 660 shader firmware inside rootfs, # since the license isn't bundled inside the repository