ci/b2c: de-duplicate the download of install.tar
By having the DUTs download and extract from a CI-Tron artifact, we deduplicate the downloading of the build artifact across all DUTs from a CI farm, leading to quicker and more reliable jobs, and lower bandwidth usage on both FD.o and the CI gateway. Inside the CI-Tron infra, this should also drastically reduce the job submission time by removing needless copies (executorctl -> executor, executor -> S3, S3 -> B2C, and even B2C -> NBD when applicable). As an additional bonus, the size of install.tar is reduced by virtue of zstd providing better compression than zip/deflate. Signed-off-by: Martin Roukala (né Peres) <martin.roukala@mupuf.org> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32927>
This commit is contained in:
Martin Roukala (né Peres)
Marge Bot
parent
ab3c0c4fa3
commit
f72ee5e587
2 changed files with 44 additions and 28 deletions
|
|
@ -97,6 +97,8 @@ deployment:
|
|||
{% endif %}
|
||||
|
||||
http:
|
||||
- path: "/install.tar.zst"
|
||||
url: "{{ install_tarball_url }}"
|
||||
- path: "/b2c-extra-args"
|
||||
data: >
|
||||
b2c.pipefail b2c.poweroff_delay={{ poweroff_delay }}
|
||||
|
|
@ -107,7 +109,7 @@ deployment:
|
|||
{% endfor %}
|
||||
b2c.run_service="--privileged --tls-verify=false --pid=host {{ telegraf_image }}" b2c.hostname=dut-{{ '{{' }} machine.full_name }}
|
||||
b2c.run="-ti --tls-verify=false {{ machine_registration_image }} check"
|
||||
b2c.run="-v {{ '{{' }} job_bucket }}-results:{{ working_dir }} -w {{ working_dir }} {% for mount_volume in mount_volumes %} -v {{ mount_volume }}{% endfor %} --tls-verify=false {{image_under_test}} {{ container_cmd | replace('"', '\\\"') }}"
|
||||
b2c.run="-v {{ '{{' }} job_bucket }}-results:{{ working_dir }} -w {{ working_dir }} {% for mount_volume in mount_volumes %} -v {{ mount_volume }}{% endfor %} --tls-verify=false --entrypoint bash {{image_under_test}} -euc 'curl -q {{ '{{' }} job.http.url }}/install.tar.zst | tar --zstd -x; {{ container_cmd }}'"
|
||||
kernel:
|
||||
{% if kernel_url %}
|
||||
url: '{{ kernel_url }}'
|
||||
|
|
|
|||
|
|
@ -319,13 +319,19 @@ yaml-toml-shell-py-test:
|
|||
# we rename the MESA_IMAGE variable into IMAGE_UNDER_TEST.
|
||||
IMAGE_UNDER_TEST: "$MESA_IMAGE"
|
||||
|
||||
INSTALL_TARBALL_NAME: "install.tar"
|
||||
INSTALL_TARBALL: "./artifacts/${INSTALL_TARBALL_NAME}"
|
||||
B2C_INSTALL_TARBALL_URL: "https://${PIPELINE_ARTIFACTS_BASE}/${S3_ARTIFACT_NAME}.tar.zst"
|
||||
B2C_CONTAINER_CMD: "./install/common/init-stage2.sh"
|
||||
|
||||
CI_B2C_ARTIFACTS: "./artifacts/b2c"
|
||||
CI_COMMON_SCRIPTS: "./artifacts/ci-common"
|
||||
B2C_JOB_TEMPLATE: "${CI_B2C_ARTIFACTS}/b2c.yml.jinja2.jinja2"
|
||||
JOB_FOLDER: "job_folder"
|
||||
|
||||
needs:
|
||||
- job: python-artifacts
|
||||
artifacts: true
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
|
||||
before_script:
|
||||
# We don't want the tarball unpacking of .test, but will take the JWT bits.
|
||||
- !reference [default, before_script]
|
||||
|
|
@ -350,12 +356,9 @@ yaml-toml-shell-py-test:
|
|||
fi
|
||||
fi
|
||||
|
||||
[ -s "$INSTALL_TARBALL" ] || exit 1
|
||||
[ -d "$CI_B2C_ARTIFACTS" ] || exit 1
|
||||
[ -d "$CI_COMMON_SCRIPTS" ] || exit 1
|
||||
|
||||
export B2C_CONTAINER_CMD="bash -euc 'tar xf ${INSTALL_TARBALL_NAME}; ./install/common/init-stage2.sh'"
|
||||
|
||||
# The Valve CI gateway receives jobs in a YAML format. Create a
|
||||
# job description from the CI environment.
|
||||
python3 "$CI_B2C_ARTIFACTS"/generate_b2c.py
|
||||
|
|
@ -374,9 +377,6 @@ yaml-toml-shell-py-test:
|
|||
cat ${JOB_FOLDER}/set-job-env-vars.sh
|
||||
set -x
|
||||
|
||||
# Copy the mesa install tarball to the job folder, for later extraction
|
||||
mv "${INSTALL_TARBALL}" "${JOB_FOLDER}"
|
||||
|
||||
script: |
|
||||
slugify () {
|
||||
echo "$1" | sed -r s/[~\^]+//g | sed -r s/[^a-zA-Z0-9]+/-/g | sed -r s/^-+\|-+$//g | tr A-Z a-z
|
||||
|
|
@ -411,6 +411,7 @@ yaml-toml-shell-py-test:
|
|||
variables:
|
||||
B2C_KERNEL_URL: 'https://gitlab.freedesktop.org/gfx-ci/ci-tron/-/package_files/519/download' # Linux 6.1
|
||||
B2C_INITRAMFS_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/initramfs.linux_amd64.cpio.xz'
|
||||
S3_ARTIFACT_NAME: "mesa-x86_64-default-debugoptimized"
|
||||
|
||||
.b2c-x86_64-test-vk:
|
||||
extends:
|
||||
|
|
@ -419,17 +420,20 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/x86_64_test-vk
|
||||
artifacts: false
|
||||
- debian-testing
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-testing
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-x86_64-test-vk-manual:
|
||||
extends:
|
||||
- .use-debian/x86_64_test-vk
|
||||
- .b2c-x86_64-test
|
||||
needs:
|
||||
- debian/x86_64_test-vk
|
||||
- debian-build-testing
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian/x86_64_test-vk
|
||||
artifacts: false
|
||||
- job: debian-build-testing
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-x86_64-test-gl:
|
||||
extends:
|
||||
|
|
@ -438,8 +442,9 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/x86_64_test-gl
|
||||
artifacts: false
|
||||
- debian-testing
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-testing
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-x86_64-test-gl-manual:
|
||||
extends:
|
||||
|
|
@ -448,8 +453,9 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/x86_64_test-gl
|
||||
artifacts: false
|
||||
- debian-build-testing
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-build-testing
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-arm64-test:
|
||||
extends:
|
||||
|
|
@ -457,6 +463,7 @@ yaml-toml-shell-py-test:
|
|||
variables:
|
||||
B2C_INITRAMFS_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/initramfs.linux_arm64.cpio.xz'
|
||||
B2C_KERNEL_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/linux-arm64'
|
||||
S3_ARTIFACT_NAME: "mesa-arm64-default-debugoptimized"
|
||||
|
||||
.b2c-arm64-test-vk:
|
||||
extends:
|
||||
|
|
@ -465,8 +472,9 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/arm64_test-vk
|
||||
artifacts: false
|
||||
- debian-arm64
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-arm64
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-arm64-test-gl:
|
||||
extends:
|
||||
|
|
@ -475,8 +483,9 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/arm64_test-gl
|
||||
artifacts: false
|
||||
- debian-arm64
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-arm64
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-arm32-test:
|
||||
extends:
|
||||
|
|
@ -484,6 +493,7 @@ yaml-toml-shell-py-test:
|
|||
variables:
|
||||
B2C_INITRAMFS_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/initramfs.linux_arm.cpio.xz'
|
||||
B2C_KERNEL_URL: 'https://gitlab.freedesktop.org/gfx-ci/boot2container/-/releases/${B2C_VERSION}/downloads/linux-arm'
|
||||
S3_ARTIFACT_NAME: "mesa-arm32-default-debugoptimized"
|
||||
|
||||
.b2c-arm32-test-vk:
|
||||
extends:
|
||||
|
|
@ -492,8 +502,9 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/arm32_test-vk
|
||||
artifacts: false
|
||||
- debian-arm32
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-arm32
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-arm32-test-gl:
|
||||
extends:
|
||||
|
|
@ -502,17 +513,20 @@ yaml-toml-shell-py-test:
|
|||
needs:
|
||||
- job: debian/arm32_test-gl
|
||||
artifacts: false
|
||||
- debian-arm32
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-arm32
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
||||
.b2c-arm32-test-asan-gl:
|
||||
extends:
|
||||
- .use-debian/arm32_test-gl
|
||||
- .b2c-arm32-test
|
||||
variables:
|
||||
S3_ARTIFACT_NAME: "mesa-arm32-asan-debugoptimized"
|
||||
DEQP_FORCE_ASAN: 1
|
||||
needs:
|
||||
- job: debian/arm32_test-gl
|
||||
artifacts: false
|
||||
- debian-arm32-asan
|
||||
- !reference [.required-for-hardware-jobs, needs]
|
||||
- job: debian-arm32-asan
|
||||
artifacts: false
|
||||
- !reference [.b2c-test, needs]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue