|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH v1 07/13] CI: rework to allow hardware runners
From: Denis Mukhin <dmukhin@xxxxxxxx> Rework current bash tooling to simplify its use for local testing and CI integration. Allow batching multiple tests in one run. The batching rules are: - Use 'SELECTED_GROUP_ONLY' for running only XTF group of jobs. If unset, run all XTF jobs. - Use file passed in 'XTF_MANIFEST' to select a group of tests to run. If unset, run all known tests. - Use regex in 'SELECTED_XTF_ONLY' environment variable to select a group of XTF tests. If unset, run all tests. Correctly resolve individual XTF test statuses returned from console.exp for pretty accounting. Include per-test title in the output. Rely on per-XTF 'info.json' for assembling per-test Xen command line. Prefix all per-arch XTF QEMU configuration fragments with "qemu-" prefix and move them under 'xtf' directory. While here, rename qemu-xtf.sh to xtf.sh since the code is now generic, not QEMU specific. Update all affected QEMU XTF jobs. Signed-off-by: Denis Mukhin <dmukhin@xxxxxxxx> --- That may clash with https://lore.kernel.org/xen-devel/20260922100900.29129-1-anthony@xxxxxxxxxxxxxx/ and https://lore.kernel.org/xen-devel/1793715358.8631fc262581453bbf619ec5b2062170.1a1a1c44b87000403d@xxxxxxxxxx My take - pytests is the way to go, but I had to unblock myself with running some AVIC testing, so ended up with existing bash infra cleanup. --- automation/gitlab-ci/test.yaml | 165 +++++--- .../scripts/include/configs/xtf-arm64-config | 2 - .../scripts/include/configs/xtf-x86_64-config | 0 .../include/configs/xtf-x86_64-efi-config | 0 .../scripts/include/manifests/qemu-x86_64 | 75 ++++ automation/scripts/include/xtf-runner | 138 ------ .../include/{xtf-arm64 => xtf/qemu-arm64} | 13 +- .../include/{xtf-x86_64 => xtf/qemu-x86_64} | 13 +- .../{xtf-x86_64-efi => xtf/qemu-x86_64-efi} | 13 +- automation/scripts/qemu-xtf.sh | 28 -- automation/scripts/xtf | 395 ++++++++++++++++++ 11 files changed, 590 insertions(+), 252 deletions(-) delete mode 100644 automation/scripts/include/configs/xtf-arm64-config delete mode 100644 automation/scripts/include/configs/xtf-x86_64-config delete mode 100644 automation/scripts/include/configs/xtf-x86_64-efi-config create mode 100644 automation/scripts/include/manifests/qemu-x86_64 delete mode 100644 automation/scripts/include/xtf-runner rename automation/scripts/include/{xtf-arm64 => xtf/qemu-arm64} (79%) rename automation/scripts/include/{xtf-x86_64 => xtf/qemu-x86_64} (53%) rename automation/scripts/include/{xtf-x86_64-efi => xtf/qemu-x86_64-efi} (75%) delete mode 100755 automation/scripts/qemu-xtf.sh create mode 100755 automation/scripts/xtf diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml index 047c4d513a65..9ca93842dfe7 100644 --- a/automation/gitlab-ci/test.yaml +++ b/automation/gitlab-ci/test.yaml @@ -5,6 +5,31 @@ - if: $CI_JOB_NAME =~ $SELECTED_JOBS_ONLY when: on_success +.test-jobs-group: + extends: + - .test-jobs-common + rules: + - if: '$CI_JOB_NAME =~ $SELECTED_JOBS_ONLY && $SELECTED_GROUP_ONLY =~ /(^|,)(all|[*])(,|$)/' + when: on_success + - if: '$CI_JOB_NAME =~ $SELECTED_JOBS_ONLY && $SELECTED_GROUP_ONLY =~ $GROUP_REGEX' + when: on_success + - if: '$CI_JOB_NAME =~ $SELECTED_JOBS_ONLY' + when: manual + allow_failure: true + - when: never + +.test-jobs-perf: + extends: + - .test-jobs-group + variables: + GROUP_REGEX: '/(^|,)perf(,|$)/' + +.test-jobs-hardware-xtf: + extends: + - .test-jobs-group + variables: + GROUP_REGEX: '/(^|,)xtf(,|$)/' + .arm64-test-needs: &arm64-test-needs - project: $ARTIFACTS_REPO job: $LINUX_JOB_ARM64 @@ -27,11 +52,44 @@ job: microcode-x86 ref: $ARTIFACTS_BRANCH +.xtf-arm64-gcc-debug: + variables: + ARCH: arm64 + script: + - time ./automation/scripts/xtf 2>&1 | tee ${LOGFILE} + artifacts: + paths: + - '*.log' + when: always + needs: + - *arm64-test-needs + - alpine-3.24-arm64-gcc-debug + - xtf-arm64-gcc-debug + +.xtf-x86_64-gcc-debug: + variables: + ARCH: x86_64 + script: + - | + if [[ "${CONTAINER}" != "debian:13-x86_64" ]]; then + apk add --no-cache binutils + fi + - time ./automation/scripts/xtf 2>&1 | tee ${LOGFILE} + artifacts: + paths: + - '*.log' + when: always + needs: + - *x86_64-test-needs + - alpine-3.24-x86_64-gcc-debug + - xtf-x86_64-gcc-debug + .qemu-arm64: extends: .test-jobs-common variables: CONTAINER: debian:13-arm64v8 LOGFILE: qemu-smoke-arm64.log + XTF_TARGET: qemu-arm64 artifacts: paths: - smoke.serial @@ -58,6 +116,7 @@ variables: CONTAINER: debian:13-x86_64 LOGFILE: qemu-smoke-x86_64.log + XTF_TARGET: qemu-x86_64 artifacts: paths: - smoke.serial @@ -66,11 +125,6 @@ tags: - x86_64 -.qemu-smoke-x86_64: - extends: .qemu-x86_64 - variables: - TEST_TIMEOUT_OVERRIDE: 120 - .qemu-riscv64: extends: .test-jobs-common variables: @@ -503,6 +557,12 @@ zen2-suspend-x86_64-gcc-debug: - *x86_64-test-needs - alpine-3.24-x86_64-gcc-debug +zen2-xtf-x86_64-gcc-debug: + extends: + - .zen2-x86_64 + - .xtf-x86_64-gcc-debug + - .test-jobs-hardware-xtf + zen3p-smoke-x86_64-gcc-debug: extends: .zen3p-x86_64 script: @@ -565,6 +625,12 @@ zen3p-tools-tests-pvh-x86_64-gcc-debug: - *x86_64-test-needs - alpine-3.24-x86_64-gcc-debug +zen3p-xtf-x86_64-gcc-debug: + extends: + - .zen3p-x86_64 + - .xtf-x86_64-gcc-debug + - .test-jobs-hardware-xtf + qemu-smoke-dom0-arm64-gcc: extends: .qemu-arm64 script: @@ -653,21 +719,13 @@ qemu-smoke-dom0less-arm64-gcc-debug-earlyprintk: - *arm64-test-needs - alpine-3.24-arm64-gcc-debug-earlyprintk -qemu-xtf-dom0less-arm64-gcc-hyp-xen-version: - extends: .qemu-arm64 - script: - - ./automation/scripts/qemu-xtf.sh arm64 mmu64le hyp-xen-version 2>&1 | tee ${LOGFILE} - needs: - - *arm64-test-needs - - alpine-3.24-arm64-gcc - -qemu-xtf-dom0less-arm64-gcc-debug-hyp-xen-version: - extends: .qemu-arm64 - script: - - ./automation/scripts/qemu-xtf.sh arm64 mmu64le hyp-xen-version 2>&1 | tee ${LOGFILE} - needs: - - *arm64-test-needs - - alpine-3.24-arm64-gcc-debug +qemu-xtf-arm64-gcc-debug: + extends: + - .qemu-arm64 + - .test-jobs-common + - .xtf-arm64-gcc-debug + variables: + TEST_TIMEOUT_OVERRIDE: 10 qemu-smoke-dom0-arm32-gcc: extends: .qemu-arm32 @@ -741,49 +799,36 @@ qemu-alpine-x86_64-gcc: - *x86_64-test-needs - alpine-3.24-x86_64-gcc -qemu-smoke-x86_64-gcc: - extends: .qemu-smoke-x86_64 - script: - - ./automation/scripts/qemu-xtf.sh x86_64 pv64 example 2>&1 | tee ${LOGFILE} - needs: - - debian-13-x86_64-gcc-debug - -qemu-smoke-x86_64-clang: - extends: .qemu-smoke-x86_64 - script: - - ./automation/scripts/qemu-xtf.sh x86_64 pv64 example 2>&1 | tee ${LOGFILE} - needs: - - debian-13-x86_64-clang-debug - -qemu-smoke-x86_64-gcc-pvh: - extends: .qemu-smoke-x86_64 - script: - - ./automation/scripts/qemu-xtf.sh x86_64 hvm64 example 2>&1 | tee ${LOGFILE} - needs: - - debian-13-x86_64-gcc-debug - -qemu-smoke-x86_64-clang-pvh: - extends: .qemu-smoke-x86_64 - script: - - ./automation/scripts/qemu-xtf.sh x86_64 hvm64 example 2>&1 | tee ${LOGFILE} - needs: - - debian-13-x86_64-clang-debug - -qemu-smoke-x86_64-gcc-efi: - extends: .qemu-smoke-x86_64 - script: - - ./automation/scripts/qemu-xtf.sh x86_64-efi pv64 example 2>&1 | tee ${LOGFILE} - needs: - - debian-13-x86_64-gcc-debug - -qemu-xtf-argo-x86_64-gcc-debug: - extends: .qemu-smoke-x86_64 +qemu-xtf-x86_64-gcc-debug: + extends: + - .qemu-x86_64 + - .test-jobs-common + - .xtf-x86_64-gcc-debug variables: - TEST_TIMEOUT_OVERRIDE: 60 + TEST_TIMEOUT_OVERRIDE: 10 + XTF_MANIFEST: include/manifests/qemu-x86_64 + +qemu-xtf-x86_64-efi-gcc-debug: + extends: qemu-xtf-x86_64-gcc-debug + variables: + XTF_TARGET: qemu-x86_64-efi + +qemu-xtf-x86_64-clang-debug: + extends: .qemu-x86_64 + variables: + ARCH: x86_64 + TEST_TIMEOUT_OVERRIDE: 10 + XTF_MANIFEST: include/manifests/qemu-x86_64 script: - - ./automation/scripts/qemu-xtf.sh x86_64 pv64 argo 2>&1 | tee ${LOGFILE} + - time ./automation/scripts/xtf 2>&1 | tee ${LOGFILE} + artifacts: + paths: + - '*.log' + when: always needs: - - alpine-3.24-x86_64-gcc-debug + - *x86_64-test-needs + - debian-13-x86_64-clang-debug + - xtf-x86_64-clang-debug qemu-smoke-riscv64-gcc: extends: .qemu-riscv64 diff --git a/automation/scripts/include/configs/xtf-arm64-config b/automation/scripts/include/configs/xtf-arm64-config deleted file mode 100644 index 99427409276d..000000000000 --- a/automation/scripts/include/configs/xtf-arm64-config +++ /dev/null @@ -1,2 +0,0 @@ -CONFIG_GICV2=y -CONFIG_SBSA_UART=y diff --git a/automation/scripts/include/configs/xtf-x86_64-config b/automation/scripts/include/configs/xtf-x86_64-config deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/automation/scripts/include/configs/xtf-x86_64-efi-config b/automation/scripts/include/configs/xtf-x86_64-efi-config deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/automation/scripts/include/manifests/qemu-x86_64 b/automation/scripts/include/manifests/qemu-x86_64 new file mode 100644 index 000000000000..82b40deb94e5 --- /dev/null +++ b/automation/scripts/include/manifests/qemu-x86_64 @@ -0,0 +1,75 @@ +test-hvm32-argo +test-hvm32-cpuid +test-hvm32-debug-regs +test-hvm32-example +test-hvm32-invlpg +test-hvm32-lm-ts +test-hvm32-selftest +test-hvm32-xsa-122 +test-hvm32-xsa-188 +test-hvm32-xsa-239 +test-hvm32-xsa-317 +test-hvm32pae-argo +test-hvm32pae-cpuid +test-hvm32pae-debug-regs +test-hvm32pae-example +test-hvm32pae-invlpg +test-hvm32pae-selftest +test-hvm32pae-xsa-122 +test-hvm32pae-xsa-188 +test-hvm32pae-xsa-317 +test-hvm32pse-argo +test-hvm32pse-cpuid +test-hvm32pse-debug-regs +test-hvm32pse-example +test-hvm32pse-selftest +test-hvm32pse-xsa-122 +test-hvm32pse-xsa-188 +test-hvm32pse-xsa-317 +test-hvm64-argo +test-hvm64-cpuid +test-hvm64-debug-regs +test-hvm64-example +test-hvm64-invlpg +test-hvm64-lbr-tsx-vmentry +test-hvm64-lm-ts +test-hvm64-rtm-check +test-hvm64-selftest +test-hvm64-xsa-122 +test-hvm64-xsa-168 +test-hvm64-xsa-173 +test-hvm64-xsa-188 +test-hvm64-xsa-195 +test-hvm64-xsa-221 +test-hvm64-xsa-231 +test-hvm64-xsa-232 +test-hvm64-xsa-269 +test-hvm64-xsa-277 +test-hvm64-xsa-278 +test-hvm64-xsa-304 +test-hvm64-xsa-317 +test-hvm64-xsa-454 +test-hvm64-xsa-463 +test-pv64-argo +test-pv64-cpuid +test-pv64-example +test-pv64-rtm-check +test-pv64-xsa-122 +test-pv64-xsa-182 +test-pv64-xsa-188 +test-pv64-xsa-193 +test-pv64-xsa-212 +test-pv64-xsa-221 +test-pv64-xsa-224 +test-pv64-xsa-227 +test-pv64-xsa-231 +test-pv64-xsa-232 +test-pv64-xsa-234 +test-pv64-xsa-255 +test-pv64-xsa-259 +test-pv64-xsa-279 +test-pv64-xsa-286 +test-pv64-xsa-296 +test-pv64-xsa-317 +test-pv64-xsa-333 +test-pv64-xsa-7 diff --git a/automation/scripts/include/xtf-runner b/automation/scripts/include/xtf-runner deleted file mode 100644 index d1ef888465c3..000000000000 --- a/automation/scripts/include/xtf-runner +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# -# XTF test utilities. -# -# Environment variables: -# BOOT_MSG: Expected boot message -# FW_PREFIX: Firmware images path including '/' at the end -# PASSED: XTF test printout in case of a pass -# QEMU_PREFIX: QEMU path including '/' at the end -# TEST_LOG: Output log file -# UBOOT_CMD: U-Boot command line -# WORKDIR: Test working directory -# XEN_BINARY: Xen binary location -# XEN_CONSOLE: Xen console device name -# XTF_SRC_CONFIG: XTF config file -# XTF_SRC_BRANCH: XTF branch -# XTF_SRC_URI: XTF source code URI - -# Output log file -TEST_LOG="${TEST_LOG:-${XEN_ROOT}/smoke.serial}" -# XTF test printout in case of a pass -PASSED="${PASSED:-Test result: SUCCESS}" -# Expected boot message -BOOT_MSG="${BOOT_MSG:-Latest ChangeSet: }" -# Test working directory -WORKDIR="${WORKDIR:-${XEN_ROOT}/binaries}" -# XTF source code -XTF_SRC_CONFIG="${XTF_SRC_CONFIG:-include/configs/xtf-${ARCH}-config}" - -function die() -{ - set +x - echo "FATAL: $*" >&2 - exit 1 -} - -# Build an XTF test binary. -# $1 Test variant. -# $2 Test name. -function xtf_build_binary() -{ - local xtf_variant=$1 - local xtf_name=$2 - local xtf_dir="xtf-${ARCH}" - - # Crude check for local testing - if [ ! -d ${xtf_dir} ]; then - git clone ${XTF_SRC_URI} ${xtf_dir} -b ${XTF_SRC_BRANCH} - fi - - make \ - -C ${xtf_dir} \ - -j$(nproc) \ - $(tr '\n' ' ' < ${XTF_SRC_CONFIG}) \ - TESTS=tests/${xtf_name} - - export XTF_NAME="${xtf_name}" - export XTF_VARIANT="${xtf_variant}" - export XTF_WORKDIR="$(readlink -f ${xtf_dir})" - export XTF_BINARY="${XTF_WORKDIR}/tests/${xtf_name}/test-${xtf_variant}-${xtf_name}" -} - -# Build Xen command line for running an XTF test. -# $1 Test variant. -# $2 Test name. -function xtf_build_cmdline() -{ - local xtf_variant=$1 - local xtf_name=$2 - declare -a cmdline=() - declare -A per_test_args=( - [argo]="argo=1,mac-permissive=1" - ) - - cmdline+=("${XEN_CMDLINE}") - - # NB: OK to have hvm64, which is x86-only variant - if [[ $xtf_variant == "hvm64" ]]; then - cmdline+=("dom0-iommu=none dom0=pvh") - fi - - if [[ -v per_test_args[${xtf_name}] ]]; then - cmdline+=("${per_test_args[${xtf_name}]}") - fi - - export XEN_CMDLINE="${cmdline[@]}" -} - -# Build an XTF test environment. -# $1 Test variant. -# $2 Test name. -function xtf_build_test() -{ - local v=$1 - local xtf_name=$2 - local xtf_variant="" - - for x in ${XTF_SRC_VARIANTS}; do - if [[ "${x}" == "${v}" ]]; then - xtf_variant=${v} - break - fi - done - if [[ -z $xtf_variant ]]; then - die "unsupported test variant '$1', supported variants: ${XTF_SRC_VARIANTS}" - fi - - xtf_build_binary ${xtf_variant} ${xtf_name} - xtf_build_cmdline ${xtf_variant} ${xtf_name} -} - -# Execute an XTF test. -function xtf_run_test() -{ - rm -f ${TEST_LOG} - export BOOT_MSG PASSED TEST_CMD TEST_LOG UBOOT_CMD - ./console.exp |& sed 's/\r\+$//' -} - -# Setup environment and run an XTF test. -# $1 Test variant. -# $2 Test name. -function xtf_test() -{ - # Out: FW_*, QEMU_*, XEN_{BINARY,CONSOLE}, XTF_SRC_* - xtf_arch_prepare - - # In: XTF_SRC_* - # OUt: XTF_{BINARY,NAME,VARIANT,WORKDIR} and XEN_CMDLINE - xtf_build_test $@ - - # In: FW_*, QEMU_*, XTF_*, XEN_* - # Out: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD - xtf_arch_setup - - # In: BOOT_MSG, PASSED, TEST_{CMD,LOG}, UBOOT_CMD - xtf_run_test -} diff --git a/automation/scripts/include/xtf-arm64 b/automation/scripts/include/xtf/qemu-arm64 similarity index 79% rename from automation/scripts/include/xtf-arm64 rename to automation/scripts/include/xtf/qemu-arm64 index 5d851af788a1..1f23e33fc0ac 100644 --- a/automation/scripts/include/xtf-arm64 +++ b/automation/scripts/include/xtf/qemu-arm64 @@ -3,20 +3,17 @@ # XTF test utilities (arm64). # -# Arch-specific environment overrides. -function xtf_arch_prepare() +# Target-specific environment overrides. +function xtf_init() { export FW_PREFIX="${FW_PREFIX:-/usr/lib/u-boot/qemu_arm64/}" export QEMU_PREFIX="${QEMU_PREFIX:-}" export XEN_BINARY="${XEN_BINARY:-${WORKDIR}/xen}" - export XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=dtuart}" - export XTF_SRC_BRANCH="${XTF_SRC_BRANCH:-xtf-arm}" - export XTF_SRC_URI="${XTF_SRC_URI:-https://gitlab.com/xen-project/fusa/xtf.git}"; - export XTF_SRC_VARIANTS="mmu64le" + export EXTRA_XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=dtuart}" } -# Perform arch-specific XTF environment setup. -function xtf_arch_setup() +# Perform target-specific XTF environment setup. +function xtf_setup_host() { # Crude check for local testing if [ ! -d imagebuilder ]; then diff --git a/automation/scripts/include/xtf-x86_64 b/automation/scripts/include/xtf/qemu-x86_64 similarity index 53% rename from automation/scripts/include/xtf-x86_64 rename to automation/scripts/include/xtf/qemu-x86_64 index 186f074bd8eb..0e079893dc25 100644 --- a/automation/scripts/include/xtf-x86_64 +++ b/automation/scripts/include/xtf/qemu-x86_64 @@ -3,20 +3,17 @@ # XTF test utilities (x86_64). # -# Arch-specific environment overrides. -function xtf_arch_prepare() +# Target-specific environment overrides. +function xtf_init() { export FW_PREFIX="${FW_PREFIX:-}" export QEMU_PREFIX="${QEMU_PREFIX:-}" export XEN_BINARY="${XEN_BINARY:-${WORKDIR}/xen}" - export XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}" - export XTF_SRC_BRANCH="${XTF_SRC_BRANCH:-master}" - export XTF_SRC_URI="${XTF_SRC_URI:-https://xenbits.xen.org/git-http/xtf.git}"; - export XTF_SRC_VARIANTS="hvm64 pv64" + export EXTRA_XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}" } -# Perform arch-specific XTF environment setup. -function xtf_arch_setup() +# Perform target-specific XTF environment setup. +function xtf_setup_host() { export TEST_CMD="${QEMU_PREFIX}qemu-system-x86_64 \ -no-reboot \ diff --git a/automation/scripts/include/xtf-x86_64-efi b/automation/scripts/include/xtf/qemu-x86_64-efi similarity index 75% rename from automation/scripts/include/xtf-x86_64-efi rename to automation/scripts/include/xtf/qemu-x86_64-efi index 15c6463dcdc5..dd2b8c3c6e52 100644 --- a/automation/scripts/include/xtf-x86_64-efi +++ b/automation/scripts/include/xtf/qemu-x86_64-efi @@ -3,20 +3,17 @@ # XTF test utilities (x86_64, EFI). # -# Arch-specific environment overrides. -function xtf_arch_prepare() +# Target-specific environment overrides. +function xtf_init() { export FW_PREFIX="${FW_PREFIX:-/usr/share/OVMF/}" export QEMU_PREFIX="${QEMU_PREFIX:-}" export XEN_BINARY="${XEN_BINARY:-${WORKDIR}/xen.efi}" - export XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}" - export XTF_SRC_BRANCH="${XTF_SRC_BRANCH:-master}" - export XTF_SRC_URI="${XTF_SRC_URI:-https://xenbits.xen.org/git-http/xtf.git}"; - export XTF_SRC_VARIANTS="hvm64 pv64" + export EXTRA_XEN_CMDLINE="${XEN_CMDLINE:-loglvl=all noreboot console_timestamps=boot console=com1}" } -# Perform arch-specific XTF environment setup. -function xtf_arch_setup() +# Perform target-specific XTF environment setup. +function xtf_setup_host() { local esp_dir="${WORKDIR}/boot-esp" local efi_dir="${esp_dir}/EFI/BOOT" diff --git a/automation/scripts/qemu-xtf.sh b/automation/scripts/qemu-xtf.sh deleted file mode 100755 index 55c221b36dc5..000000000000 --- a/automation/scripts/qemu-xtf.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# -# XTF test runner (QEMU). -# - -set -e -o pipefail - -if [ $# -lt 3 ]; then - echo "Usage: $(basename $0) ARCH XTF-VARIANT XTF-NAME" - exit 0 -fi - -export ARCH="$1" -shift - -set -x - -export XEN_ROOT="${PWD}" -cd $(dirname $0) - -source include/xtf-runner - -if [ ! -f "include/xtf-${ARCH}" ]; then - die "unsupported architecture '${ARCH}'" -fi -source include/xtf-${ARCH} - -xtf_test $@ diff --git a/automation/scripts/xtf b/automation/scripts/xtf new file mode 100755 index 000000000000..9f75ad5f99d0 --- /dev/null +++ b/automation/scripts/xtf @@ -0,0 +1,395 @@ +#!/bin/bash +# +# XTF test utilities. +# +# Environment variables: +# ARCH: Target CPU architecture +# BOOT_MSG: Expected boot message +# FW_PREFIX: Firmware images path including '/' at the end +# QEMU_PREFIX: QEMU path including '/' at the end +# TEST_LOG: Output log file +# UBOOT_CMD: U-Boot command line +# WORKDIR: Test working directory +# XEN_BINARY: Xen binary location +# XEN_CONSOLE: Xen console device name +# XTF_SRC_CONFIG: XTF config file +# XTF_SRC_BRANCH: XTF branch +# XTF_SRC_URI: XTF source code URI +# XTF_TARGET: Target name + +function die() +{ + set +x + echo "FATAL: $*" >&2 + exit 255 +} + +set -e -o pipefail +set -x + +if [[ -z "${ARCH}" ]]; then + die "'\$ARCH' is not defined" +fi + +if [[ -z "${XTF_TARGET}" ]]; then + die "'\$XTF_TARGET' is not defined" +fi + +TOP="$(readlink -f $(dirname $0))" +xtf_include="${TOP}/include/xtf/${XTF_TARGET}" + +if [[ ! -e "${xtf_include}" ]]; then + die "cannot find '${xtf_include}'" +fi + +# Scratch directory +XEN_ROOT="${PWD}" +# Output log file +TEST_LOG="${TEST_LOG:-${XEN_ROOT}/smoke.serial}" +# Expected boot message +BOOT_MSG="${BOOT_MSG:-Latest ChangeSet: }" +# Test working directory +WORKDIR="${WORKDIR:-${XEN_ROOT}/binaries}" +# XTF source code +XTF_SRC_CONFIG="${XTF_SRC_CONFIG:-include/configs/xtf-${ARCH}-config}" + +cd ${TOP} + +source include/xtf/${XTF_TARGET} + +function xtf_string_from_elf() +{ + str="$1" + xtf_binary="$2" + local addr=0 + local size=0 + local idx=0 + local sec_off="" + local offset=0 + + if ! command -v readelf &>/dev/null; then + die "readelf not found (install binutils)" + fi + + # Get symbol address, size, and section index. + read -r addr size idx < <( + readelf -Ws "${xtf_binary}" | + awk '$8 == "'${str}'" && $7 ~ /^[0-9]+$/ { + print $2, $3, $7 + exit + }' + ) + + if [[ -z "${idx}" || ${size} == 0 ]]; then + die "no ${str} in '${xtf_binary}'" + fi + + # Get section virtual address and file offset. + read -r sec_addr sec_off < <( + readelf -WS "${xtf_binary}" | + awk -v idx="${idx}" \ + '$0 ~ "^[[:space:]]*\\[[[:space:]]*" idx "\\]" { + sub(/^[[:space:]]*\[[[:space:]]*[0-9]+\][[:space:]]*/, "") + print $3, $4 + exit + }' + ) + + if [[ -z "${sec_off}" ]]; then + die "failed to extact ${str} from '${xtf_binary}'" + fi + + # xtf_binary symbol address -> file offset. + offset=$((16#${sec_off} + 16#${addr} - 16#${sec_addr})) + + # Extract the NUL-terminated string. + dd if="${xtf_binary}" bs=1 skip="${offset}" count="${size}" status=none | + tr -d '\000' +} + +function xtf_get_bootargs() +{ + local xtf_info=$1 + local line + local re='"xen,xen-bootargs"[[:space:]]*:[[:space:]]*"([^"]*)"' + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" =~ $re ]]; then + printf '%s\n' "${BASH_REMATCH[1]}" + return 0 + fi + done < "$xtf_info" +} + +# Build Xen command line for running an XTF test. +# $1 Test variant. +# $2 Test manifest. +function xtf_setup_cmdline() +{ + local xtf_variant=$1 + local xtf_info=$2 + local -a cmdline=("${EXTRA_XEN_CMDLINE}") + + if [[ "$xtf_variant" == hvm* ]]; then + cmdline+=("iommu=0 dom0-iommu=none dom0=pvh") + fi + + if [[ -f "$xtf_info" ]]; then + cat "${xtf_info}" + cmdline+=($(xtf_get_bootargs "${xtf_info}")) + fi + + export XEN_CMDLINE="${cmdline[*]}" +} + +# Execute an XTF test. +function xtf_run() +{ + rm -f ${TEST_LOG} + export BOOT_MSG TEST_CMD TEST_LOG UBOOT_CMD + + printf "XTF binary: %s\n" "${XTF_BINARY}" + printf "XTF log: %s\n" "${XTF_LOG}" + printf "Xen binary: %s\n" "${XEN_BINARY}" + printf "Xen Command Line: %s\n" "${XEN_CMDLINE}" + + TEST_XTF=y ./console.exp |& sed 's/\r\+$//' +} + +function print_title() +{ + local str="$1" + local width=$(( 80 - ${#str} - 2 )) + local left=$(( width / 2 )) + local right=$(( width - left )) + + printf -v lhs '%*s' "$left" '' + printf -v rhs '%*s' "$right" '' + printf '\n%s %s %s\n' "${lhs// /=}" "$str" "${rhs// /=}" +} + +function main() +{ + declare -A nr_tests_by_status=( + [PASS]=0 + [TIMEOUT]=0 + [EOF]=0 + [SKIP]=0 + [ERROR]=0 + [FAIL]=0 + [CRASH]=0 + [UNKNOWN]=0 + ) + local -A xtf_completed=() + local -A xtf_binaries_selected=() + local -A xtf_binary_by_path=() + local -a xtf_binaries=() + local -a xtf_binaries_filtered=() + local nr_tests=0 + local xtf_binary xtf_file xtf_variant xtf_name xtf_title + + xtf_init + + # + # Pass 1: Discover all XTF ELF binaries. + # + + while IFS= read -r -d '' xtf_binary; do + if ! readelf -h "$xtf_binary" &>/dev/null; then + continue + fi + + # Some XTF build quirks + case "$ARCH" in + arm64) + if [[ "$xtf_binary" != *-syms ]]; then + continue + fi + + xtf_binary="${xtf_binary%-syms}" + + if [[ ! -s "$xtf_binary" ]]; then + die "Missing ARM64 boot image '$xtf_binary'" + fi + ;; + *) + ;; + esac + + xtf_binaries+=(${xtf_binary}) + done < <(find "$WORKDIR" -mindepth 2 -type f -name 'test-*' -print0) + + if (( ${#xtf_binaries[@]} == 0 )); then + die "No XTF ELF binaries found in '$WORKDIR'" + fi + + # + # Load test suite manifest, if any. + # + + if [[ -n ${XTF_MANIFEST:-} ]]; then + local -a lines=() + + if [[ ! -r "$XTF_MANIFEST" ]]; then + die "Cannot read '$XTF_MANIFEST'" + fi + + mapfile -t lines < "$XTF_MANIFEST" + for x in "${lines[@]}"; do + xtf_binaries_selected["${x}"]=1 + done + + if (( ${#xtf_binaries_selected[@]} == 0 )); then + die "No tests found in '$XTF_MANIFEST'" + fi + fi + + # + # Pass 2: Filter discovered XTF binaries. + # + + for xtf_binary in "${xtf_binaries[@]}"; do + xtf_file=$(basename "${xtf_binary}") + + # Filter 1: YAML + if [[ -n ${XTF_MANIFEST:-} && -z ${xtf_binaries_selected[$xtf_file]:-} ]]; then + continue + fi + + # Filter 2: regex + if [[ -n ${SELECTED_XTF_ONLY:-} && ! $xtf_file =~ $SELECTED_XTF_ONLY ]]; then + continue + fi + + xtf_binaries_filtered+=("$xtf_binary") + done + + for xtf_binary in "${xtf_binaries_filtered[@]}"; do + local xtf_dir=$(dirname "$xtf_binary") + xtf_binary_by_path["$xtf_binary"]="${xtf_dir}/info.json" + done + nr_tests=${#xtf_binary_by_path[@]} + + if (( ${nr_tests} == 0 )); then + die "No XTF tests matched the selection" + fi + + # + # Pass 3: Execute selected tests. + # + + print_title "Selected tests (${nr_tests})" + + local nr=1 + for xtf_binary in "${!xtf_binary_by_path[@]}"; do + local ret status_str + local xtf_pid progress_pid + local t_start t_end + local xtf_info="${xtf_binary_by_path[${xtf_binary}]}" + + xtf_file="${xtf_binary##*/}" + + xtf_variant="${xtf_file#test-}" + xtf_name="${xtf_variant#*-}" + xtf_variant="${xtf_variant%%-*}" + + # Some XTF build quirks + case "$ARCH" in + arm64) + xtf_title=$(xtf_string_from_elf "test_title" "${xtf_binary}-syms") + ;; + *) + xtf_title=$(xtf_string_from_elf "test_title" "${xtf_binary}") + ;; + esac + + export XTF_BINARY="${xtf_binary}" + export XTF_LOG="${TOP}/${xtf_file}.log" + + printf "%3d/%3d %s (%s): " "$nr" "$nr_tests" "${xtf_title}" "${xtf_file}" + + # Spawn in a subshell so that bash traps could be used + # (e.g. for power cycling the target) + t_start=$SECONDS + ( + set -x + + xtf_setup_cmdline "${xtf_variant}" "${xtf_info}" + + # In: FW_*, QEMU_*, XTF_*, XEN_* + # Out: BOOT_MSG, TEST_{CMD,LOG}, UBOOT_CMD + xtf_setup_host + + # In: BOOT_MSG, TEST_{CMD,LOG}, UBOOT_CMD + xtf_run + ) &> "${XTF_LOG}" & + xtf_pid=$! + + # Print a dot every second while the test is running. + ( + while kill -0 "${xtf_pid}" 2>/dev/null; do + sleep 1 + + if ! kill -0 "$xtf_pid" 2>/dev/null; then + break + fi + + printf '.' + done + ) & + progress_pid=$! + + if wait "${xtf_pid}"; then + ret=0 + else + ret=$? + fi + t_end=$SECONDS + + # Stop the progress indicator. + kill "$progress_pid" 2>/dev/null || true + wait "$progress_pid" 2>/dev/null || true + + # See console.exp and xtf-runner + case "$ret" in + 0) status_str="PASS" ;; + 1) status_str="TIMEOUT" ;; + 2) status_str="EOF" ;; + 3) status_str="SKIP" ;; + 4) status_str="ERROR" ;; + 5) status_str="FAIL" ;; + 6) status_str="CRASH" ;; + *) status_str="UNKNOWN" ;; + esac + + (( nr_tests_by_status[${status_str}] += 1 )) + + xtf_completed["${xtf_title}"]="${XTF_LOG}" + + printf '%s (%ss)\n' "${status_str}" $(( t_end - t_start )) + (( nr += 1 )) + done + + for xtf_title in "${!xtf_completed[@]}"; do + print_title "${xtf_title}" + cat -v "${xtf_completed[$xtf_title]}" || true + done + + print_title "Report" + printf " Total : %d\n" ${nr_tests} + for x in "${!nr_tests_by_status[@]}"; do + printf " %s: %d\n" "$x " "${nr_tests_by_status[$x]}" + done + + nr_success=$(( nr_tests_by_status[PASS] + nr_tests_by_status[SKIP] )) + if (( nr_tests == 0 )); then + exit 255 + elif (( nr_success == nr_tests )); then + exit 0 + else + exit 1 + fi +} + +set +x +main $@ -- 2.55.0
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |