Tests

feature: slim docker images #4593

Workflow file for this run

	name: Tests

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	types: [opened, synchronize, reopened, ready_for_review]

	defaults:
	run:
	shell: bash

	# Cancel previous running actions for the same PR
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

	# Top-level env. vars shared by most jobs
	env:
	COMPOSE_VERSION: 2.29.2
	# This path must match the mount-path where the conan-cache-action restores
	# all the pre-built conan binaries and libraries. In addition, this env.
	# variable is picked-up by faasmctl to mount this pre-built dependencies
	# into the Faasm cluster.
	FAASMCTL_CONAN_CACHE_MOUNT_SOURCE: ./dev/faasm/conan2
	FAABRIC_VERSION: 0.22.0
	FAASM_INI_FILE: ./faasm.ini
	FAASM_VERSION: 0.33.0
	FAASMCTL_VERSION: 0.51.0

	jobs:
	checks:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	container:
	image: ghcr.io/faasm/cli:0.33.0
	steps:
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: true
	fetch-depth: 0
	# We need to set the safe git directory as formatting relies on git-ls
	# See actions/checkout#766
	- name: "Set the GH workspace as a safe git directory"
	run: \|
	git config --global --add safe.directory "$GITHUB_WORKSPACE"
	git config --global --add safe.directory "$GITHUB_WORKSPACE/faabric"
	git config --global --add safe.directory "$GITHUB_WORKSPACE/cpp"
	git config --global --add safe.directory "$GITHUB_WORKSPACE/python"
	# Formatting checks
	- name: "Code formatting check"
	run: ./bin/inv_wrapper.sh format-code --check
	# Sanity checks
	- name: "Check that all the submodules point to their 'main' branch"
	run: ./bin/inv_wrapper.sh git.check-submodule-branch

	docs:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: true
	- name: "Build docs"
	run: \|
	sudo apt install -y doxygen
	./bin/inv_wrapper.sh docs

	# Work out if we need to re-cross-compile the WASM functions used in the
	# tests. We need to run `cpp-funcs` if there's a change in the `clients/cpp`
	# submodule, and `py-funcs` if there's a change in the `clients/python`
	# submodule. Additionally, even if there are no changes, we also want to run
	# `clients/cpp` and `clients/python` if the WASM cache is not in place
	wasm-funcs-cache:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	outputs:
	needs-cpp-wasm: ${{ (steps.filter.outputs.cpp-changed == 'true') \|\| (steps.wasm-cpp-cache.outputs.cache-hit != 'true') }}
	needs-py-wasm: ${{ (steps.filter.outputs.py-changed == 'true') \|\| (steps.wasm-py-cache.outputs.cache-hit != 'true') }}
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: true
	# Check if any of the submodules have been modified
	- uses: dorny/paths-filter@v3
	id: filter
	with:
	filters: \|
	cpp-changed:
	- './clients/cpp/**'
	py-changed:
	- './clients/python/**'
	# Even if they have not been modified, check whether the WASM cache is
	# available
	- name: "Get CPP/Python commits"
	id: submodule-commit
	run: \|
	echo "cpp-commit=$(git submodule status ./clients/cpp \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	echo "py-commit=$(git submodule status ./clients/python \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	- uses: actions/cache/restore@v4
	id: wasm-cpp-cache
	with:
	path: /usr/local/faasm/wasm
	key: wasm-cpp-${{ steps.submodule-commit.outputs.cpp-commit }}
	lookup-only: true
	- uses: actions/cache/restore@v4
	id: wasm-py-cache
	with:
	path: /usr/local/faasm/wasm/python
	key: wasm-py-${{ steps.submodule-commit.outputs.py-commit }}
	lookup-only: true

	cpp-funcs:
	needs: wasm-funcs-cache
	if: ${{ needs.wasm-funcs-cache.outputs.needs-cpp-wasm == 'true' }}
	runs-on: ubuntu-latest
	container:
	image: ghcr.io/faasm/cpp-sysroot:0.7.0
	steps:
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: recursive
	- name: "Build C++ functions"
	run: ./bin/inv_wrapper.sh func.local --clean
	working-directory: ${{ github.workspace }}/clients/cpp
	- name: "Build libfake"
	run: ./bin/inv_wrapper.sh libfake
	working-directory: ${{ github.workspace }}/clients/cpp
	- name: "Prepare Cpp WASM cache"
	id: submodule-commit
	run: \|
	# Github Caches are versioned with a combination of the key, the
	# cache path, and the compression algorithm. We install `zstd` to
	# get the same version hash than in the step that runs outside of
	# the containers
	apt install -y zstd
	# There's a misalignment between the github.workspace value reported
	# in the `run` command, and the one in the `working-directory` one,
	# so we use the environment variable here instead. See:
	# https://github.com/actions/runner/issues/2058#issuecomment-1541828550
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	echo "cpp-commit=$(git submodule status ./clients/cpp \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	# We move the libfake shared library to the WASM dir to use only one
	# cache
	mv /usr/local/faasm/runtime_root/lib/fake /usr/local/faasm/wasm/
	working-directory: ${{ github.workspace }}
	- name: "Cpp WASM cache"
	uses: actions/cache@v4
	id: cpp-wasm-cache
	with:
	path: /usr/local/faasm/wasm
	key: wasm-cpp-${{ steps.submodule-commit.outputs.cpp-commit }}

	py-funcs:
	needs: wasm-funcs-cache
	if: ${{ needs.wasm-funcs-cache.outputs.needs-py-wasm == 'true' }}
	runs-on: ubuntu-latest
	container:
	image: ghcr.io/faasm/cpython:0.4.0
	steps:
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: recursive
	- name: "Build CPython function"
	run: ./bin/inv_wrapper.sh cpython.func
	working-directory: ${{ github.workspace }}/clients/python
	- name: "Put Python functions in place"
	run: ./bin/inv_wrapper.sh func.upload-all --local
	working-directory: ${{ github.workspace }}/clients/python
	- name: "Prepare Python WASM cache"
	id: submodule-commit
	run: \|
	apt install -y zstd
	git config --global --add safe.directory ${GITHUB_WORKSPACE}
	echo "py-commit=$(git submodule status ./clients/python \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	# We temporarily move the pyfuncs to only use one cache
	mv /usr/local/faasm/shared/pyfuncs /usr/local/faasm/wasm/python/
	working-directory: ${{ github.workspace }}
	- name: "Python WASM cache"
	uses: actions/cache@v4
	id: py-wasm-cache
	with:
	path: /usr/local/faasm/wasm/python
	key: wasm-py-${{ steps.submodule-commit.outputs.py-commit }}

	# Run once the cache, to guarantee that downstream jobs see a cache hit
	image-cache:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-latest
	steps:
	- uses: faasm/image-cache-action@main
	with:
	faasm-version: ${{ env.FAASM_VERSION }}
	faabric-version: ${{ env.FAABRIC_VERSION }}

	conan-cache:
	if: github.event.pull_request.draft == false
	needs: [image-cache]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	build-type: [Debug, Release]
	steps:
	- name: "Check-out code"
	uses: actions/checkout@v4
	with:
	submodules: true

	# First, check if the conan cache is already there, if so we can skip
	# the rest of this job
	- name: "Conan cache"
	id: conan-cache-lookup
	uses: faasm/conan-cache-action@main
	with:
	build-type: ${{ matrix.build-type }}
	from: faasm
	mount-path: ${{ env.FAASMCTL_CONAN_CACHE_MOUNT_SOURCE }}

	# The following steps populate the conan cache
	- name: "Image cache"
	if: ${{ steps.conan-cache-lookup.outputs.cache-hit-faasm != 'true' }}
	uses: faasm/image-cache-action@main
	with:
	faasm-version: ${{ env.FAASM_VERSION }}
	faabric-version: ${{ env.FAABRIC_VERSION }}
	read-only: 'true'
	# Check-out the code again, as the image-cache action wipes everything
	- name: "Check-out code"
	if: ${{ steps.conan-cache-lookup.outputs.cache-hit-faasm != 'true' }}
	uses: actions/checkout@v4
	with:
	clean: true
	submodules: true
	- name: "Install faasmctl"
	if: ${{ steps.conan-cache-lookup.outputs.cache-hit-faasm != 'true' }}
	run: pip3 install faasmctl==${{ env.FAASMCTL_VERSION }}
	- name: "Start a worker-less Faasm cluster to run unit tests"
	if: ${{ steps.conan-cache-lookup.outputs.cache-hit-faasm != 'true' }}
	run: faasmctl deploy.compose --mount-source . --workers=0
	- name: "Build Conan dependencies to be shared by all runs"
	if: ${{ steps.conan-cache-lookup.outputs.cache-hit-faasm != 'true' }}
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.conan --build ${{ matrix.build-type }}"

	tests:
	needs: [cpp-funcs, py-funcs, conan-cache, wasm-funcs-cache]
	# Run the tests both if the dependencies have succeeded or have been
	# skipped
	if: \|
	always() &&
	!cancelled() &&
	github.event.pull_request.draft == false &&
	needs.conan-cache.result == 'success' &&
	(needs.cpp-funcs.result == 'success' \|\| needs.cpp-funcs.result == 'skipped') &&
	(needs.py-funcs.result == 'success' \|\| needs.py-funcs.result == 'skipped')
	env:
	FAASM_DEPLOYMENT_TYPE: gha-ci
	runs-on: ubuntu-latest
	# This timeout should be long enough to let slow tests (w/ sanitisers)
	# build and run, but still kill the GHA if something goes wrong
	timeout-minutes: 70
	strategy:
	fail-fast: false
	matrix:
	# Even though it is not truly a sanitiser, we include Coverage and
	# SGX in the mix to benefit from GHAs parallelism
	sanitiser: [None, Undefined, Coverage, Sgx]
	steps:
	# ----- Prepare docker images cache -----
	- name: "Image cache (must run as first step!)"
	uses: faasm/image-cache-action@main
	with:
	faasm-version: ${{ env.FAASM_VERSION }}
	faabric-version: ${{ env.FAABRIC_VERSION }}
	read-only: 'true'
	- uses: csegarragonz/set-compose-version-action@main
	with:
	compose-version: ${{ env.COMPOSE_VERSION }}
	- name: "Check out code"
	uses: actions/checkout@v4
	with:
	submodules: true
	- name: "Install faasmctl"
	run: pip3 install faasmctl==${{ env.FAASMCTL_VERSION }}

	# ----- Prepare C++ deps cache (via Conan) -----
	- name: "Conan cache"
	uses: faasm/conan-cache-action@main
	with:
	build-type: Debug
	from: faasm
	mount-path: ${{ env.FAASMCTL_CONAN_CACHE_MOUNT_SOURCE }}

	# ----- Prepare WASM cache -----
	# Download wasm generated by previous steps
	- name: "Get CPP/Python commits"
	id: submodule-commit
	run: \|
	# Temporary directory to restore the cache into
	# sudo mkdir -p /usr/local/faasm/
	git config --global --add safe.directory "$GITHUB_WORKSPACE"
	echo "cpp-commit=$(git submodule status ./clients/cpp \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT
	echo "py-commit=$(git submodule status ./clients/python \| cut '-d ' -f 2)" >> $GITHUB_OUTPUT

	# ----- Deploy Faasm cluster -----
	- name: "Start a worker-less Faasm cluster to run unit tests"
	run: faasmctl deploy.compose --mount-source . --workers=0
	if: ${{ matrix.sanitiser != 'Sgx' }}
	- name: "Start a worker-less (SGX) Faasm cluster to run unit tests"
	run: \|
	# First remove unnecessary images
	docker rmi -f ghcr.io/faasm/cli:${{ env.FAASM_VERSION }}
	docker rmi -f ghcr.io/faasm/upload:${{ env.FAASM_VERSION }}
	docker rmi -f ghcr.io/faasm/worker:${{ env.FAASM_VERSION }}
	faasmctl deploy.compose --mount-source . --workers=0
	env:
	FAASM_WASM_VM: sgx-sim
	if: ${{ matrix.sanitiser == 'Sgx' }}

	- name: "Get Cpp WASM cache"
	uses: actions/cache/restore@v4
	id: cpp-wasm-cache
	with:
	path: /usr/local/faasm/wasm
	key: wasm-cpp-${{ steps.submodule-commit.outputs.cpp-commit }}
	- name: "Get Python WASM cache"
	uses: actions/cache/restore@v4
	id: py-wasm-cache
	with:
	path: /usr/local/faasm/wasm/python
	key: wasm-py-${{ steps.submodule-commit.outputs.py-commit }}
	# Move libfake and pyfuncs that are misplaced to only use one cache
	- name: "Post-WASM cache"
	run: \|
	# For some reason, non-docker runners resolve /usr into ../../../usr
	sudo chown -R $(id -u):$(id -g) ./dev/faasm-local
	mv ../../..//usr/local/faasm/wasm ./dev/faasm-local/
	mv ./dev/faasm-local/wasm/fake ./dev/faasm-local/runtime_root/lib/fake
	mkdir -p ./dev/faasm-local/shared
	mv ./dev/faasm-local/wasm/python/pyfuncs ./dev/faasm-local/shared/pyfuncs

	# ----- Prepare machine code cache -----
	- name: "Get and print CPU info"
	run: \|
	cat /proc/cpuinfo
	echo "CPU_MODEL=$(./bin/print_cpu.sh)" >> $GITHUB_ENV
	echo "${{ env.CPU_MODEL}}"
	# We want to invalide the codegen cache if we change the WAVM or the
	# WAMR dependency, as changes in the library may change the codegen step
	- name: "Get WAVM and WAMR git tags"
	run: \|
	echo "WAMR_VERSION=$(cat cmake/ExternalProjects.cmake \| grep -C 2 'wamr' \| grep 'GIT_TAG' \| cut -d' ' -f6)" >> $GITHUB_ENV
	echo "WAVM_VERSION=$(cat cmake/ExternalProjects.cmake \| grep -C 2 'wavm' \| grep 'GIT_TAG' \| cut -d' ' -f6)" >> $GITHUB_ENV
	- name: "Configure machine code cache"
	uses: actions/cache@v4
	id: machine-code-cache
	with:
	path: ./dev/faasm-local/object
	key: ${{ env.CPU_MODEL }}-machine-code-${{ secrets.CACHE_VERSION }}-${{ env.WAVM_VERSION }}-${{ env.WAMR_VERSION }}

	# ----- Build the dependencies and run CMake -----
	# We only need to run dev.conan with the --clean flag when building for
	# the different sanitisers, as sanitised builds like TSAN or ASAN
	# require all dependencies to also be (re-)built with the right
	# sanitiser flags.
	# TODO: also cache the conan-deps with sanitisers
	- name: "Run CMake with sanitiser set"
	if: ${{ (matrix.sanitiser != 'Coverage') && (matrix.sanitiser != 'Sgx') }}
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.conan --clean dev.cmake --clean --disable-spinlock --build Debug --sanitiser ${{ matrix.sanitiser }}"
	- name: "Run CMake with coverage"
	if: ${{ matrix.sanitiser == 'Coverage' }}
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.conan dev.cmake --clean --disable-spinlock --build Debug --coverage"
	- name: "Run CMake with SGX"
	if: ${{ matrix.sanitiser == 'Sgx' }}
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.conan dev.cmake --clean --disable-spinlock --build Debug --sgx Simulation"

	# We only re-build the codegen binaries if `clients/cpp` or
	# `clients/python` have changed, or (ii) `cpp` and `python` have not
	# changed, but we have a cache miss (i.e. new CPU, or new WAMR/WAVM)
	- name: "Run codegen for the tests"
	if: ${{ (needs.wasm-funcs-cache.outputs.needs-cpp-wasm == 'true') \|\| (needs.wasm-funcs-cache.outputs.needs-py-wasm == 'true') \|\| (steps.machine-code-cache.outputs.cache-hit != 'true') }}
	run: \|
	faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.cc codegen_func dev.cc codegen_shared_obj"
	faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh codegen.tests"
	env:
	# Disable sanitiser warnings when running code generation.
	TSAN_OPTIONS: "report_bugs=0 exitcode=0"
	UBSAN_OPTIONS: "halt_on_error=0:print_stacktrace=0:exitcode=0:log_path=/dev/null"
	- name: "Clear existing pyc files"
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh python.clear-runtime-pyc"

	# ----- Build the tests -----
	- name: "Build only the tests target"
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.cc tests"
	# Until we bump to LLVM >= 18.1.0, TSAN comes with bug with ASLR
	# https://stackoverflow.com/questions/77850769/fatal-threadsanitizer-unexpected-memory-mapping-when-running-on-linux-kernels
	- name: "Workaround TSAN ASLR bug with LLVM 17"
	run: sudo sysctl vm.mmap_rnd_bits=28
	if: ${{ matrix.sanitiser == 'Thread' }}
	# Run tests
	- name: "Run the tests"
	if: ${{ matrix.sanitiser != 'Coverage' }}
	run: faasmctl cli.faasm --env HOST_TYPE=ci --cmd "./bin/inv_wrapper.sh tests"
	- name: "Run the tests and fetch coverage report"
	if: ${{ matrix.sanitiser == 'Coverage' }}
	run: faasmctl cli.faasm --env HOST_TYPE=ci,LLVM_PROFILE_FILE=/tmp/faasm.profraw --cmd "./bin/inv_wrapper.sh tests"
	- name: "Generate code coverage report"
	if: ${{ matrix.sanitiser == 'Coverage' }}
	run: faasmctl cli.faasm --cmd "./bin/inv_wrapper.sh dev.coverage-report --file-in /tmp/faasm.profraw --file-out coverage.txt" --cp-out /tmp/faasm.profraw:coverage.txt
	- name: "Upload coverage report to CodeCov"
	if: ${{ matrix.sanitiser == 'Coverage' }}
	uses: codecov/codecov-action@v4
	with:
	# Note that this secret is specific to this repository
	token: ${{ secrets.CODECOV_TOKEN }}

	dist-tests:
	if: github.event.pull_request.draft == false
	needs: conan-cache
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	wasm_vm: [wamr, wavm]
	env:
	FAASM_DEPLOYMENT_TYPE: gha-ci
	FAASM_WASM_VM: ${{ matrix.wasm_vm }}
	steps:
	- name: "Image cache (must run as first step!)"
	uses: faasm/image-cache-action@main
	with:
	faasm-version: ${{ env.FAASM_VERSION }}
	faabric-version: ${{ env.FAABRIC_VERSION }}
	read-only: 'true'
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: true
	- name: "Conan cache"
	uses: faasm/conan-cache-action@main
	with:
	build-type: Debug
	from: faasm
	mount-path: ${{ env.FAASMCTL_CONAN_CACHE_MOUNT_SOURCE }}
	- name: "Install faasmctl"
	run: pip3 install faasmctl==${{ env.FAASMCTL_VERSION }}
	# Cache contains architecture-specific machine code
	- name: "Get CPU model name"
	run: echo "CPU_MODEL=$(./bin/print_cpu.sh)" >> $GITHUB_ENV
	- name: "Print CPU model"
	run: echo "${{ env.CPU_MODEL}}"
	- name: "Configure S3 cache"
	uses: actions/cache@v4
	with:
	path: ./dev/minio/data/faasm
	key: ${{ env.CPU_MODEL }}-s3-data-${{ secrets.CACHE_VERSION }}
	- name: "Start a Faasm cluster to run the distributed tests"
	run: faasmctl deploy.dist-tests --mount-source .
	- name: "Build dist-tests and other targets"
	run: faasmctl cli.faasm --cmd "./deploy/dist-test/build_internal.sh"
	- name: "Re-start the services to pick up new binaries (if necessary)"
	run: faasmctl restart -s upload -s dist-test-server
	# This can fail when the container isn't ready, so we want to retry
	- name: "Wait for upload server to be available"
	run: \|
	(echo "Attempt 1" && faasmctl cli.faasm --cmd "./deploy/local/wait_for_upload.sh upload 8002") \|\| \
	(echo "Attempt 2" && faasmctl cli.faasm --cmd "./deploy/local/wait_for_upload.sh upload 8002") \|\| \
	(echo "Attempt 3" && faasmctl cli.faasm --cmd "./deploy/local/wait_for_upload.sh upload 8002") \|\| \
	(echo "Wait for upload failed after retries" && faasmctl logs -s upload && exit 1)
	- name: "Build and upload functions for tests"
	run: ./deploy/dist-test/upload.sh
	- name: "Run tests"
	run: ./deploy/dist-test/run.sh
	- name: "Unconditional deletion of the cluster"
	if: always()
	run: faasmctl delete

	e2e-tests:
	if: github.event.pull_request.draft == false
	needs: [image-cache, conan-cache]
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	faasm_wasm_vm: [wamr, wavm]
	mount_source: [on, off]
	env:
	FAASM_DEPLOYMENT_TYPE: gha-ci
	FAASM_MOUNT_SOURCE: ${{ matrix.mount_source }}
	FAASM_WASM_VM: ${{ matrix.faasm_wasm_vm }}
	steps:
	- uses: csegarragonz/set-compose-version-action@main
	with:
	compose-version: ${{ env.COMPOSE_VERSION }}
	- name: "Image cache (must run as first step!)"
	uses: faasm/image-cache-action@main
	with:
	faasm-version: ${{ env.FAASM_VERSION }}
	faabric-version: ${{ env.FAABRIC_VERSION }}
	read-only: 'true'
	- name: "Checkout code"
	uses: actions/checkout@v4
	with:
	submodules: recursive

	# ----- Prepare C++ deps cache (via Conan) -----
	- name: "Conan cache"
	uses: faasm/conan-cache-action@main
	with:
	build-type: "release"
	from: faasm
	mount-path: ${{ env.FAASMCTL_CONAN_CACHE_MOUNT_SOURCE }}

	- name: "Run e2e tests"
	run: ./tests/e2e/run.sh
	env:
	FAASM_SOURCE: ${{ github.workspace }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

feature: slim docker images #4593

Workflow file

feature: slim docker images #4593

Uh oh!

Workflow file for this run