[POC] Distributed scheduler #4064

Workflow file for this run

.github/workflows/testing-pipeline.yml at 3ac24dc

	name: OpenCue Testing Pipeline

	on:
	push:
	branches: [master]
	pull_request:
	branches: [master]

	jobs:
	check_opencue_licenses:
	name: Check Licenses of OpenCue Dependencies
	runs-on: ubuntu-22.04
	container: python:3.9
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Run license check
	shell: bash
	run: ci/check_licenses.sh

	build_opencue_packages:
	name: Build Python Packages
	runs-on: ubuntu-22.04
	container: python:3.7
	outputs:
	opencue_proto_path: ${{ steps.package_outputs.outputs.opencue_proto_path }}
	opencue_pycue_path: ${{ steps.package_outputs.outputs.opencue_pycue_path }}
	opencue_pyoutline_path: ${{ steps.package_outputs.outputs.opencue_pyoutline_path }}
	opencue_cueadmin_path: ${{ steps.package_outputs.outputs.opencue_cueadmin_path }}
	opencue_cueman_path: ${{ steps.package_outputs.outputs.opencue_cueman_path }}
	opencue_cuecmd_path: ${{ steps.package_outputs.outputs.opencue_cuecmd_path }}
	opencue_cuesubmit_path: ${{ steps.package_outputs.outputs.opencue_cuesubmit_path }}
	opencue_rqd_path: ${{ steps.package_outputs.outputs.opencue_rqd_path }}
	opencue_cuenimby_path: ${{ steps.package_outputs.outputs.opencue_cuenimby_path }}
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-tags: true
	fetch-depth: 0

	- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Set build ID
	run: \|
	set -e
	echo "Build ID: $(ci/generate_version_number.py)"
	echo "BUILD_ID=$(ci/generate_version_number.py)" >> ${GITHUB_ENV}

	- uses: ./.github/actions/build-python-packages

	- name: Gather package paths
	id: package_outputs
	run: \|
	find . -name *.whl
	echo "opencue_proto_path=$(find ./packages -name 'opencue_proto-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_rqd_path=$(find ./packages -name 'opencue_rqd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_pycue_path=$(find ./packages -name 'opencue_pycue-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_pyoutline_path=$(find ./packages -name 'opencue_pyoutline-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_cuesubmit_path=$(find ./packages -name 'opencue_cuesubmit-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_cueadmin_path=$(find ./packages -name 'opencue_cueadmin-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_cueman_path=$(find ./packages -name 'opencue_cueman-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_cuecmd_path=$(find ./packages -name 'opencue_cuecmd-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_cuegui_path=$(find ./packages -name 'opencue_cuegui-*.whl' -print -quit)" >> $GITHUB_OUTPUT
	echo "opencue_cuenimby_path=$(find ./packages -name 'opencue_cuenimby-*.whl' -print -quit)" >> $GITHUB_OUTPUT

	install_opencue_packages:
	needs: build_opencue_packages
	name: Test installing packages with python ${{ matrix.python-version }}
	runs-on: ubuntu-22.04
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.7", "3.9", "3.10", "3.11"]
	container: python:${{ matrix.python-version }}
	steps:
	- name: Download artifact
	uses: actions/download-artifact@v4
	with:
	name: opencue_packages
	path: packages

	- name: Install package
	run: \|
	set -e
	pip install \
	${{ needs.build_opencue_packages.outputs.opencue_proto_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_pycue_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_cueman_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_cuecmd_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_rqd_path }} \
	${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}
	python -c "import opencue_proto"
	python -c "import opencue_proto.report_pb2"
	python -c "import opencue_proto.report_pb2_grpc"

	test_python_2023:
	needs: build_opencue_packages
	name: Run Python Unit Tests (CY2023)
	runs-on: ubuntu-22.04
	container: aswf/ci-opencue:2023
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Download a single artifact
	uses: actions/download-artifact@v4
	with:
	name: opencue_packages
	path: packages

	- name: Run Python Tests
	run: \|
	export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
	export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
	export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
	export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
	export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
	export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
	export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
	export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
	ci/run_python_tests.sh

	test_cuebot_2023:
	needs: build_opencue_packages
	name: Build Cuebot and Run Unit Tests (CY2023)
	runs-on: ubuntu-22.04
	container:
	image: aswf/ci-opencue:2023
	steps:
	- uses: actions/checkout@v4
	- name: Build with Gradle
	run: \|
	chown -R aswfuser:aswfgroup .
	su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser

	test_python_2024:
	needs: build_opencue_packages
	name: Run Python Unit Tests (CY2024)
	runs-on: ubuntu-22.04
	container: aswf/ci-opencue:2024
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Download a single artifact
	uses: actions/download-artifact@v4
	with:
	name: opencue_packages
	path: packages

	- name: Run Python Tests
	run: \|
	export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
	export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
	export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
	export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
	export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
	export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
	export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
	export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
	export OPENCUE_CUENIMBY_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}"
	ci/run_python_tests.sh

	test_cuebot_2024:
	needs: build_opencue_packages
	name: Build Cuebot and Run Unit Tests (CY2024)
	runs-on: ubuntu-22.04
	container:
	image: aswf/ci-opencue:2024
	steps:
	- uses: actions/checkout@v4
	- name: Build with Gradle
	run: \|
	chown -R aswfuser:aswfgroup .
	su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser

	integration_test:
	needs: build_opencue_packages
	name: Run Integration Test
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Download a single artifact
	uses: actions/download-artifact@v4
	with:
	name: opencue_packages
	path: packages

	- name: Run test
	run: \|
	export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
	export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
	export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
	export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
	export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
	export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
	export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
	export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
	export OPENCUE_CUENIMBY_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}"
	ci/run_integration_test.sh

	- name: Archive log files
	uses: actions/upload-artifact@v4
	if: ${{ always() }}
	with:
	name: test-logs
	path: /tmp/opencue-test/*.log

	lint_python:
	needs: build_opencue_packages
	name: Lint Python Code
	runs-on: ubuntu-22.04
	container: aswf/ci-opencue:2024.1
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Download a single artifact
	uses: actions/download-artifact@v4
	with:
	name: opencue_packages
	path: packages

	- name: Set package vars from parent action
	run: \|
	export OPENCUE_PROTO_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_proto_path }}"
	export OPENCUE_PYCUE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pycue_path }}"
	export OPENCUE_PYOUTLINE_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_pyoutline_path }}"
	export OPENCUE_CUEADMIN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueadmin_path }}"
	export OPENCUE_CUEMAN_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cueman_path }}"
	export OPENCUE_CUESUBMIT_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuesubmit_path }}"
	export OPENCUE_CUEGUI_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuegui_path }}"
	export OPENCUE_RQD_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_rqd_path }}"
	export OPENCUE_CUENIMBY_PACKAGE_PATH="${{ needs.build_opencue_packages.outputs.opencue_cuenimby_path }}"

	- name: Lint Python Code
	run: ci/run_python_lint.sh

	test_sphinx:
	needs: build_opencue_packages
	name: Test Documentation Build
	runs-on: ubuntu-22.04
	container:
	image: aswf/ci-opencue:2023
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Mark repository as safe (Fix for https://github.com/actions/checkout/issues/1048)
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Run Sphinx build
	run: ci/build_sphinx_docs.sh

	check_changed_files:
	name: Check Changed Files
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 2
	set-safe-directory: true
	fetch-tags: true

	- name: Check for Version Change
	run: \|
	git diff --name-status HEAD^1 HEAD\| grep '^M' \| awk '{print $2}'
	readarray -t changed_files < <(git diff --name-status HEAD^1 HEAD\| grep '^M' \| awk '{print $2}')
	ci/check_changed_files.py "${changed_files[@]}"

	check_migration_files:
	name: Check Database Migration Files
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v4
	- name: Check Migration Files
	run: ci/check_database_migrations.py

	check_for_version_bump:
	name: Check for Version Bump
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 2
	set-safe-directory: true
	fetch-tags: true

	- name: Check for Version Change
	run: \|
	git diff --name-status HEAD^1 HEAD\| awk '{print $2}'
	readarray -t changed_files < <(git diff --name-status HEAD^1 HEAD\| awk '{print $2}')
	ci/check_version_bump.py "${changed_files[@]}"

	test_rest_gateway:
	name: Test REST Gateway
	runs-on: ubuntu-22.04
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	fetch-tags: true

	- name: Mark repository as safe
	run: git config --global --add safe.directory $GITHUB_WORKSPACE

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version: '1.21'

	- name: Install Protocol Buffers Compiler
	run: \|
	sudo apt-get update
	sudo apt-get install -y protobuf-compiler

	- name: Install Go tools
	run: \|
	go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway@latest
	go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@latest
	go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest

	- name: Generate protobuf code
	working-directory: rest_gateway/opencue_gateway
	run: \|
	mkdir -p gen/go
	protoc -I ../../proto/src/ \
	--go_out ./gen/go/ \
	--go_opt paths=source_relative \
	--go-grpc_out ./gen/go/ \
	--go-grpc_opt paths=source_relative \
	../../proto/src/*.proto
	protoc -I ../../proto/src/ \
	--grpc-gateway_out ./gen/go \
	--grpc-gateway_opt paths=source_relative \
	--grpc-gateway_opt generate_unbound_methods=true \
	../../proto/src/*.proto

	- name: Install Go dependencies
	working-directory: rest_gateway/opencue_gateway
	run: \|
	if [ ! -f go.mod ]; then go mod init opencue_gateway; fi
	go mod tidy

	- name: Run unit tests with coverage
	working-directory: rest_gateway/opencue_gateway
	run: \|
	go test -v -coverprofile=coverage.out -covermode=atomic .
	go tool cover -func=coverage.out

	- name: Upload coverage report
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: rest-gateway-coverage
	path: rest_gateway/opencue_gateway/coverage.out

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[POC] Distributed scheduler #4064

Workflow file

[POC] Distributed scheduler #4064

Uh oh!

Jobs

Run details

Workflow file for this run