Build and Test #39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - main | |
| tags: | |
| - v* | |
| pull_request: | |
| branches: | |
| - '**' | |
| workflow_dispatch: | |
| inputs: | |
| logLevel: | |
| default: warning | |
| description: "Log level" | |
| required: true | |
| tags: | |
| description: "Test scenario tags" | |
| concurrency: | |
| # older builds for the same pull request number or branch should be cancelled | |
| cancel-in-progress: true | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| jobs: | |
| lint: | |
| name: Check linting | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Display Python version | |
| run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);" | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox>=4 | |
| - name: Set PY | |
| run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Run fix_lint | |
| run: python -m tox run -e fix_lint | |
| dependency: | |
| name: Check dependency | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| python-version: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| - name: Install tox | |
| run: python -m pip install tox>=4 | |
| - name: Run tests | |
| run: python -m tox run -e dependency | |
| build: | |
| needs: lint | |
| strategy: | |
| matrix: | |
| os: | |
| - image: ubuntu-latest | |
| id: manylinux_x86_64 | |
| # - image: ubuntu-20.04 | |
| # id: manylinux_aarch64 | |
| # - image: windows-2019 | |
| # id: win_amd64 | |
| # - image: macos-latest | |
| # id: macosx_x86_64 | |
| # - image: macos-latest | |
| # id: macosx_arm64 | |
| # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| python-version: ["3.11"] | |
| name: Build ${{ matrix.os.id }}-py${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os.image }} | |
| steps: | |
| - name: Set shortver | |
| run: echo "shortver=${longver//./}" >> $GITHUB_ENV | |
| env: | |
| longver: ${{ matrix.python-version }} | |
| shell: bash | |
| - name: Set up QEMU | |
| if: ${{ matrix.os.id == 'manylinux_aarch64' }} | |
| uses: docker/setup-qemu-action@v2 | |
| with: | |
| # xref https://github.com/docker/setup-qemu-action/issues/188 | |
| # xref https://github.com/tonistiigi/binfmt/issues/215 | |
| image: tonistiigi/binfmt:qemu-v8.1.5 | |
| platforms: all | |
| - uses: actions/checkout@v4 | |
| - name: Building wheel | |
| uses: pypa/[email protected] | |
| env: | |
| CIBW_BUILD: cp${{ env.shortver }}-${{ matrix.os.id }} | |
| MACOSX_DEPLOYMENT_TARGET: 10.14 # Should be kept in sync with ci/build_darwin.sh | |
| with: | |
| output-dir: dist | |
| - name: Show wheels generated | |
| run: ls -lh dist | |
| shell: bash | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: ${{ matrix.os.id }}_py${{ matrix.python-version }} | |
| path: dist/ | |
| test: | |
| name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| needs: build | |
| timeout-minutes: 30 | |
| runs-on: ${{ matrix.os.image_name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - image_name: ubuntu-latest | |
| download_name: manylinux_x86_64 | |
| # - image_name: macos-latest | |
| # download_name: macosx_x86_64 | |
| # - image_name: windows-2019 | |
| # download_name: win_amd64 | |
| # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] | |
| python-version: ["3.11"] | |
| # cloud-provider: [aws, azure, gcp] | |
| cloud-provider: [aws] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Display Python version | |
| run: python -c "import sys; print(sys.version)" | |
| # - name: Set up Java | |
| # uses: actions/setup-java@v4 # for wiremock | |
| # with: | |
| # java-version: 11 | |
| # distribution: 'temurin' | |
| # java-package: 'jre' | |
| # - name: Fetch Wiremock | |
| # shell: bash | |
| # run: curl https://repo1.maven.org/maven2/org/wiremock/wiremock-standalone/3.11.0/wiremock-standalone-3.11.0.jar --output .wiremock/wiremock-standalone.jar | |
| - name: Start up LocalStack Snowflake | |
| env: | |
| LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} | |
| run: | | |
| image_name=localstack/snowflake | |
| # image_name=whummer/snowflake-tmp | |
| docker pull $image_name & | |
| pip install localstack | |
| IMAGE_NAME=$image_name DEBUG=1 DOCKER_FLAGS='-e SF_LOG=trace' localstack start -d | |
| localstack wait | |
| # - name: Setup parameters file | |
| # shell: bash | |
| # env: | |
| # PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} | |
| # run: | | |
| # gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ | |
| # .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py | |
| - name: Download wheel(s) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ matrix.os.download_name }}_py${{ matrix.python-version }} | |
| path: dist | |
| - name: Show wheels downloaded | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upgrade setuptools, pip and wheel | |
| run: python -m pip install -U setuptools pip wheel | |
| - name: Install tox | |
| run: python -m pip install tox>=4 | |
| - name: Run tests | |
| # run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,integ,pandas,sso}-ci | sed 's/ /,/g'` | |
| run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,integ}-ci | sed 's/ /,/g'` | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| cloud_provider: ${{ matrix.cloud-provider }} | |
| PYTEST_ADDOPTS: --color=yes --tb=short | |
| TOX_PARALLEL_NO_SPINNER: 1 | |
| shell: bash | |
| timeout-minutes: 15 | |
| - name: Combine coverages | |
| run: python -m tox run -e coverage --skip-missing-interpreters false | |
| shell: bash | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| include-hidden-files: true | |
| name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| path: | | |
| .tox/.coverage | |
| .tox/coverage.xml | |
| - name: Print LocalStack logs | |
| run: localstack logs | |
| if: ${{ success() || failure() }} | |
| # test-olddriver: | |
| # name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| # needs: lint | |
| # runs-on: ${{ matrix.os.image_name }} | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # os: | |
| # - image_name: ubuntu-latest | |
| # download_name: linux | |
| # python-version: [3.8] | |
| # cloud-provider: [aws] | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: Set up Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: ${{ matrix.python-version }} | |
| # - name: Display Python version | |
| # run: python -c "import sys; print(sys.version)" | |
| # - name: Setup parameters file | |
| # shell: bash | |
| # env: | |
| # PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} | |
| # run: | | |
| # gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ | |
| # .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py | |
| # - name: Upgrade setuptools, pip and wheel | |
| # run: python -m pip install -U setuptools pip wheel | |
| # - name: Install tox | |
| # run: python -m pip install tox>=4 | |
| # - name: Run tests | |
| # run: python -m tox run -e olddriver | |
| # env: | |
| # PYTHON_VERSION: ${{ matrix.python-version }} | |
| # cloud_provider: ${{ matrix.cloud-provider }} | |
| # PYTEST_ADDOPTS: --color=yes --tb=short | |
| # shell: bash | |
| # test-noarrowextension: | |
| # name: No Arrow Extension Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| # needs: lint | |
| # runs-on: ${{ matrix.os.image_name }} | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # os: | |
| # - image_name: ubuntu-latest | |
| # download_name: linux | |
| # python-version: [3.8] | |
| # cloud-provider: [aws] | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: Set up Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: ${{ matrix.python-version }} | |
| # - name: Display Python version | |
| # run: python -c "import sys; print(sys.version)" | |
| # - name: Upgrade setuptools, pip and wheel | |
| # run: python -m pip install -U setuptools pip wheel | |
| # - name: Install tox | |
| # run: python -m pip install tox>=4 | |
| # - name: Run tests | |
| # run: python -m tox run -e noarrowextension | |
| # env: | |
| # PYTHON_VERSION: ${{ matrix.python-version }} | |
| # cloud_provider: ${{ matrix.cloud-provider }} | |
| # PYTEST_ADDOPTS: --color=yes --tb=short | |
| # shell: bash | |
| # test-fips: | |
| # name: Test FIPS linux-3.8-${{ matrix.cloud-provider }} | |
| # needs: build | |
| # runs-on: ubuntu-latest | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # cloud-provider: [aws] | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - name: Setup parameters file | |
| # shell: bash | |
| # env: | |
| # PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} | |
| # run: | | |
| # gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ | |
| # .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py | |
| # - name: Download wheel(s) | |
| # uses: actions/download-artifact@v3 | |
| # with: | |
| # name: manylinux_x86_64_py3.8 | |
| # path: dist | |
| # - name: Show wheels downloaded | |
| # run: ls -lh dist | |
| # shell: bash | |
| # - name: Run tests | |
| # run: ./ci/test_fips_docker.sh | |
| # env: | |
| # PYTHON_VERSION: 3.8 | |
| # cloud_provider: ${{ matrix.cloud-provider }} | |
| # PYTEST_ADDOPTS: --color=yes --tb=short | |
| # TOX_PARALLEL_NO_SPINNER: 1 | |
| # shell: bash | |
| # - uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }} | |
| # path: | | |
| # .coverage | |
| # coverage.xml | |
| # test-lambda: | |
| # name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| # needs: build | |
| # runs-on: ubuntu-latest | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| ## python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| # python-version: ["3.11"] | |
| # cloud-provider: [aws] | |
| # steps: | |
| # - name: Set shortver | |
| # run: echo "shortver=${longver//./}" >> $GITHUB_ENV | |
| # env: | |
| # longver: ${{ matrix.python-version }} | |
| # shell: bash | |
| # - uses: actions/checkout@v3 | |
| # - name: Setup parameters file | |
| # shell: bash | |
| # env: | |
| # PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} | |
| # run: | | |
| # gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ | |
| # .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py | |
| # - name: Download wheel(s) | |
| # uses: actions/download-artifact@v3 | |
| # with: | |
| # name: manylinux_x86_64_py${{ matrix.python-version }} | |
| # path: dist | |
| # - name: Show wheels downloaded | |
| # run: ls -lh dist | |
| # shell: bash | |
| # - name: Run tests | |
| # run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION} | |
| # env: | |
| # PYTHON_VERSION: ${{ matrix.python-version }} | |
| # cloud_provider: ${{ matrix.cloud-provider }} | |
| # PYTEST_ADDOPTS: --color=yes --tb=short | |
| # TOX_PARALLEL_NO_SPINNER: 1 | |
| # shell: bash | |
| # - uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }} | |
| # path: | | |
| # .coverage.py${{ env.shortver }}-lambda-ci | |
| # junit.py${{ env.shortver }}-lambda-ci-dev.xml | |
| # combine-coverage: | |
| # if: ${{ success() || failure() }} | |
| # name: Combine coverage | |
| # needs: [lint, test, test-fips, test-lambda] | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - uses: actions/download-artifact@v3 | |
| # with: | |
| # path: artifacts | |
| # - name: Set up Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.8' | |
| # - name: Display Python version | |
| # run: python -c "import sys; print(sys.version)" | |
| # - name: Upgrade setuptools and pip | |
| # run: python -m pip install -U setuptools pip wheel | |
| # - name: Install tox | |
| # run: python -m pip install tox>=4 | |
| # - name: Collect all coverages to one dir | |
| # run: | | |
| # python -c ' | |
| # from pathlib import Path | |
| # import shutil | |
| # | |
| # src_dir = Path("artifacts") | |
| # dst_dir = Path(".") / ".tox" | |
| # dst_dir.mkdir() | |
| # for src_file in src_dir.glob("*/.coverage"): | |
| # dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) | |
| # print("{} copy to {}".format(src_file, dst_file)) | |
| # shutil.copy(str(src_file), str(dst_file))' | |
| # - name: Combine coverages | |
| # run: python -m tox run -e coverage | |
| ## - name: Publish html coverage | |
| ## uses: actions/upload-artifact@v3 | |
| ## with: | |
| ## name: overall_cov_html | |
| ## path: .tox/htmlcov | |
| ## - name: Publish xml coverage | |
| ## uses: actions/upload-artifact@v3 | |
| ## with: | |
| ## name: overall_cov_xml | |
| ## path: .tox/coverage.xml | |
| # - uses: codecov/codecov-action@v3 | |
| # with: | |
| # files: .tox/coverage.xml |