diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 59ad718cf..0b4a8e6c4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,198 +1,85 @@ -name: CI/CD +# Derived from https://github.com/actions/starter-workflows/blob/main/ci/python-package.yml +# +name: Python Package on: push: branches: ["master"] pull_request: branches: ["master"] - release: - types: [created] - branches: - - 'master' - workflow_dispatch: env: FORCE_COLOR: "1" # Make tools pretty. PIP_DISABLE_PIP_VERSION_CHECK: "1" PIP_NO_PYTHON_VERSION_WARNING: "1" - PYTHON_LATEST: "3.12" - KAFKA_LATEST: "2.6.0" - - # For re-actors/checkout-python-sdist - sdist-artifact: python-package-distributions jobs: + build: - build-sdist: - name: 📦 Build the source distribution - runs-on: ubuntu-latest - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_LATEST }} - cache: pip - - run: python -m pip install build - name: Install core libraries for build and install - - name: Build artifacts - run: python -m build - - name: Upload built artifacts for testing - uses: actions/upload-artifact@v3 - with: - name: ${{ env.sdist-artifact }} - # NOTE: Exact expected file names are specified here - # NOTE: as a safety measure — if anything weird ends - # NOTE: up being in this dir or not all dists will be - # NOTE: produced, this will fail the workflow. - path: dist/${{ env.sdist-name }} - retention-days: 15 - - test-python: - name: Tests on ${{ matrix.python-version }} - needs: - - build-sdist runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} + name: "Test: python ${{ matrix.python }} / kafka ${{ matrix.kafka }}" + continue-on-error: ${{ matrix.experimental || false }} strategy: fail-fast: false matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" + kafka: + - "0.8.2.2" + - "0.9.0.1" + - "0.10.2.2" + - "0.11.0.3" + - "1.1.1" + - "2.4.0" + - "2.5.0" + - "2.6.0" + python: - "3.12" - experimental: [ false ] include: - - python-version: "pypy3.9" - experimental: true - - python-version: "~3.13.0-0" - experimental: true + #- python: "pypy3.9" + # kafka: "2.6.0" + # experimental: true + #- python: "~3.13.0-0" + # kafka: "2.6.0" + # experimental: true + - python: "3.8" + kafka: "2.6.0" + - python: "3.9" + kafka: "2.6.0" + - python: "3.10" + kafka: "2.6.0" + - python: "3.11" + kafka: "2.6.0" + steps: - - name: Checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 11 - - name: Set up Python + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.python }} cache: pip cache-dependency-path: | requirements-dev.txt - - name: Check Java installation - run: source travis_java_install.sh - - name: Pull Kafka releases - run: ./build_integration.sh - env: - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ env.KAFKA_LATEST }} - # TODO: Cache releases to expedite testing - name: Install dependencies run: | sudo apt install -y libsnappy-dev libzstd-dev python -m pip install --upgrade pip - python -m pip install tox tox-gh-actions - pip install . pip install -r requirements-dev.txt - - name: Test with tox - run: tox - env: - PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ env.KAFKA_LATEST }} - - test-kafka: - name: Tests for Kafka ${{ matrix.kafka-version }} - needs: - - build-sdist - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - kafka-version: - - "0.8.2.2" - - "0.9.0.1" - - "0.10.2.2" - - "0.11.0.2" - - "0.11.0.3" - - "1.1.1" - - "2.4.0" - - "2.5.0" - - "2.6.0" - steps: - - name: Checkout the source code - uses: actions/checkout@v4 - with: - fetch-depth: 0 + pip install tox-gh-actions + - name: Pylint + run: pylint --recursive=y --errors-only --exit-zero kafka test - name: Setup java uses: actions/setup-java@v4 with: distribution: temurin - java-version: 8 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_LATEST }} - cache: pip - cache-dependency-path: | - requirements-dev.txt + java-version: 11 + - name: Check Java installation + run: source travis_java_install.sh - name: Pull Kafka releases run: ./build_integration.sh env: - # This is fast enough as long as you pull only one release at a time, - # no need to worry about caching PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ matrix.kafka-version }} - - name: Install dependencies - run: | - sudo apt install -y libsnappy-dev libzstd-dev - python -m pip install --upgrade pip - python -m pip install tox tox-gh-actions - pip install . - pip install -r requirements-dev.txt + KAFKA_VERSION: ${{ matrix.kafka }} - name: Test with tox run: tox env: PLATFORM: ${{ matrix.platform }} - KAFKA_VERSION: ${{ matrix.kafka-version }} - - check: # This job does nothing and is only used for the branch protection - name: ✅ Ensure the required checks passing - if: always() - needs: - - build-sdist - - test-python - - test-kafka - runs-on: ubuntu-latest - steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 - with: - jobs: ${{ toJSON(needs) }} - publish: - name: 📦 Publish to PyPI - runs-on: ubuntu-latest - needs: [build-sdist] - permissions: - id-token: write - environment: pypi - if: github.event_name == 'release' && github.event.action == 'created' - steps: - - name: Download the sdist artifact - uses: actions/download-artifact@v3 - with: - name: artifact - path: dist - - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} + KAFKA_VERSION: ${{ matrix.kafka }} diff --git a/build_integration.sh b/build_integration.sh index c020b0fe2..b74d86479 100755 --- a/build_integration.sh +++ b/build_integration.sh @@ -48,7 +48,7 @@ pushd servers echo "Downloading kafka ${kafka} tarball" TARBALL=${DIST_BASE_URL}${kafka}/${KAFKA_ARTIFACT} if command -v wget 2>/dev/null; then - wget -N $TARBALL + wget -nv -N $TARBALL else echo "wget not found... using curl" curl -f $TARBALL -o ${KAFKA_ARTIFACT} diff --git a/tox.ini b/tox.ini index d9b1e36d4..7417387ed 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ deps = xxhash crc32c commands = - pytest {posargs:--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka --cov-config=.covrc} + pytest {posargs:--cov=kafka --cov-config=.covrc} setenv = CRC32C_SW_MODE = auto PROJECT_ROOT = {toxinidir}