# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements. See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License. You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

name: PreCommit Python

on:

pull_request_target:

branches: [ "master", "release-*" ]

paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python.json']

issue_comment:

types: [created]

push:

tags: ['v*']

branches: ['master', 'release-*']

paths: [ "model/**","sdks/python/**","release/**",".github/workflows/beam_PreCommit_Python.yml"]

schedule:

- cron: '0 3/6 * * *'

workflow_dispatch:

#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event

permissions:

actions: write

pull-requests: write

checks: write

contents: read

deployments: read

id-token: none

issues: write

discussions: read

packages: read

pages: read

repository-projects: read

security-events: read

statuses: read

# This allows a subsequently queued workflow run to interrupt previous runs

concurrency:

group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'

cancel-in-progress: true

env:

DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}

GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}

GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}

# Aggressive stability settings for flaky CI environment

PYTHONHASHSEED: "0"

OMP_NUM_THREADS: "1"

OPENBLAS_NUM_THREADS: "1"

# gRPC stability - more conservative for unstable networks

GRPC_ARG_KEEPALIVE_TIME_MS: "10000"

GRPC_ARG_KEEPALIVE_TIMEOUT_MS: "15000"

GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS: "1"

GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA: "0"

GRPC_ARG_MAX_RECONNECT_BACKOFF_MS: "30000"

# Beam-specific - very generous timeouts

BEAM_RETRY_MAX_ATTEMPTS: "5"

BEAM_RETRY_INITIAL_DELAY_MS: "5000"

BEAM_RETRY_MAX_DELAY_MS: "120000"

# Force stable execution

BEAM_TESTING_FORCE_SINGLE_BUNDLE: "true"

BEAM_TESTING_DETERMINISTIC_ORDER: "true"

jobs:

beam_PreCommit_Python:

name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})

runs-on: [self-hosted, ubuntu-20.04, main]

timeout-minutes: 180

strategy:

fail-fast: false

matrix:

job_name: ['beam_PreCommit_Python']

job_phrase: ['Run Python PreCommit']

python_version: ['3.10','3.11','3.12','3.13']

if: |

github.event_name == 'push' ||

github.event_name == 'pull_request_target' ||

(github.event_name == 'schedule' && github.repository == 'apache/beam') ||

github.event_name == 'workflow_dispatch' ||

startsWith(github.event.comment.body, 'Run Python PreCommit')

steps:

- uses: actions/checkout@v4

- name: Setup repository

uses: ./.github/actions/setup-action

with:

comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}

github_token: ${{ secrets.GITHUB_TOKEN }}

github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})

- name: Setup environment

uses: ./.github/actions/setup-environment-action

with:

java-version: default

python-version: ${{ matrix.python_version }}

- name: Set PY_VER_CLEAN

id: set_py_ver_clean

run: |

PY_VER=${{ matrix.python_version }}

PY_VER_CLEAN=${PY_VER//.}

echo "py_ver_clean=$PY_VER_CLEAN" >> $GITHUB_OUTPUT

- name: Run pythonPreCommit

env:

TOX_TESTENV_PASSENV: "DOCKER_*,TESTCONTAINERS_*,TC_*,BEAM_*,GRPC_*,OMP_*,OPENBLAS_*,PYTHONHASHSEED,PYTEST_*"

# Aggressive retry and timeout settings for flaky CI

PYTEST_ADDOPTS: "-v --tb=short --maxfail=5 --durations=30 --reruns=5 --reruns-delay=15 --timeout=600 --disable-warnings"

# Container stability - much more generous timeouts

TC_TIMEOUT: "300"

TC_MAX_TRIES: "15"

TC_SLEEP_TIME: "5"

# Additional gRPC stability for flaky environment

GRPC_ARG_MAX_CONNECTION_IDLE_MS: "60000"

GRPC_ARG_HTTP2_BDP_PROBE: "1"

GRPC_ARG_SO_REUSEPORT: "1"

# Force sequential execution to reduce load

PYTEST_XDIST_WORKER_COUNT: "1"

# Additional gRPC settings

GRPC_ARG_MAX_RECONNECT_BACKOFF_MS: "120000"

GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS: "2000"

uses: ./.github/actions/gradle-command-self-hosted-action

with:

gradle-command: :sdks:python:test-suites:tox:py${{steps.set_py_ver_clean.outputs.py_ver_clean}}:preCommitPy${{steps.set_py_ver_clean.outputs.py_ver_clean}}

arguments: |

-Pposargs="--ignore=apache_beam/dataframe/ --ignore=apache_beam/ml/ --ignore=apache_beam/examples/ --ignore=apache_beam/runners/ --ignore=apache_beam/transforms/" \

-PpythonVersion=${{ matrix.python_version }}

- name: Archive Python Test Results

uses: actions/upload-artifact@v4

if: failure()

with:

name: Python ${{ matrix.python_version }} Test Results

path: '**/pytest*.xml'

- name: Publish Python Test Results

uses: EnricoMi/publish-unit-test-result-action@v2

if: always()

with:

commit: '${{ env.prsha || env.GITHUB_SHA }}'

comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}

files: '**/pytest*.xml'

large_files: true

- name: Cleanup

if: always()

run: |

# Kill any remaining processes

sudo pkill -f "gradle" || true

sudo pkill -f "java" || true

sudo pkill -f "python.*pytest" || true

# Clean up temp files

sudo rm -rf /tmp/beam-* || true

sudo rm -rf /tmp/gradle-* || true