Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit cbaa083

Browse files
damccormdamccorm
committed
Merge branch 'master' of https://github.com/apache/beam into users/damccorm/prismByDefault
2 parents 20370cb + 458d7a7 commit cbaa083

43 files changed

Lines changed: 2374 additions & 330 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/trigger_files/IO_Iceberg_Integration_Tests.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 4
3+
"modification": 3
44
}

‎.github/workflows/beam_PostCommit_Python_Arm.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,12 @@ jobs:
8585
sudo curl -L https://github.com/docker/compose/releases/download/1.22.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose
8686
sudo chmod +x /usr/local/bin/docker-compose
8787
- name: Authenticate on GCP
88-
uses: google-github-actions/setup-gcloud@v0
88+
uses: google-github-actions/auth@v2
8989
with:
90-
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
91-
service_account_key: ${{ secrets.GCP_SA_KEY }}
92-
export_default_credentials: true
90+
service_account: ${{ secrets.GCP_SA_EMAIL }}
91+
credentials_json: ${{ secrets.GCP_SA_KEY }}
92+
- name: Set up Cloud SDK
93+
uses: google-github-actions/setup-gcloud@v2
9394
- name: Set up Docker Buildx
9495
uses: docker/setup-buildx-action@v2
9596
- name: GCloud Docker credential helper

‎.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,12 @@ jobs:
9595
- name: Set up Docker Buildx
9696
uses: docker/setup-buildx-action@v1
9797
- name: Authenticate on GCP
98-
uses: google-github-actions/setup-gcloud@v0
98+
uses: google-github-actions/auth@v2
9999
with:
100-
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
101-
service_account_key: ${{ secrets.GCP_SA_KEY }}
100+
service_account: ${{ secrets.GCP_SA_EMAIL }}
101+
credentials_json: ${{ secrets.GCP_SA_KEY }}
102+
- name: Set up Cloud SDK
103+
uses: google-github-actions/setup-gcloud@v2
102104
- name: GCloud Docker credential helper
103105
run: |
104106
gcloud auth configure-docker ${{ env.docker_registry }}

‎.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,12 @@ jobs:
7575
with:
7676
python-version: ${{ matrix.python_version }}
7777
- name: Authenticate on GCP
78-
uses: google-github-actions/setup-gcloud@v0
78+
uses: google-github-actions/auth@v2
7979
with:
80-
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
81-
service_account_key: ${{ secrets.GCP_SA_KEY }}
82-
export_default_credentials: true
80+
service_account: ${{ secrets.GCP_SA_EMAIL }}
81+
credentials_json: ${{ secrets.GCP_SA_KEY }}
82+
- name: Set up Cloud SDK
83+
uses: google-github-actions/setup-gcloud@v2
8384
- name: Set up Docker Buildx
8485
uses: docker/setup-buildx-action@v2
8586
- name: GCloud Docker credential helper

‎.github/workflows/refresh_looker_metrics.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@ jobs:
4343
python-version: 3.11
4444
- run: pip install requests google-cloud-storage looker-sdk
4545
- name: Authenticate on GCP
46-
uses: google-github-actions/setup-gcloud@v0
46+
uses: google-github-actions/auth@v2
4747
with:
48-
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
49-
service_account_key: ${{ secrets.GCP_SA_KEY }}
50-
export_default_credentials: true
48+
service_account: ${{ secrets.GCP_SA_EMAIL }}
49+
credentials_json: ${{ secrets.GCP_SA_KEY }}
50+
- name: Set up Cloud SDK
51+
uses: google-github-actions/setup-gcloud@v2
5152
- run: python .test-infra/tools/refresh_looker_metrics.py

‎.github/workflows/republish_released_docker_containers.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,12 @@ jobs:
7272
with:
7373
python-version: '3.9'
7474
- name: Authenticate on GCP
75-
uses: google-github-actions/setup-gcloud@v0
75+
uses: google-github-actions/auth@v2
7676
with:
77-
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
78-
service_account_key: ${{ secrets.GCP_SA_KEY }}
77+
service_account: ${{ secrets.GCP_SA_EMAIL }}
78+
credentials_json: ${{ secrets.GCP_SA_KEY }}
79+
- name: Set up Cloud SDK
80+
uses: google-github-actions/setup-gcloud@v2
7981
- name: Set up Docker Buildx
8082
uses: docker/setup-buildx-action@v2
8183
- name: Remove default github maven configuration

‎.github/workflows/run_rc_validation_python_mobile_gaming.yml‎

Lines changed: 14 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -86,96 +86,18 @@ jobs:
8686
GAME_STATS_WINDOW_DURATION: 20
8787
SUBMISSION_TIMEOUT_SECONDS: 120 # Timeout for the python submission script itself
8888
# --- Define the validation function with enhanced debugging (FIXED QUOTING) ---
89-
VALIDATE_TABLE_FUNC: |
90-
validate_table() {
91-
local table_name=$1
92-
echo "DEBUG: ===== Starting validate_table for table: $table_name ====="
93-
# Ensure required env vars are set (GCP_PROJECT_ID, BQ_DATASET are inherited)
94-
if [[ -z "$GCP_PROJECT_ID" || -z "$BQ_DATASET" ]]; then
95-
echo "ERROR: GCP_PROJECT_ID and BQ_DATASET must be set in the environment."
96-
exit 1
97-
fi
98-
99-
local full_table_id="${GCP_PROJECT_ID}.${BQ_DATASET}.${table_name}"
100-
local full_table_id_show="${GCP_PROJECT_ID}:${BQ_DATASET}.${table_name}"
101-
local count=""
102-
local exit_code=1
103-
local retries=10
104-
local delay=60 # Default seconds between retries
105-
106-
# Allow overriding delay via second argument (optional)
107-
if [[ -n "$2" && "$2" =~ ^[0-9]+$ ]]; then
108-
delay=$2
109-
echo "DEBUG: Using custom retry delay: ${delay}s for table ${table_name}"
110-
else
111-
echo "DEBUG: Using default retry delay: ${delay}s for table ${table_name}"
112-
fi
113-
echo "DEBUG: Full table ID: ${full_table_id}, Max retries: ${retries}"
114-
115-
for i in $(seq 1 $retries); do
116-
echo "DEBUG: Starting attempt $i/$retries..."
117-
local query_output
118-
119-
echo "DEBUG: Executing: bq query --project_id=${GCP_PROJECT_ID} --use_legacy_sql=false --format=sparse --max_rows=1 \"SELECT COUNT(*) FROM \`${full_table_id}\`\""
120-
query_output=$(bq query --project_id=${GCP_PROJECT_ID} \
121-
--use_legacy_sql=false \
122-
--format=sparse \
123-
--max_rows=1 \
124-
"SELECT COUNT(*) FROM \`${full_table_id}\`" 2>&1)
125-
exit_code=$?
126-
127-
echo "DEBUG: bq query exit code: $exit_code"
128-
echo "DEBUG: bq query raw output: [$query_output]"
129-
130-
if [ $exit_code -eq 0 ]; then
131-
echo "DEBUG: bq query exited successfully (code 0)."
132-
count=$(echo "$query_output" | tail -n 1 | tr -d '[:space:]')
133-
echo "DEBUG: Processed count after removing whitespace (from last line): [$count]"
134-
if [[ "$count" =~ ^[0-9]+$ ]] && [ "$count" -gt 0 ]; then
135-
echo "DEBUG: Count [$count] is a positive integer. Validation successful for this attempt."
136-
break # Success! Found non-zero rows
137-
else
138-
echo "DEBUG: Count [$count] is zero or not a positive integer."
139-
if [[ "$count" == "0" ]]; then
140-
echo "DEBUG: Explicit count of 0 received."
141-
fi
142-
fi
143-
else
144-
echo "DEBUG: bq query failed (exit code: $exit_code)."
145-
echo "DEBUG: Checking table existence with bq show..."
146-
if ! bq show --project_id=${GCP_PROJECT_ID} "${full_table_id_show}" > /dev/null 2>&1; then
147-
echo "DEBUG: Table ${full_table_id_show} appears not to exist (bq show failed)."
148-
else
149-
echo "DEBUG: Table ${full_table_id_show} appears to exist (bq show succeeded), but query failed."
150-
fi
151-
fi
152-
153-
if [ $i -lt $retries ]; then
154-
echo "DEBUG: Validation condition not met on attempt $i. Retrying in $delay seconds..."
155-
sleep $delay
156-
else
157-
echo "DEBUG: Final attempt ($i) failed."
158-
fi
159-
done
160-
161-
echo "DEBUG: ===== Final validation check for table: $table_name ====="
162-
if [[ "$count" =~ ^[0-9]+$ ]] && [ "$count" -gt 0 ]; then
163-
echo "SUCCESS: Table ${table_name} has ${count} rows. Final validation OK."
164-
echo "DEBUG: validate_table returning 0 (success)."
165-
return 0 # Indicate success
166-
else
167-
echo "ERROR: Failed to get a non-zero row count for table ${table_name} after $retries retries (Last exit code: $exit_code, Last processed count: '$count')."
168-
echo "DEBUG: validate_table returning 1 (failure)."
169-
return 1 # Indicate failure
170-
fi
171-
}
17289

17390
steps:
17491
- name: Checkout code at RC tag
17592
uses: actions/checkout@v4
17693
with:
17794
ref: ${{ env.RC_TAG }}
17895

96+
- name: Download validate_table.sh from master branch
97+
run: |
98+
curl -o ./scripts/tools/validate_table.sh https://raw.githubusercontent.com/apache/beam/master/scripts/tools/validate_table.sh
99+
chmod +x ./scripts/tools/validate_table.sh
100+
179101
- name: Setup environment
180102
uses: ./.github/actions/setup-environment-action
181103
with:
@@ -347,11 +269,10 @@ jobs:
347269
- name: Validate Leaderboard Results (Direct Runner)
348270
run: |
349271
source beam_env/bin/activate
350-
eval "$VALIDATE_TABLE_FUNC"
351272
echo "Validating BigQuery results for Leaderboard (DirectRunner)..."
352273
sleep 90
353-
validate_table "leader_board_users" || exit 1
354-
validate_table "leader_board_teams" || exit 1
274+
./scripts/tools/validate_table.sh "leader_board_users" || exit 1
275+
./scripts/tools/validate_table.sh "leader_board_teams" || exit 1
355276
echo "Leaderboard (Direct Runner) BQ validation finished successfully."
356277
shell: bash
357278

@@ -426,11 +347,10 @@ jobs:
426347
exit 0 # Exit step successfully to allow cancellation/cleanup
427348
fi
428349
source beam_env/bin/activate
429-
eval "$VALIDATE_TABLE_FUNC"
430350
echo "Validating BigQuery results for Leaderboard (DataflowRunner)..."
431351
sleep 240
432-
validate_table "leader_board_users" 15 || exit 1 # Use 15s retry delay
433-
validate_table "leader_board_teams" 15 || exit 1 # Use 15s retry delay
352+
./scripts/tools/validate_table.sh "leader_board_users" 15 || exit 1 # Use 15s retry delay
353+
./scripts/tools/validate_table.sh "leader_board_teams" 15 || exit 1 # Use 15s retry delay
434354
echo "Leaderboard (Dataflow Runner) BQ validation finished successfully."
435355
shell: bash
436356

@@ -451,7 +371,7 @@ jobs:
451371
echo "leaderboard_dataflow_jobid.txt not found, cannot cancel job (it might have failed before ID extraction)."
452372
fi
453373
shell: bash
454-
374+
455375
# ================== GameStats Tests ==================
456376
- name: Run GameStats (Direct Runner) in Background
457377
run: |
@@ -471,12 +391,11 @@ jobs:
471391
- name: Validate GameStats Results (Direct Runner)
472392
run: |
473393
source beam_env/bin/activate
474-
eval "$VALIDATE_TABLE_FUNC"
475394
echo "Validating BigQuery results for GameStats (DirectRunner)..."
476395
echo "* Sleeping for 25mins"
477396
sleep 25m
478-
validate_table "game_stats_teams" || exit 1
479-
validate_table "game_stats_sessions" || exit 1
397+
./scripts/tools/validate_table.sh "game_stats_teams" || exit 1
398+
./scripts/tools/validate_table.sh "game_stats_sessions" || exit 1
480399
echo "GameStats (Direct Runner) BQ validation finished successfully."
481400
shell: bash
482401

@@ -552,12 +471,11 @@ jobs:
552471
exit 0 # Exit step successfully to allow cleanup
553472
fi
554473
source beam_env/bin/activate
555-
eval "$VALIDATE_TABLE_FUNC"
556474
echo "Validating BigQuery results for GameStats (DataflowRunner)..."
557475
echo "* Sleeping for 25mins"
558476
sleep 25m
559-
validate_table "game_stats_teams" 15 || exit 1 # Use 15s retry delay
560-
validate_table "game_stats_sessions" 15 || exit 1 # Use 15s retry delay
477+
./scripts/tools/validate_table.sh "game_stats_teams" 15 || exit 1 # Use 15s retry delay
478+
./scripts/tools/validate_table.sh "game_stats_sessions" 15 || exit 1 # Use 15s retry delay
561479
echo "GameStats (Dataflow Runner) BQ validation finished successfully."
562480
shell: bash
563481

‎runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
*/
1818
package org.apache.beam.runners.flink;
1919

20-
import static org.apache.beam.sdk.util.Preconditions.checkStateNotNull;
2120
import static org.apache.beam.sdk.util.construction.resources.PipelineResources.detectClassPathResourcesToStage;
2221

2322
import java.util.UUID;
@@ -32,10 +31,14 @@
3231
import org.apache.beam.vendor.grpc.v1p69p0.com.google.protobuf.Struct;
3332
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings;
3433
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService;
34+
import org.checkerframework.checker.nullness.qual.Nullable;
3535
import org.slf4j.Logger;
3636
import org.slf4j.LoggerFactory;
3737

3838
/** Job Invoker for the {@link FlinkRunner}. */
39+
@SuppressWarnings({
40+
"nullness" // TODO(https://github.com/apache/beam/issues/20497)
41+
})
3942
public class FlinkJobInvoker extends JobInvoker {
4043
private static final Logger LOG = LoggerFactory.getLogger(FlinkJobInvoker.class);
4144

@@ -54,7 +57,7 @@ protected FlinkJobInvoker(FlinkJobServerDriver.FlinkServerConfiguration serverCo
5457
protected JobInvocation invokeWithExecutor(
5558
RunnerApi.Pipeline pipeline,
5659
Struct options,
57-
String retrievalToken,
60+
@Nullable String retrievalToken,
5861
ListeningExecutorService executorService) {
5962

6063
// TODO: How to make Java/Python agree on names of keys and their values?
@@ -71,22 +74,20 @@ protected JobInvocation invokeWithExecutor(
7174

7275
PortablePipelineOptions portableOptions = flinkOptions.as(PortablePipelineOptions.class);
7376

74-
ClassLoader thisClassLoader =
75-
checkStateNotNull(
76-
FlinkJobInvoker.class.getClassLoader(),
77-
"FlinkJobInvoker class loader is null - this means it was loaded by the bootstrap classloader, which should be impossible");
78-
7977
PortablePipelineRunner pipelineRunner;
8078
if (Strings.isNullOrEmpty(portableOptions.getOutputExecutablePath())) {
8179
pipelineRunner =
8280
new FlinkPipelineRunner(
8381
flinkOptions,
8482
serverConfig.getFlinkConfDir(),
85-
detectClassPathResourcesToStage(thisClassLoader, flinkOptions));
83+
detectClassPathResourcesToStage(
84+
FlinkJobInvoker.class.getClassLoader(), flinkOptions));
8685
} else {
8786
pipelineRunner = new PortablePipelineJarCreator(FlinkPipelineRunner.class);
8887
}
8988

89+
flinkOptions.setRunner(null);
90+
9091
LOG.info("Invoking job {} with pipeline runner {}", invocationId, pipelineRunner);
9192
return createJobInvocation(
9293
invocationId, retrievalToken, executorService, pipeline, flinkOptions, pipelineRunner);

‎runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/JobInvoker.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.io.IOException;
2121
import java.util.concurrent.Executors;
2222
import java.util.concurrent.ThreadFactory;
23+
import javax.annotation.Nullable;
2324
import org.apache.beam.model.pipeline.v1.RunnerApi;
2425
import org.apache.beam.vendor.grpc.v1p69p0.com.google.protobuf.Struct;
2526
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ListeningExecutorService;
@@ -38,11 +39,11 @@ public abstract class JobInvoker {
3839
protected abstract JobInvocation invokeWithExecutor(
3940
RunnerApi.Pipeline pipeline,
4041
Struct options,
41-
String retrievalToken,
42+
@Nullable String retrievalToken,
4243
ListeningExecutorService executorService)
4344
throws IOException;
4445

45-
JobInvocation invoke(RunnerApi.Pipeline pipeline, Struct options, String retrievalToken)
46+
JobInvocation invoke(RunnerApi.Pipeline pipeline, Struct options, @Nullable String retrievalToken)
4647
throws IOException {
4748
return invokeWithExecutor(pipeline, options, retrievalToken, this.executorService);
4849
}

‎runners/prism/java/build.gradle‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,6 @@ def sickbayTests = [
110110
'org.apache.beam.sdk.transforms.GroupIntoBatchesTest.testWithShardedKeyInGlobalWindow',
111111

112112
// Java side dying during execution.
113-
// https://github.com/apache/beam/issues/32930
114-
'org.apache.beam.sdk.transforms.FlattenTest.testFlattenMultipleCoders',
115113
// Stream corruption error java side: failed:java.io.StreamCorruptedException: invalid stream header: 206E6F74
116114
// Likely due to prism't coder changes.
117115
'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2',

0 commit comments

Comments
 (0)