Add --region to more Python tests and examples.

Kyle Weaver · Kyle Weaver · commit 55d6ca2da036 · 2020-04-03T12:45:38.000-04:00
diff --git a/release/src/main/python-release/python_release_automation_utils.sh b/release/src/main/python-release/python_release_automation_utils.sh
@@ -325,6 +325,7 @@ RC_STAGING_URL="https://dist.apache.org/repos/dist/dev/beam/$VERSION/python"
 
 # Cloud Configurations
 PROJECT_ID='apache-beam-testing'
+REGION_ID='us-central1'
 BUCKET_NAME='temp-storage-for-release-validation-tests/nightly-snapshot-validation'
 TEMP_DIR='/tmp'
 DATASET='beam_postrelease_mobile_gaming'
diff --git a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
@@ -80,6 +80,7 @@ function verify_userscore_dataflow() {
   output_file_name="$USERSCORE_OUTPUT_PREFIX-dataflow-runner.txt"
   python -m apache_beam.examples.complete.game.user_score \
     --project=$PROJECT_ID \
+    --region=$REGION_ID \
     --runner=DataflowRunner \
     --temp_location=gs://$BUCKET_NAME/temp/ \
     --sdk_location=$BEAM_PYTHON_SDK \
@@ -121,6 +122,7 @@ function verify_hourlyteamscore_dataflow() {
   print_separator "Running HourlyTeamScore example with DataflowRunner"
   python -m apache_beam.examples.complete.game.hourly_team_score \
     --project=$PROJECT_ID \
+    --region=$REGION_ID \
     --dataset=$DATASET \
     --runner=DataflowRunner \
     --temp_location=gs://$BUCKET_NAME/temp/ \
diff --git a/release/src/main/python-release/run_release_candidate_python_quickstart.sh b/release/src/main/python-release/run_release_candidate_python_quickstart.sh
@@ -116,6 +116,7 @@ function verify_wordcount_dataflow() {
     --runner DataflowRunner \
     --job_name wordcount \
     --project $PROJECT_ID \
+    --region $REGION_ID \
     --num_workers $NUM_WORKERS \
     --sdk_location $BEAM_PYTHON_SDK
 
@@ -177,6 +178,7 @@ function verify_streaming_wordcount_dataflow() {
     --streaming \
     --job_name pyflow-wordstream-candidate \
     --project $PROJECT_ID \
+    --region $REGION_ID \
     --runner DataflowRunner \
     --input_topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC1 \
     --output_topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC2 \
diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh
@@ -140,6 +140,7 @@ if [[ -z `which gcloud` ]]; then
 
     gcloud init
     gcloud config set project ${USER_GCP_PROJECT}
+    gcloud config set compute/region ${USER_GCP_REGION}
 
     echo "-----------------Setting Up Service Account-----------------"
     if [[ ! -z "${USER_SERVICE_ACCOUNT_EMAIL}" ]]; then
@@ -468,6 +469,7 @@ if [[ ("$python_leaderboard_direct" = true \
       . ${LOCAL_BEAM_DIR}/beam_env_${py_version}/bin/activate
       python -m apache_beam.examples.complete.game.leader_board \
       --project=${USER_GCP_PROJECT} \
+      --region=${USER_GCP_REGION} \
       --topic projects/${USER_GCP_PROJECT}/topics/${SHARED_PUBSUB_TOPIC} \
       --dataset ${LEADERBOARD_DF_DATASET} \
       --runner DataflowRunner \
@@ -542,6 +544,7 @@ if [[ ("$python_leaderboard_direct" = true \
       . ${LOCAL_BEAM_DIR}/beam_env_${py_version}/bin/activate
       python -m apache_beam.examples.complete.game.game_stats \
       --project=${USER_GCP_PROJECT} \
+      --region=${USER_GCP_REGION} \
       --topic projects/${USER_GCP_PROJECT}/topics/${SHARED_PUBSUB_TOPIC} \
       --dataset ${GAMESTATS_DF_DATASET} \
       --runner DataflowRunner \
diff --git a/release/src/main/scripts/script.config b/release/src/main/scripts/script.config
@@ -31,6 +31,7 @@ RC_VALIDATE_CONFIGS=(
   REPO_URL
   INSTALL_GCLOUD
   USER_GCP_PROJECT
+  USER_GCP_REGION
   USER_GCS_BUCKET
   USER_SERVICE_ACCOUNT_EMAIL
   INSTALL_HUB
@@ -90,6 +91,11 @@ INSTALL_GCLOUD=true
 # e.g. apache-beam-testing
 USER_GCP_PROJECT=
 
+# GCP region id
+# Required for running pipeline with DataflowRunner for validation.
+# e.g. us-central1
+USER_GCP_REGION=
+
 # GCS bucket name
 # Required for running pipeline with DataflowRunner for validation.
 # e.g. gs://bucket-name (please include 'gs://' prefix)
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py
@@ -64,6 +64,7 @@
 # DataflowRunner
 python game_stats.py \
     --project $PROJECT_ID \
+    --region $REGION_ID \
     --topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC \
     --dataset $BIGQUERY_DATASET \
     --runner DataflowRunner \
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
@@ -23,6 +23,7 @@
   python setup.py nosetests --test-pipeline-options=" \
       --runner=TestDataflowRunner \
       --project=... \
+      --region=... \
       --staging_location=gs://... \
       --temp_location=gs://... \
       --output=gs://... \
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score.py
@@ -59,6 +59,7 @@
 # DataflowRunner
 python hourly_team_score.py \
     --project $PROJECT_ID \
+    --region $REGION_ID \
     --dataset $BIGQUERY_DATASET \
     --runner DataflowRunner \
     --temp_location gs://$BUCKET/user_score/temp
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
@@ -23,6 +23,7 @@
   python setup.py nosetests --test-pipeline-options=" \
       --runner=TestDataflowRunner \
       --project=... \
+      --region=... \
       --staging_location=gs://... \
       --temp_location=gs://... \
       --output=gs://... \
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py
@@ -72,6 +72,7 @@
 # DataflowRunner
 python leader_board.py \
     --project $PROJECT_ID \
+    --region $REGION_ID \
     --topic projects/$PROJECT_ID/topics/$PUBSUB_TOPIC \
     --dataset $BIGQUERY_DATASET \
     --runner DataflowRunner \
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
@@ -23,6 +23,7 @@
   python setup.py nosetests --test-pipeline-options=" \
       --runner=TestDataflowRunner \
       --project=... \
+      --region=... \
       --staging_location=gs://... \
       --temp_location=gs://... \
       --output=gs://... \
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score.py b/sdks/python/apache_beam/examples/complete/game/user_score.py
@@ -50,6 +50,7 @@
     --output gs://$BUCKET/user_score/output \
     --runner DataflowRunner \
     --project $PROJECT_ID \
+    --region $REGION_ID \
     --temp_location gs://$BUCKET/user_score/temp
 """
 
diff --git a/sdks/python/apache_beam/examples/complete/game/user_score_it_test.py b/sdks/python/apache_beam/examples/complete/game/user_score_it_test.py
@@ -23,6 +23,7 @@
   python setup.py nosetests --test-pipeline-options=" \
       --runner=TestDataflowRunner \
       --project=... \
+      --region=... \
       --staging_location=gs://... \
       --temp_location=gs://... \
       --output=gs://... \
diff --git a/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py b/sdks/python/apache_beam/examples/complete/juliaset/juliaset_main.py
@@ -38,6 +38,7 @@
 python juliaset_main.py \
   --job_name juliaset-$USER \
   --project YOUR-PROJECT \
+  --region GCE-REGION \
   --runner DataflowRunner \
   --setup_file ./setup.py \
   --staging_location gs://YOUR-BUCKET/juliaset/staging \
diff --git a/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py b/sdks/python/apache_beam/examples/complete/top_wikipedia_sessions.py
@@ -31,6 +31,7 @@
 
   --job_name NAME_FOR_YOUR_JOB
   --project YOUR_PROJECT_ID
+  --region GCE_REGION
   --staging_location gs://YOUR_STAGING_DIRECTORY
   --temp_location gs://YOUR_TEMPORARY_DIRECTORY
   --runner DataflowRunner
diff --git a/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py b/sdks/python/apache_beam/examples/cookbook/multiple_output_pardo.py
@@ -38,6 +38,7 @@
 pipeline configuration:::
 
   --project YOUR_PROJECT_ID
+  --region GCE_REGION
   --staging_location gs://YOUR_STAGING_DIRECTORY
   --temp_location gs://YOUR_TEMP_DIRECTORY
   --job_name YOUR_JOB_NAME
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py b/sdks/python/apache_beam/io/gcp/bigquery_read_perf_test.py
@@ -33,6 +33,7 @@
     --test-pipeline-options="
     --runner=TestDataflowRunner
     --project=...
+    --region=...
     --staging_location=gs://...
     --temp_location=gs://...
     --sdk_location=.../dist/apache-beam-x.x.x.dev0.tar.gz
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py b/sdks/python/apache_beam/io/gcp/bigquery_write_perf_test.py
@@ -31,6 +31,7 @@
     --test-pipeline-options="
     --runner=TestDataflowRunner
     --project=...
+    --region=...
     --staging_location=gs://...
     --temp_location=gs://...
     --sdk_location=.../dist/apache-beam-x.x.x.dev0.tar.gz
diff --git a/sdks/python/apache_beam/testing/benchmarks/nexmark/nexmark_launcher.py b/sdks/python/apache_beam/testing/benchmarks/nexmark/nexmark_launcher.py
@@ -47,6 +47,7 @@
       python nexmark_launcher.py \
           --query/q <query number> \
           --project <project id> \
+          --region <GCE region> \
           --loglevel=DEBUG (optional) \
           --wait_until_finish_duration <time_in_ms> \
           --streaming \
diff --git a/sdks/python/apache_beam/transforms/external_java.py b/sdks/python/apache_beam/transforms/external_java.py
@@ -95,6 +95,7 @@ def test_java_expansion_dataflow():
         pipeline_options = PipelineOptions([
             '--runner=DataflowRunner',
             '--project=dummyproject',
+            '--region=some-region1',
             '--experiments=beam_fn_api',
             '--temp_location=gs://dummybucket/'
         ])
diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh
@@ -130,6 +130,7 @@ python setup.py nosetests \
   --test-pipeline-options=" \
     --runner=TestDataflowRunner \
     --project=$PROJECT \
+    --region=$REGION \
     --worker_harness_container_image=$CONTAINER:$TAG \
     --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
     --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
diff --git a/website/src/contribute/release-guide.md b/website/src/contribute/release-guide.md
@@ -1185,6 +1185,7 @@ _Note_: -Prepourl and -Pver can be found in the RC vote email sent by Release Ma
     ```
     python -m apache_beam.examples.complete.game.leader_board \ 
     --project=${YOUR_PROJECT} \ 
+    --region=${GCE_REGION} \
     --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ 
     --dataset ${USER}_test \ 
     --runner DataflowRunner \ 
@@ -1214,6 +1215,7 @@ _Note_: -Prepourl and -Pver can be found in the RC vote email sent by Release Ma
     ```
     python -m apache_beam.examples.complete.game.game_stats \ 
     --project=${YOUR_PROJECT} \ 
+    --region=${GCE_REGION} \
     --topic projects/${YOUR_PROJECT}/topics/${YOUR_PUBSUB_TOPIC} \ 
     --dataset ${USER}_test \ 
     --runner DataflowRunner \ 
diff --git a/website/src/documentation/runtime/environments.md b/website/src/documentation/runtime/environments.md
@@ -109,6 +109,7 @@ python -m apache_beam.examples.wordcount \
 --output=/path/to/write/counts \
 --runner=DataflowRunner \
 --project={gcp_project_id} \
+--region={gce_region_id} \
 --temp_location={gcs_location} \ \
 --experiment=beam_fn_api \
 --sdk_location=[…]/beam/sdks/python/container/py{version}/build/target/apache-beam.tar.gz \
diff --git a/website/src/documentation/sdks/python-streaming.md b/website/src/documentation/sdks/python-streaming.md
@@ -158,6 +158,7 @@ pip install apache-beam[gcp]
 python -m apache_beam.examples.streaming_wordcount \
   --runner DataflowRunner \
   --project YOUR_GCP_PROJECT \
+  --region YOUR_GCP_REGION \
   --temp_location gs://YOUR_GCS_BUCKET/tmp/ \
   --input_topic "projects/YOUR_PUBSUB_PROJECT_NAME/topics/YOUR_INPUT_TOPIC" \
   --output_topic "projects/YOUR_PUBSUB_PROJECT_NAME/topics/YOUR_OUTPUT_TOPIC" \
diff --git a/website/src/get-started/quickstart-py.md b/website/src/get-started/quickstart-py.md
@@ -203,6 +203,7 @@ python -m apache_beam.examples.wordcount --input gs://dataflow-samples/shakespea
                                          --output gs://<your-gcs-bucket>/counts \
                                          --runner DataflowRunner \
                                          --project your-gcp-project \
+                                         --region your-gcp-region \
                                          --temp_location gs://<your-gcs-bucket>/tmp/
 ```
 
