Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e164d17

Browse files
author
Tobiasz Kędzierski
committed
[BEAM-10682] Add workflow to run Java tests on Linux/Windows/Mac
1 parent cfa448d commit e164d17

19 files changed

Lines changed: 289 additions & 6 deletions

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,6 @@ GitHub Actions Tests Status (on master branch)
3636
------------------------------------------------------------------------------------------------
3737
![Build python source distribution and wheels](https://github.com/apache/beam/workflows/Build%20python%20source%20distribution%20and%20wheels/badge.svg)
3838
![Python tests](https://github.com/apache/beam/workflows/Python%20tests/badge.svg)
39+
![Java tests](https://github.com/apache/beam/workflows/Java%20Tests/badge.svg)
3940

4041
See [CI.md](https://github.com/apache/beam/blob/master/CI.md) for more information about GitHub Actions CI.

‎.github/workflows/cancel.yml‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,17 @@
1616
# under the License.
1717

1818
name: Cancel
19+
1920
on: [push, pull_request]
21+
2022
jobs:
21-
cancel:
22-
name: 'Cancel Previous Runs'
23+
24+
cancel_build_wheels:
25+
name: 'Cancel Previous Runs of python source distribution and wheels build'
2326
runs-on: ubuntu-latest
2427
timeout-minutes: 3
2528
steps:
26-
- name: Get workflow id
29+
- name: Get build_wheels workflow id
2730
run: |
2831
WORKFLOW_ID=$(curl "Authorization: token ${{ github.token }}" https://api.github.com/repos/${{ github.repository }}/actions/workflows/build_wheels.yml | jq '.id')
2932
echo "Workflow id: ${WORKFLOW_ID}"
@@ -32,3 +35,18 @@ jobs:
3235
with:
3336
workflow_id: ${{ env.WORKFLOW_ID }}
3437
access_token: ${{ github.token }}
38+
39+
cancel_java_tests:
40+
name: 'Cancel Previous Runs of Java Tests'
41+
runs-on: ubuntu-latest
42+
timeout-minutes: 3
43+
steps:
44+
- name: Get java_tests workflow id
45+
run: |
46+
WORKFLOW_ID=$(curl "Authorization: token ${{ github.token }}" https://api.github.com/repos/${{ github.repository }}/actions/workflows/java_tests.yml | jq '.id')
47+
echo "Workflow id: ${WORKFLOW_ID}"
48+
echo "::set-env name=WORKFLOW_ID::${WORKFLOW_ID}"
49+
- uses: styfle/[email protected]
50+
with:
51+
workflow_id: ${{ env.WORKFLOW_ID }}
52+
access_token: ${{ github.token }}

‎.github/workflows/java_tests.yml‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# To learn more about GitHub Actions in Apache Beam check the CI.md
19+
20+
name: Java Tests
21+
22+
on:
23+
workflow_dispatch:
24+
inputs:
25+
runDataflow:
26+
description: 'Type "true" if you want to run Dataflow tests (GCP variables must be configured, check CI.md)'
27+
default: false
28+
schedule:
29+
- cron: '10 2 * * *'
30+
push:
31+
branches: ['master', 'release-*']
32+
tags: 'v*'
33+
pull_request:
34+
branches: ['master', 'release-*']
35+
tags: 'v*'
36+
paths: ['sdks/java/**', 'model/**', 'runners/**', 'examples/java/**', 'examples/kotlin/**', 'release/**']
37+
38+
39+
jobs:
40+
41+
check_gcp_variables:
42+
timeout-minutes: 5
43+
name: "Check GCP variables set"
44+
runs-on: ubuntu-latest
45+
outputs:
46+
gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
47+
steps:
48+
- uses: actions/checkout@v2
49+
- name: "Check are GCP variables set"
50+
run: "./scripts/ci/ci_check_are_gcp_variables_set.sh"
51+
id: check_gcp_variables
52+
env:
53+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
54+
GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }}
55+
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
56+
GCP_TESTING_BUCKET: ${{ secrets.GCP_TESTING_BUCKET }}
57+
GCP_REGION: "not-needed-here"
58+
GCP_PYTHON_WHEELS_BUCKET: "not-needed-here"
59+
60+
java_unit_tests:
61+
name: 'Java Unit Tests'
62+
runs-on: ${{ matrix.os }}
63+
strategy:
64+
fail-fast: false
65+
matrix:
66+
os: [ubuntu-latest, macos-latest, windows-latest]
67+
steps:
68+
- name: Checkout code
69+
uses: actions/checkout@v2
70+
# :sdks:java:core:test
71+
- name: Run :sdks:java:core:test
72+
run: ./gradlew :sdks:java:core:test
73+
- name: Upload test logs for :sdks:java:core:test
74+
uses: actions/upload-artifact@v2
75+
if: always()
76+
with:
77+
name: java_unit_tests-sdks-java-core-test-${{ matrix.os }}
78+
path: sdks/java/core/build/reports/tests/test
79+
# :sdks:java:harness:test
80+
- name: Run :sdks:java:harness:test
81+
if: always()
82+
run: ./gradlew :sdks:java:harness:test
83+
- name: Upload test logs for :sdks:java:harness:test
84+
uses: actions/upload-artifact@v2
85+
if: always()
86+
with:
87+
name: java_unit_tests-sdks-java-harness-test-${{ matrix.os }}
88+
path: sdks/java/harness/build/reports/tests/test
89+
# :runners:core-java:test
90+
- name: Run :runners:core-java:test
91+
if: always()
92+
run: ./gradlew :runners:core-java:test
93+
- name: Upload test logs for :runners:core-java:test
94+
uses: actions/upload-artifact@v2
95+
if: always()
96+
with:
97+
name: java_unit_tests-runners-core-java-test-${{ matrix.os }}
98+
path: runners/core-java/build/reports/tests/test
99+
100+
java_wordcount_direct_runner:
101+
name: 'Java Wordcount Direct Runner'
102+
runs-on: ${{ matrix.os }}
103+
strategy:
104+
fail-fast: false
105+
matrix:
106+
os: [ubuntu-latest, macos-latest, windows-latest]
107+
steps:
108+
- name: Checkout code
109+
uses: actions/checkout@v2
110+
- name: Run WordCount Unix
111+
shell: bash
112+
run: |
113+
bash ./gradlew -p examples/ \
114+
integrationTest -DintegrationTestPipelineOptions='[
115+
"--runner=DirectRunner", "--tempRoot=./tmp" ]' \
116+
--tests org.apache.beam.examples.WordCountIT \
117+
-DintegrationTestRunner=direct
118+
- name: Upload test logs
119+
uses: actions/upload-artifact@v2
120+
if: always()
121+
with:
122+
name: java_wordcount_direct_runner-${{matrix.os}}
123+
path: examples/java/build/reports/tests/integrationTest
124+
125+
java_wordcount_dataflow:
126+
name: 'Java Wordcount Dataflow'
127+
needs:
128+
- check_gcp_variables
129+
runs-on: ${{ matrix.os }}
130+
if: |
131+
needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && (
132+
(github.event_name == 'push' || github.event_name == 'schedule') ||
133+
(github.event_name == 'workflow_dispatch' && github.event.inputs.runDataflow == 'true')
134+
)
135+
strategy:
136+
fail-fast: false
137+
matrix:
138+
os: [ubuntu-latest, macos-latest, windows-latest]
139+
steps:
140+
- name: Checkout code
141+
uses: actions/checkout@v2
142+
- name: Authenticate on GCP
143+
uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
144+
with:
145+
service_account_email: ${{ secrets.GCP_SA_EMAIL }}
146+
service_account_key: ${{ secrets.GCP_SA_KEY }}
147+
project_id: ${{ secrets.GCP_PROJECT_ID }}
148+
export_default_credentials: true
149+
- name: Run WordCount
150+
shell: bash
151+
run: |
152+
bash ./gradlew -p examples/ \
153+
integrationTest -DintegrationTestPipelineOptions='[
154+
"--runner=DataflowRunner", "--project=${{ secrets.GCP_PROJECT_ID }}", "--tempRoot=gs://${{ secrets.GCP_TESTING_BUCKET }}/tmp/"]' \
155+
--tests org.apache.beam.examples.WordCountIT \
156+
-DintegrationTestRunner=dataflow
157+
- name: Upload test logs
158+
uses: actions/upload-artifact@v2
159+
if: always()
160+
with:
161+
name: java_wordcount_dataflow-${{matrix.os}}
162+
path: examples/java/build/reports/tests/integrationTest

‎CI.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,15 @@ Service Account shall have following permissions ([IAM roles](https://cloud.goog
116116
| Python Wordcount Direct Runner | Runs python WordCount example with Direct Runner. | Yes | Yes | Yes | - |
117117
| Python Wordcount Dataflow | Runs python WordCount example with DataFlow Runner. | - | Yes | Yes | Yes |
118118

119+
#### Java tests - [java_tests.yml](.github/workflows/java_tests.yml)
120+
121+
| Job | Description | Pull Request Run | Direct Push/Merge Run | Scheduled Run | Requires GCP Credentials |
122+
|------------------------------|-----------------------------------------------------------------------------------------------|------------------|-----------------------|---------------|--------------------------|
123+
| Check GCP variables | Checks that GCP variables are set. Jobs which required them depend on the output of this job. | Yes | Yes | Yes | Yes/No |
124+
| Java Unit Tests | Runs Java unit tests. | Yes | Yes | Yes | - |
125+
| Java Wordcount Direct Runner | Runs Java WordCount example with Direct Runner. | Yes | Yes | Yes | - |
126+
| Java Wordcount Dataflow | Runs Java WordCount example with DataFlow Runner. | - | Yes | Yes | Yes |
127+
119128
### GitHub Action Tips
120129

121130
* If you introduce changes to the workflow it is possible that your changes will not be present in the check run triggered in Pull Request.

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
[![Compat Check at master](https://python-compatibility-tools.appspot.com/one_badge_image?package=git%2Bgit%3A//github.com/apache/beam.git%23subdirectory%3Dsdks/python)](https://python-compatibility-tools.appspot.com/one_badge_target?package=git%2Bgit%3A//github.com/apache/beam.git%23subdirectory%3Dsdks/python)
3232
![Build python source distribution and wheels](https://github.com/apache/beam/workflows/Build%20python%20source%20distribution%20and%20wheels/badge.svg)
3333
![Python tests](https://github.com/apache/beam/workflows/Python%20tests/badge.svg)
34+
![Java Tests](https://github.com/apache/beam/workflows/Java%20Tests/badge.svg)
3435

3536
### Post-commit tests status (on master branch)
3637

‎buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,9 +1377,15 @@ class BeamModulePlugin implements Plugin<Project> {
13771377
include "**/*IT.class"
13781378

13791379
def pipelineOptionsString = configuration.integrationTestPipelineOptions
1380+
def pipelineOptionsStringFormatted
1381+
def allOptionsList
1382+
1383+
if(pipelineOptionsString) {
1384+
allOptionsList = (new JsonSlurper()).parseText(pipelineOptionsString)
1385+
}
1386+
13801387
if(pipelineOptionsString && configuration.runner?.equalsIgnoreCase('dataflow')) {
13811388
project.evaluationDependsOn(":runners:google-cloud-dataflow-java:worker:legacy-worker")
1382-
def allOptionsList = (new JsonSlurper()).parseText(pipelineOptionsString)
13831389
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
13841390
project.project(":runners:google-cloud-dataflow-java:worker:legacy-worker").shadowJar.archivePath
13851391
def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1'
@@ -1388,11 +1394,17 @@ class BeamModulePlugin implements Plugin<Project> {
13881394
"--dataflowWorkerJar=${dataflowWorkerJar}",
13891395
"--region=${dataflowRegion}"
13901396
])
1397+
}
13911398

1392-
pipelineOptionsString = JsonOutput.toJson(allOptionsList)
1399+
// Windows handles quotation marks differently
1400+
if (pipelineOptionsString && System.properties['os.name'].toLowerCase().contains('windows')) {
1401+
def allOptionsListFormatted = allOptionsList.collect{ "\"$it\"" }
1402+
pipelineOptionsStringFormatted = JsonOutput.toJson(allOptionsListFormatted)
1403+
} else if (pipelineOptionsString) {
1404+
pipelineOptionsStringFormatted = JsonOutput.toJson(allOptionsList)
13931405
}
13941406

1395-
systemProperties.beamTestPipelineOptions = pipelineOptionsString
1407+
systemProperties.beamTestPipelineOptions = pipelineOptionsStringFormatted ?: pipelineOptionsString
13961408
}
13971409
}
13981410

‎sdks/java/core/src/test/java/org/apache/beam/sdk/io/DefaultFilenamePolicyTest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@
1818
package org.apache.beam.sdk.io;
1919

2020
import static org.junit.Assert.assertEquals;
21+
import static org.junit.Assume.assumeFalse;
2122

2223
import java.io.IOException;
2324
import org.apache.beam.sdk.io.fs.ResourceId;
2425
import org.apache.beam.sdk.util.CoderUtils;
26+
import org.apache.commons.lang3.SystemUtils;
27+
import org.junit.Before;
2528
import org.junit.Test;
2629
import org.junit.runner.RunWith;
2730
import org.junit.runners.JUnit4;
@@ -30,6 +33,12 @@
3033
@RunWith(JUnit4.class)
3134
public class DefaultFilenamePolicyTest {
3235

36+
@Before
37+
public void setup() {
38+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10738
39+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
40+
}
41+
3342
private static String constructName(
3443
String baseFilename,
3544
String shardTemplate,

‎sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileBasedSinkTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import static org.junit.Assert.assertThat;
2727
import static org.junit.Assert.assertTrue;
2828
import static org.junit.Assert.fail;
29+
import static org.junit.Assume.assumeFalse;
2930

3031
import java.io.BufferedReader;
3132
import java.io.BufferedWriter;
@@ -63,6 +64,7 @@
6364
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Sets;
6465
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
6566
import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream;
67+
import org.apache.commons.lang3.SystemUtils;
6668
import org.junit.Rule;
6769
import org.junit.Test;
6870
import org.junit.rules.TemporaryFolder;
@@ -166,13 +168,17 @@ public void testRemoveWithTempFilename() throws Exception {
166168
/** Finalize copies temporary files to output files and removes any temporary files. */
167169
@Test
168170
public void testFinalize() throws Exception {
171+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10743
172+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
169173
List<File> files = generateTemporaryFilesForFinalize(3);
170174
runFinalize(buildWriteOperation(), files);
171175
}
172176

173177
/** Finalize can be called repeatedly. */
174178
@Test
175179
public void testFinalizeMultipleCalls() throws Exception {
180+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10744
181+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
176182
List<File> files = generateTemporaryFilesForFinalize(3);
177183
SimpleSink.SimpleWriteOperation writeOp = buildWriteOperation();
178184
runFinalize(writeOp, files);
@@ -182,6 +188,8 @@ public void testFinalizeMultipleCalls() throws Exception {
182188
/** Finalize can be called when some temporary files do not exist and output files exist. */
183189
@Test
184190
public void testFinalizeWithIntermediateState() throws Exception {
191+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10745
192+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
185193
SimpleSink.SimpleWriteOperation writeOp = buildWriteOperation();
186194
List<File> files = generateTemporaryFilesForFinalize(3);
187195
runFinalize(writeOp, files);

‎sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static org.junit.Assert.assertFalse;
2323
import static org.junit.Assert.assertThat;
2424
import static org.junit.Assert.assertTrue;
25+
import static org.junit.Assume.assumeFalse;
2526

2627
import java.io.Writer;
2728
import java.nio.channels.Channels;
@@ -57,6 +58,8 @@ public class FileSystemsTest {
5758

5859
@Test
5960
public void testGetLocalFileSystem() throws Exception {
61+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10740
62+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
6063
assertTrue(
6164
FileSystems.getFileSystemInternal(toLocalResourceId("~/home/").getScheme())
6265
instanceof LocalFileSystem);

‎sdks/java/core/src/test/java/org/apache/beam/sdk/io/LocalFileSystemTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static org.junit.Assert.assertThat;
2626
import static org.junit.Assert.assertThrows;
2727
import static org.junit.Assert.assertTrue;
28+
import static org.junit.Assume.assumeFalse;
2829

2930
import java.io.File;
3031
import java.io.FileNotFoundException;
@@ -407,6 +408,8 @@ public void testMatchWithDirectoryFiltersOutDirectory() throws Exception {
407408

408409
@Test
409410
public void testMatchWithoutParentDirectory() throws Exception {
411+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10741
412+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
410413
Path pattern =
411414
LocalResourceId.fromPath(temporaryFolder.getRoot().toPath(), true /* isDirectory */)
412415
.resolve("non_existing_dir", StandardResolveOptions.RESOLVE_DIRECTORY)
@@ -417,6 +420,8 @@ public void testMatchWithoutParentDirectory() throws Exception {
417420

418421
@Test
419422
public void testMatchNewResource() {
423+
// TODO: Java core test failing on windows, https://issues.apache.org/jira/browse/BEAM-10742
424+
assumeFalse(SystemUtils.IS_OS_WINDOWS);
420425
LocalResourceId fileResource =
421426
localFileSystem.matchNewResource("/some/test/resource/path", false /* isDirectory */);
422427
LocalResourceId dirResource =

0 commit comments

Comments
 (0)