diff --git a/.kokoro/build.sh b/.kokoro/build.sh index d4f07415d9..c1c6a48ff7 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -74,6 +74,11 @@ graalvm) mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test RETURN_CODE=$? ;; +graalvm17) + # Run Unit and Integration Tests with Native Image + mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test + RETURN_CODE=$? + ;; samples) SAMPLES_DIR=samples # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg new file mode 100644 index 0000000000..833f9269bb --- /dev/null +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -0,0 +1,27 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17" +} + +env_vars: { + key: "JOB_TYPE" + value: "graalvm17" +} + +env_vars: { + key: "GCLOUD_PROJECT" + value: "java-review" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-review_firestore-java-it" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-review_firestore-java-it" +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index b7344160d2..0515e05a24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [3.3.0](https://github.com/googleapis/java-firestore/compare/v3.2.0...v3.3.0) (2022-07-01) + + +### Features + +* add logical termination for RunQueryResponse ([#956](https://github.com/googleapis/java-firestore/issues/956)) ([1d869c8](https://github.com/googleapis/java-firestore/commit/1d869c8fbe5e4009bbded2fb24f978f7cf48c515)) + + +### Bug Fixes + +* add build script for JDK 17 native image testing ([#965](https://github.com/googleapis/java-firestore/issues/965)) ([963e384](https://github.com/googleapis/java-firestore/commit/963e384b254056b6879744ba0b5d69fc0699f0bf)) + + +### Performance Improvements + +* Change how proto was called in Query.java ([#970](https://github.com/googleapis/java-firestore/issues/970)) ([f6f6352](https://github.com/googleapis/java-firestore/commit/f6f63520e93d9b3db0a915be58ea3cd37cec68d2)) + + +### Dependencies + +* update beam.version to v2.40.0 ([#978](https://github.com/googleapis/java-firestore/issues/978)) ([44276f8](https://github.com/googleapis/java-firestore/commit/44276f84ac927e02105608d81bb198f45793f823)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v2.13.0 ([#974](https://github.com/googleapis/java-firestore/issues/974)) ([6def73d](https://github.com/googleapis/java-firestore/commit/6def73dbdbdc645d1148b9cc8e390e307a6896d4)) +* update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.12 ([#973](https://github.com/googleapis/java-firestore/issues/973)) ([bfb4ec9](https://github.com/googleapis/java-firestore/commit/bfb4ec9f9de6114259328bbe9a14c0cc7fae84c8)) + ## [3.2.0](https://github.com/googleapis/java-firestore/compare/v3.1.0...v3.2.0) (2022-05-23) diff --git a/README.md b/README.md index 7ad8136003..970c84fb8f 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-firestore - 3.1.0 + 3.2.0 ``` @@ -49,20 +49,20 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:25.3.0') +implementation platform('com.google.cloud:libraries-bom:25.4.0') implementation 'com.google.cloud:google-cloud-firestore' ``` If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-firestore:3.1.0' +implementation 'com.google.cloud:google-cloud-firestore:3.2.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-firestore" % "3.1.0" +libraryDependencies += "com.google.cloud" % "google-cloud-firestore" % "3.2.0" ``` ## Authentication @@ -109,6 +109,8 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-firestore/tre | Native Image Firestore Sample | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/native-image-sample/src/main/java/com/example/firestore/NativeImageFirestoreSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/native-image-sample/src/main/java/com/example/firestore/NativeImageFirestoreSample.java) | | Person | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/native-image-sample/src/main/java/com/example/firestore/Person.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/native-image-sample/src/main/java/com/example/firestore/Person.java) | | Quickstart | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/Quickstart.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/Quickstart.java) | +| Example Firestore Beam Read | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java) | +| Example Firestore Beam Write | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java) | | Listen Data Snippets | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/snippets/ListenDataSnippets.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/snippets/ListenDataSnippets.java) | | Manage Data Snippets | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/snippets/ManageDataSnippets.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/snippets/ManageDataSnippets.java) | | Query Data Snippets | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/snippets/QueryDataSnippets.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/snippets/QueryDataSnippets.java) | diff --git a/google-cloud-firestore-admin/pom.xml b/google-cloud-firestore-admin/pom.xml index 717ccf5ea9..1e857e4661 100644 --- a/google-cloud-firestore-admin/pom.xml +++ b/google-cloud-firestore-admin/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 google-cloud-firestore-admin - 3.2.0 + 3.3.0 jar Google Cloud Firestore Admin Client https://github.com/googleapis/java-firestore @@ -14,7 +14,7 @@ com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 diff --git a/google-cloud-firestore-bom/pom.xml b/google-cloud-firestore-bom/pom.xml index a832fe8aff..f7541447a2 100644 --- a/google-cloud-firestore-bom/pom.xml +++ b/google-cloud-firestore-bom/pom.xml @@ -3,12 +3,12 @@ 4.0.0 com.google.cloud google-cloud-firestore-bom - 3.2.0 + 3.3.0 pom com.google.cloud google-cloud-shared-config - 1.4.0 + 1.5.0 Google Cloud Firestore BOM @@ -52,37 +52,37 @@ com.google.cloud google-cloud-firestore - 3.2.0 + 3.3.0 com.google.cloud google-cloud-firestore-admin - 3.2.0 + 3.3.0 com.google.api.grpc grpc-google-cloud-firestore-admin-v1 - 3.2.0 + 3.3.0 com.google.api.grpc grpc-google-cloud-firestore-v1 - 3.2.0 + 3.3.0 com.google.api.grpc proto-google-cloud-firestore-admin-v1 - 3.2.0 + 3.3.0 com.google.api.grpc proto-google-cloud-firestore-v1 - 3.2.0 + 3.3.0 com.google.cloud proto-google-cloud-firestore-bundle-v1 - 3.2.0 + 3.3.0 diff --git a/google-cloud-firestore/pom.xml b/google-cloud-firestore/pom.xml index 25a2a2202d..b2c5ed5b0f 100644 --- a/google-cloud-firestore/pom.xml +++ b/google-cloud-firestore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-firestore - 3.2.0 + 3.3.0 jar Google Cloud Firestore https://github.com/googleapis/java-firestore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 google-cloud-firestore @@ -140,7 +140,7 @@ com.google.cloud google-cloud-conformance-tests - 0.2.7 + 0.3.0 test diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Query.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Query.java index c8dac33964..048f5ca695 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Query.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Query.java @@ -1330,6 +1330,8 @@ public void stream(@Nonnull final ApiStreamObserver responseOb internalStream( new QuerySnapshotObserver() { + boolean hasCompleted = false; + @Override public void onNext(QueryDocumentSnapshot documentSnapshot) { responseObserver.onNext(documentSnapshot); @@ -1342,6 +1344,8 @@ public void onError(Throwable throwable) { @Override public void onCompleted() { + if (hasCompleted) return; + hasCompleted = true; responseObserver.onCompleted(); } }, @@ -1537,6 +1541,16 @@ public void onResponse(RunQueryResponse response) { if (readTime == null) { readTime = Timestamp.fromProto(response.getReadTime()); } + + if (response.getDone()) { + Tracing.getTracer() + .getCurrentSpan() + .addAnnotation( + "Firestore.Query: Completed", + ImmutableMap.of( + "numDocuments", AttributeValue.longAttributeValue(numDocuments))); + documentObserver.onCompleted(readTime); + } } @Override @@ -1640,6 +1654,9 @@ ApiFuture get(@Nullable ByteString transactionId) { internalStream( new QuerySnapshotObserver() { final List documentSnapshots = new ArrayList<>(); + // The stream's onCompleted could be called more than once, + // this flag makes sure only the first one is actually processed. + boolean hasCompleted = false; @Override public void onNext(QueryDocumentSnapshot documentSnapshot) { @@ -1653,6 +1670,9 @@ public void onError(Throwable throwable) { @Override public void onCompleted() { + if (hasCompleted) return; + hasCompleted = true; + // The results for limitToLast queries need to be flipped since we reversed the // ordering constraints before sending the query to the backend. List resultView = diff --git a/google-cloud-firestore/src/test/java/com/google/cloud/firestore/LocalFirestoreHelper.java b/google-cloud-firestore/src/test/java/com/google/cloud/firestore/LocalFirestoreHelper.java index e73241a002..38f853e7e8 100644 --- a/google-cloud-firestore/src/test/java/com/google/cloud/firestore/LocalFirestoreHelper.java +++ b/google-cloud-firestore/src/test/java/com/google/cloud/firestore/LocalFirestoreHelper.java @@ -311,6 +311,29 @@ public static Answer queryResponse( return streamingResponse(responses, throwable); } + /** Returns a stream of responses when RunQueryResponse.done set to true */ + public static Answer queryResponseWithDone( + boolean callWithoutOnComplete, String... documentNames) { + RunQueryResponse[] responses = new RunQueryResponse[documentNames.length]; + + for (int i = 0; i < documentNames.length; ++i) { + final RunQueryResponse.Builder runQueryResponse = RunQueryResponse.newBuilder(); + runQueryResponse.setDocument( + Document.newBuilder().setName(documentNames[i]).putAllFields(SINGLE_FIELD_PROTO)); + runQueryResponse.setReadTime( + com.google.protobuf.Timestamp.newBuilder().setSeconds(1).setNanos(2)); + if (i == (documentNames.length - 1)) { + runQueryResponse.setDone(true); + } + responses[i] = runQueryResponse.build(); + } + if (callWithoutOnComplete) { + return streamingResponseWithoutOnComplete(responses); + } else { + return streamingResponse(responses, null); + } + } + /** Returns a stream of responses followed by an optional exception. */ public static Answer streamingResponse( final T[] response, @Nullable final Throwable throwable) { @@ -328,6 +351,18 @@ public static Answer streamingResponse( }; } + /** Returns a stream of responses even though onComplete() wasn't triggered */ + public static Answer streamingResponseWithoutOnComplete(final T[] response) { + return invocation -> { + Object[] args = invocation.getArguments(); + ResponseObserver observer = (ResponseObserver) args[1]; + for (T resp : response) { + observer.onResponse(resp); + } + return null; + }; + } + public static ApiFuture commitResponse(int adds, int deletes) { CommitResponse.Builder commitResponse = CommitResponse.newBuilder(); commitResponse.getCommitTimeBuilder().setSeconds(0).setNanos(0); diff --git a/google-cloud-firestore/src/test/java/com/google/cloud/firestore/QueryTest.java b/google-cloud-firestore/src/test/java/com/google/cloud/firestore/QueryTest.java index 90520e3c5c..061d345f6d 100644 --- a/google-cloud-firestore/src/test/java/com/google/cloud/firestore/QueryTest.java +++ b/google-cloud-firestore/src/test/java/com/google/cloud/firestore/QueryTest.java @@ -27,6 +27,7 @@ import static com.google.cloud.firestore.LocalFirestoreHelper.order; import static com.google.cloud.firestore.LocalFirestoreHelper.query; import static com.google.cloud.firestore.LocalFirestoreHelper.queryResponse; +import static com.google.cloud.firestore.LocalFirestoreHelper.queryResponseWithDone; import static com.google.cloud.firestore.LocalFirestoreHelper.reference; import static com.google.cloud.firestore.LocalFirestoreHelper.select; import static com.google.cloud.firestore.LocalFirestoreHelper.startAt; @@ -952,6 +953,86 @@ public void onCompleted() { semaphore.acquire(); } + @Test + public void successfulReturnWithoutOnComplete() throws Exception { + doAnswer( + queryResponseWithDone( + /* callWithoutOnComplete */ true, DOCUMENT_NAME + "1", DOCUMENT_NAME + "2")) + .when(firestoreMock) + .streamRequest( + runQuery.capture(), + streamObserverCapture.capture(), + Matchers.any()); + + final Semaphore semaphore = new Semaphore(0); + final Iterator iterator = Arrays.asList("doc1", "doc2").iterator(); + + query.stream( + new ApiStreamObserver() { + @Override + public void onNext(DocumentSnapshot documentSnapshot) { + assertEquals(iterator.next(), documentSnapshot.getId()); + } + + @Override + public void onError(Throwable throwable) { + fail(); + } + + @Override + public void onCompleted() { + semaphore.release(); + } + }); + + semaphore.acquire(); + } + + @Test + /** + * onComplete() will be called twice. The first time is when it detects RunQueryResponse.done set + * to true. The second time is when it receives half close + */ + public void successfulReturnCallsOnCompleteTwice() throws Exception { + doAnswer( + queryResponseWithDone( + /* callWithoutOnComplete */ false, DOCUMENT_NAME + "1", DOCUMENT_NAME + "2")) + .when(firestoreMock) + .streamRequest( + runQuery.capture(), + streamObserverCapture.capture(), + Matchers.any()); + + final Semaphore semaphore = new Semaphore(0); + final Iterator iterator = Arrays.asList("doc1", "doc2").iterator(); + final int[] counter = {0}; + + query.stream( + new ApiStreamObserver() { + @Override + public void onNext(DocumentSnapshot documentSnapshot) { + assertEquals(iterator.next(), documentSnapshot.getId()); + } + + @Override + public void onError(Throwable throwable) { + fail(); + } + + @Override + public void onCompleted() { + counter[0]++; + semaphore.release(); + } + }); + + semaphore.acquire(); + + // Wait for some time to see whether onCompleted() has been called more than once + Thread.sleep(200); + assertEquals(1, counter[0]); + } + @Test public void retriesAfterRetryableError() throws Exception { final boolean[] returnError = new boolean[] {true}; diff --git a/grpc-google-cloud-firestore-admin-v1/pom.xml b/grpc-google-cloud-firestore-admin-v1/pom.xml index 5bc46de8be..d2ca7d66cf 100644 --- a/grpc-google-cloud-firestore-admin-v1/pom.xml +++ b/grpc-google-cloud-firestore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-firestore-admin-v1 - 3.2.0 + 3.3.0 grpc-google-cloud-firestore-admin-v1 GRPC library for grpc-google-cloud-firestore-admin-v1 com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 diff --git a/grpc-google-cloud-firestore-v1/pom.xml b/grpc-google-cloud-firestore-v1/pom.xml index 64ae75a2e9..3ad4355f2f 100644 --- a/grpc-google-cloud-firestore-v1/pom.xml +++ b/grpc-google-cloud-firestore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-firestore-v1 - 3.2.0 + 3.3.0 grpc-google-cloud-firestore-v1 GRPC library for grpc-google-cloud-firestore-v1 com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 diff --git a/owlbot.py b/owlbot.py index 37229a3067..55984884f6 100644 --- a/owlbot.py +++ b/owlbot.py @@ -83,5 +83,6 @@ '.kokoro/presubmit/samples.cfg', '.kokoro/nightly/integration.cfg', '.kokoro/nightly/java11-integration.cfg', - '.kokoro/nightly/samples.cfg' + '.kokoro/nightly/samples.cfg', + '.kokoro/build.sh' ]) \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3e2bd62799..d3d2f7e5bd 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-firestore-parent pom - 3.2.0 + 3.3.0 Google Cloud Firestore Parent https://github.com/googleapis/java-firestore @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.4.0 + 1.5.0 @@ -150,38 +150,38 @@ com.google.api.grpc proto-google-cloud-firestore-admin-v1 - 3.2.0 + 3.3.0 com.google.cloud google-cloud-firestore - 3.2.0 + 3.3.0 com.google.cloud proto-google-cloud-firestore-bundle-v1 - 3.2.0 + 3.3.0 com.google.api.grpc proto-google-cloud-firestore-v1 - 3.2.0 + 3.3.0 com.google.api.grpc grpc-google-cloud-firestore-admin-v1 - 3.2.0 + 3.3.0 com.google.api.grpc grpc-google-cloud-firestore-v1 - 3.2.0 + 3.3.0 com.google.cloud google-cloud-shared-dependencies - 2.12.0 + 2.13.0 pom import diff --git a/proto-google-cloud-firestore-admin-v1/pom.xml b/proto-google-cloud-firestore-admin-v1/pom.xml index 7d188d769a..aa08b6286e 100644 --- a/proto-google-cloud-firestore-admin-v1/pom.xml +++ b/proto-google-cloud-firestore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-firestore-admin-v1 - 3.2.0 + 3.3.0 proto-google-cloud-firestore-admin-v1 PROTO library for proto-google-cloud-firestore-admin-v1 com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 diff --git a/proto-google-cloud-firestore-bundle-v1/pom.xml b/proto-google-cloud-firestore-bundle-v1/pom.xml index 53b55413c2..2dbdaa1fdf 100644 --- a/proto-google-cloud-firestore-bundle-v1/pom.xml +++ b/proto-google-cloud-firestore-bundle-v1/pom.xml @@ -5,14 +5,14 @@ 4.0.0 proto-google-cloud-firestore-bundle-v1 - 3.2.0 + 3.3.0 proto-google-cloud-firestore-bundle-v1 PROTO library for proto-google-cloud-firestore-bundle-v1 com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 diff --git a/proto-google-cloud-firestore-v1/pom.xml b/proto-google-cloud-firestore-v1/pom.xml index 33c14f8b36..8267c91fb1 100644 --- a/proto-google-cloud-firestore-v1/pom.xml +++ b/proto-google-cloud-firestore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-firestore-v1 - 3.2.0 + 3.3.0 proto-google-cloud-firestore-v1 PROTO library for proto-google-cloud-firestore-v1 com.google.cloud google-cloud-firestore-parent - 3.2.0 + 3.3.0 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index d6b9914a2a..5d2e03afcf 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -20,6 +20,7 @@ 1.8 1.8 + 2.40.0 UTF-8 @@ -29,10 +30,22 @@ com.google.cloud google-cloud-firestore - 3.1.0 + 3.2.0 + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + junit junit diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index e7f3a9809e..39915e82a0 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -23,6 +23,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> 1.8 1.8 + 2.40.0 UTF-8 @@ -44,6 +45,18 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> google-cloud-firestore + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + junit junit @@ -110,7 +123,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.graalvm.buildtools native-maven-plugin - 0.9.11 + 0.9.12 true com.example.firestore.NativeImageFirestoreSample diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index ed3e78d9cc..60870c85ac 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -21,6 +21,7 @@ 1.8 1.8 UTF-8 + 2.40.0 @@ -28,7 +29,7 @@ com.google.cloud google-cloud-firestore - 3.1.0 + 3.2.0 @@ -43,9 +44,22 @@ 1.1.3 test - + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + + + diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index be7100ffac..8e42070c02 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -21,6 +21,7 @@ 1.8 1.8 UTF-8 + 2.40.0 @@ -47,6 +48,18 @@ + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + junit junit diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java new file mode 100644 index 0000000000..35b824b2a6 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java @@ -0,0 +1,135 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.firestore.beam; + +import com.google.cloud.firestore.FirestoreOptions; +import com.google.firestore.v1.DocumentRootName; +import com.google.firestore.v1.RunQueryRequest; +import com.google.firestore.v1.RunQueryResponse; +import com.google.firestore.v1.StructuredQuery; +import com.google.firestore.v1.Value; +import java.util.Objects; +import java.util.UUID; +import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; + +public class ExampleFirestoreBeamRead { + + public static void main(String[] args) { + runRead(args, "cities-collection-" + UUID.randomUUID().toString().substring(0, 10)); + } + + public static void runRead(String[] args, String collectionId) { + FirestoreOptions firestoreOptions = FirestoreOptions.getDefaultInstance(); + + PipelineOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); + Pipeline pipeline = Pipeline.create(options); + + RpcQosOptions rpcQosOptions = + RpcQosOptions.newBuilder() + .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) + .build(); + + pipeline + .apply(Create.of(collectionId)) + .apply( + new FilterDocumentsQuery( + firestoreOptions.getProjectId(), firestoreOptions.getDatabaseId())) + .apply(FirestoreIO.v1().read().runQuery().withRpcQosOptions(rpcQosOptions).build()) + .apply( + ParDo.of( + // transform each document to its name + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + c.output(Objects.requireNonNull(c.element()).getDocument().getName()); + } + })) + .apply( + ParDo.of( + // print the document name + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + System.out.println(c.element()); + } + })); + + pipeline.run().waitUntilFinish(); + } + + private static final class FilterDocumentsQuery + extends PTransform, PCollection> { + + private final String projectId; + private final String databaseId; + + public FilterDocumentsQuery(String projectId, String databaseId) { + this.projectId = projectId; + this.databaseId = databaseId; + } + + @Override + public PCollection expand(PCollection input) { + return input.apply( + ParDo.of( + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + // select from collection "cities-collection-" + StructuredQuery.CollectionSelector collection = + StructuredQuery.CollectionSelector.newBuilder() + .setCollectionId(Objects.requireNonNull(c.element())) + .build(); + // filter where country is equal to USA + StructuredQuery.Filter countryFilter = + StructuredQuery.Filter.newBuilder() + .setFieldFilter( + StructuredQuery.FieldFilter.newBuilder() + .setField( + StructuredQuery.FieldReference.newBuilder() + .setFieldPath("country") + .build()) + .setValue(Value.newBuilder().setStringValue("USA").build()) + .setOp(StructuredQuery.FieldFilter.Operator.EQUAL)) + .buildPartial(); + + RunQueryRequest runQueryRequest = + RunQueryRequest.newBuilder() + .setParent(DocumentRootName.format(projectId, databaseId)) + .setStructuredQuery( + StructuredQuery.newBuilder() + .addFrom(collection) + .setWhere(countryFilter) + .build()) + .build(); + c.output(runQueryRequest); + } + })); + } + } +} diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java new file mode 100644 index 0000000000..a4921c57e8 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java @@ -0,0 +1,92 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.firestore.beam; + +import com.google.cloud.firestore.FirestoreOptions; +import com.google.firestore.v1.Document; +import com.google.firestore.v1.Value; +import com.google.firestore.v1.Write; +import java.util.UUID; +import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Create; + +public class ExampleFirestoreBeamWrite { + private static final FirestoreOptions FIRESTORE_OPTIONS = FirestoreOptions.getDefaultInstance(); + + public static void main(String[] args) { + runWrite(args, "cities-collection-" + UUID.randomUUID().toString().substring(0, 10)); + } + + public static void runWrite(String[] args, String collectionId) { + // create pipeline options from the passed in arguments + PipelineOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); + Pipeline pipeline = Pipeline.create(options); + + RpcQosOptions rpcQosOptions = + RpcQosOptions.newBuilder() + .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) + .build(); + + // create some writes + Write write1 = + Write.newBuilder() + .setUpdate( + Document.newBuilder() + // resolves to + // projects//databases//documents//NYC + .setName(createDocumentName(collectionId, "NYC")) + .putFields("name", Value.newBuilder().setStringValue("New York City").build()) + .putFields("state", Value.newBuilder().setStringValue("New York").build()) + .putFields("country", Value.newBuilder().setStringValue("USA").build())) + .build(); + + Write write2 = + Write.newBuilder() + .setUpdate( + Document.newBuilder() + // resolves to + // projects//databases//documents//TOK + .setName(createDocumentName(collectionId, "TOK")) + .putFields("name", Value.newBuilder().setStringValue("Tokyo").build()) + .putFields("country", Value.newBuilder().setStringValue("Japan").build()) + .putFields("capital", Value.newBuilder().setBooleanValue(true).build())) + .build(); + + // batch write the data + pipeline + .apply(Create.of(write1, write2)) + .apply(FirestoreIO.v1().write().batchWrite().withRpcQosOptions(rpcQosOptions).build()); + + // run the pipeline + pipeline.run().waitUntilFinish(); + } + + private static String createDocumentName(String collectionId, String cityDocId) { + String documentPath = + String.format( + "projects/%s/databases/%s/documents", + FIRESTORE_OPTIONS.getProjectId(), FIRESTORE_OPTIONS.getDatabaseId()); + + return documentPath + "/" + collectionId + "/" + cityDocId; + } +} diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/README.md b/samples/snippets/src/main/java/com/example/firestore/beam/README.md new file mode 100644 index 0000000000..33b10f9392 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/firestore/beam/README.md @@ -0,0 +1,35 @@ +# Firestore Beam Samples + +## Running instructions +These examples will show you how to read and write data from/into a Firestore database using the Firestore Beam connector. + +1. Set up the following environment variables +``` +GOOGLE_CLOUD_PROJECT= +REGION= +TEMP_LOCATION=gs:///temp/ +NUM_WORKERS= +MAX_NUM_WORKERS= +``` +2. Run the command to write example "cities" data: +``` +mvn compile exec:java -Dexec.mainClass=ExampleFirestoreBeamWrite \ +"-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT \ +--runner=DataflowRunner \ +--project=$GOOGLE_CLOUD_PROJECT \ +--region=$REGION \ +--tempLocation=$TEMP_LOCATION \ +--numWorkers=$NUM_WORKERS \ +--maxNumWorkers=$MAX_NUM_WORKERS" +``` + +3. Run the command to filter and read the data you wrote: +``` +mvn compile exec:java -Dexec.mainClass=ExampleFirestoreBeamRead \ +"-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT \ +--runner=DataflowRunner \ +--region=$REGION \ +--tempLocation=$TEMP_LOCATION \ +--numWorkers=$NUM_WORKERS \ +--maxNumWorkers=$MAX_NUM_WORKERS" +``` \ No newline at end of file diff --git a/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java b/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java new file mode 100644 index 0000000000..bd94fcf817 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.firestore.beam; + +import com.google.api.core.ApiFuture; +import com.google.cloud.firestore.CollectionReference; +import com.google.cloud.firestore.Firestore; +import com.google.cloud.firestore.FirestoreOptions; +import com.google.cloud.firestore.QueryDocumentSnapshot; +import com.google.cloud.firestore.QuerySnapshot; +import com.google.common.truth.Truth; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ExampleFirestoreWriteReadTest { + private ByteArrayOutputStream bout; + private static String projectId; + private static Firestore firestore; + private static String collectionId; + + @BeforeClass + public static void setUp() { + collectionId = "cities-collection-" + UUID.randomUUID().toString().substring(0, 10); + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + firestore = FirestoreOptions.getDefaultInstance().getService(); + } + + @Before + public void setUpStream() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @AfterClass + public static void tearDown() { + deleteCollection(firestore.collection(collectionId), 1); + } + + @Test + public void testWriteAndRead() throws ExecutionException, InterruptedException { + String[] args = { + "--project=" + projectId, "--region=us-central1", "--numWorkers=1", "--maxNumWorkers=1" + }; + // write the data to Firestore + ExampleFirestoreBeamWrite.runWrite(args, collectionId); + + ApiFuture apiFuture = firestore.collection(collectionId).get(); + QuerySnapshot querySnapshot = apiFuture.get(); + List documents = querySnapshot.getDocuments(); + Truth.assertThat(documents).hasSize(2); + + // filter and read it back + ExampleFirestoreBeamRead.runRead(args, collectionId); + + String output = bout.toString(); + Truth.assertThat(output).contains("/documents/" + collectionId + "/NYC"); + } + + private static String requireEnv(String varName) { + String value = System.getenv(varName); + Assert.assertNotNull( + String.format("Environment variable '%s' is required to perform these tests.", varName), + value); + return value; + } + + // from: https://firebase.google.com/docs/firestore/manage-data/delete-data#collections + private static void deleteCollection(CollectionReference collection, int batchSize) { + try { + // retrieve a small batch of documents to avoid out-of-memory errors + ApiFuture future = collection.limit(batchSize).get(); + int deleted = 0; + // future.get() blocks on document retrieval + List documents = future.get().getDocuments(); + for (QueryDocumentSnapshot document : documents) { + document.getReference().delete(); + ++deleted; + } + if (deleted >= batchSize) { + // retrieve and delete another batch + deleteCollection(collection, batchSize); + } + } catch (Exception e) { + System.err.println("Error deleting collection : " + e.getMessage()); + } + } +} diff --git a/versions.txt b/versions.txt index a53560c856..6c445fcd37 100644 --- a/versions.txt +++ b/versions.txt @@ -1,11 +1,11 @@ # Format: # module:released-version:current-version -google-cloud-firestore:3.2.0:3.2.0 -google-cloud-firestore-admin:3.2.0:3.2.0 -google-cloud-firestore-bom:3.2.0:3.2.0 -grpc-google-cloud-firestore-admin-v1:3.2.0:3.2.0 -grpc-google-cloud-firestore-v1:3.2.0:3.2.0 -proto-google-cloud-firestore-admin-v1:3.2.0:3.2.0 -proto-google-cloud-firestore-v1:3.2.0:3.2.0 -proto-google-cloud-firestore-bundle-v1:3.2.0:3.2.0 +google-cloud-firestore:3.3.0:3.3.0 +google-cloud-firestore-admin:3.3.0:3.3.0 +google-cloud-firestore-bom:3.3.0:3.3.0 +grpc-google-cloud-firestore-admin-v1:3.3.0:3.3.0 +grpc-google-cloud-firestore-v1:3.3.0:3.3.0 +proto-google-cloud-firestore-admin-v1:3.3.0:3.3.0 +proto-google-cloud-firestore-v1:3.3.0:3.3.0 +proto-google-cloud-firestore-bundle-v1:3.3.0:3.3.0