From a618c8d2bf101ae3ba1c0d1d67bf9d4542762199 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Thu, 2 Jun 2022 13:31:51 -0400 Subject: [PATCH 1/7] samples: firestore beam connector examples --- README.md | 2 + samples/install-without-bom/pom.xml | 13 ++ samples/native-image-sample/pom.xml | 13 ++ samples/snapshot/pom.xml | 16 ++- samples/snippets/pom.xml | 13 ++ .../beam/ExampleFirestoreBeamRead.java | 128 ++++++++++++++++++ .../beam/ExampleFirestoreBeamWrite.java | 84 ++++++++++++ .../beam/ExampleFirestoreWriteReadTest.java | 106 +++++++++++++++ 8 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java create mode 100644 samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java create mode 100644 samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java diff --git a/README.md b/README.md index f9bc566350..a6e3695a9a 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-firestore/tre | Native Image Firestore Sample | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/native-image-sample/src/main/java/com/example/firestore/NativeImageFirestoreSample.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/native-image-sample/src/main/java/com/example/firestore/NativeImageFirestoreSample.java) | | Person | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/native-image-sample/src/main/java/com/example/firestore/Person.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/native-image-sample/src/main/java/com/example/firestore/Person.java) | | Quickstart | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/Quickstart.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/Quickstart.java) | +| Example Firestore Beam Read | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java) | +| Example Firestore Beam Write | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java) | | Listen Data Snippets | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/snippets/ListenDataSnippets.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/snippets/ListenDataSnippets.java) | | Manage Data Snippets | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/snippets/ManageDataSnippets.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/snippets/ManageDataSnippets.java) | | Query Data Snippets | [source code](https://github.com/googleapis/java-firestore/blob/main/samples/snippets/src/main/java/com/example/firestore/snippets/QueryDataSnippets.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-firestore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/firestore/snippets/QueryDataSnippets.java) | diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index a46b04ac94..16afb8d6ee 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -20,6 +20,7 @@ 1.8 1.8 + 2.38.0 UTF-8 @@ -33,6 +34,18 @@ + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + junit junit diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index e7f3a9809e..3001c9bd60 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -23,6 +23,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> 1.8 1.8 + 2.38.0 UTF-8 @@ -44,6 +45,18 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> google-cloud-firestore + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + junit junit diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index daf907a5a6..486d4d6d16 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -21,6 +21,7 @@ 1.8 1.8 UTF-8 + 2.38.0 @@ -43,9 +44,22 @@ 1.1.3 test - + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + + + diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index be7100ffac..b7525f9e37 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -21,6 +21,7 @@ 1.8 1.8 UTF-8 + 2.38.0 @@ -47,6 +48,18 @@ + + + org.apache.beam + beam-runners-direct-java + ${beam.version} + + + org.apache.beam + beam-runners-google-cloud-dataflow-java + ${beam.version} + + junit junit diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java new file mode 100644 index 0000000000..f5766081e1 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java @@ -0,0 +1,128 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.firestore.beam; + +import com.google.cloud.firestore.FirestoreOptions; +import com.google.firestore.v1.DocumentRootName; +import com.google.firestore.v1.RunQueryRequest; +import com.google.firestore.v1.RunQueryResponse; +import com.google.firestore.v1.StructuredQuery; +import com.google.firestore.v1.Value; +import java.util.Objects; +import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.transforms.ParDo; +import org.apache.beam.sdk.values.PCollection; + +public class ExampleFirestoreBeamRead { + public static void main(String[] args) { + FirestoreOptions firestoreOptions = FirestoreOptions.getDefaultInstance(); + + PipelineOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); + Pipeline pipeline = Pipeline.create(options); + + String collectionId = "cities-collection"; + RpcQosOptions rpcQosOptions = + RpcQosOptions.newBuilder() + .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) + .build(); + + pipeline + .apply(Create.of(collectionId)) + .apply( + new ReadDocumentsQuery( + firestoreOptions.getProjectId(), firestoreOptions.getDatabaseId())) + .apply(FirestoreIO.v1().read().runQuery().withRpcQosOptions(rpcQosOptions).build()) + .apply( + ParDo.of( + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + c.output(Objects.requireNonNull(c.element()).getDocument().getName()); + } + })) + .apply( + ParDo.of( + new DoFn() { + @ProcessElement + public void processElement(ProcessContext c) { + System.out.println(c.element()); + } + })); + + pipeline.run().waitUntilFinish(); + } + + private static final class ReadDocumentsQuery + extends PTransform, PCollection> { + + private final String projectId; + private final String databaseId; + + public ReadDocumentsQuery(String projectId, String databaseId) { + this.projectId = projectId; + this.databaseId = databaseId; + } + + @Override + public PCollection expand(PCollection input) { + return input.apply( + ParDo.of( + new DoFn() { + + @ProcessElement + public void processElement(ProcessContext c) { + RunQueryRequest runQueryRequest = + RunQueryRequest.newBuilder() + .setParent(DocumentRootName.format(projectId, databaseId)) + .setStructuredQuery( + StructuredQuery.newBuilder() + .addFrom( + StructuredQuery.CollectionSelector.newBuilder() + .setCollectionId(Objects.requireNonNull(c.element())) + .build()) + .setWhere( + StructuredQuery.Filter.newBuilder() + .setFieldFilter( + StructuredQuery.FieldFilter.newBuilder() + .setField( + StructuredQuery.FieldReference.newBuilder() + .setFieldPath("country") + .build()) + .setValue( + Value.newBuilder() + .setStringValue("USA") + .build()) + .setOp( + StructuredQuery.FieldFilter.Operator.EQUAL)) + .buildPartial()) + .build()) + .build(); + c.output(runQueryRequest); + } + })); + } + } +} diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java new file mode 100644 index 0000000000..bd69e3437b --- /dev/null +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java @@ -0,0 +1,84 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.firestore.beam; + +import com.google.cloud.firestore.FirestoreOptions; +import com.google.firestore.v1.Document; +import com.google.firestore.v1.Value; +import com.google.firestore.v1.Write; +import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; +import org.apache.beam.sdk.io.gcp.firestore.RpcQosOptions; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.Create; + +public class ExampleFirestoreBeamWrite { + private static final FirestoreOptions FIRESTORE_OPTIONS = FirestoreOptions.getDefaultInstance(); + + public static void main(String[] args) { + PipelineOptions options = + PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); + Pipeline pipeline = Pipeline.create(options); + + String collectionId = "cities-collection"; + RpcQosOptions rpcQosOptions = + RpcQosOptions.newBuilder() + .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) + .build(); + + Write write1 = + Write.newBuilder() + .setUpdate( + Document.newBuilder() + // resolves to + // projects//databases//documents/cities/NYC + .setName(createDocumentName(collectionId, "NYC")) + .putFields("name", Value.newBuilder().setStringValue("New York City").build()) + .putFields("state", Value.newBuilder().setStringValue("New York").build()) + .putFields("country", Value.newBuilder().setStringValue("USA").build())) + .build(); + + Write write2 = + Write.newBuilder() + .setUpdate( + Document.newBuilder() + // resolves to + // projects//databases//documents/cities/TOK + .setName(createDocumentName(collectionId, "TOK")) + .putFields("name", Value.newBuilder().setStringValue("Tokyo").build()) + .putFields("country", Value.newBuilder().setStringValue("Japan").build()) + .putFields("capital", Value.newBuilder().setBooleanValue(true).build())) + .build(); + + pipeline + .apply(Create.of(write1, write2)) + .apply(FirestoreIO.v1().write().batchWrite().withRpcQosOptions(rpcQosOptions).build()); + + pipeline.run().waitUntilFinish(); + } + + private static String createDocumentName(String collectionId, String cityDocId) { + String documentPath = + String.format( + "projects/%s/databases/%s/documents", + FIRESTORE_OPTIONS.getProjectId(), FIRESTORE_OPTIONS.getDatabaseId()); + + return documentPath + "/" + collectionId + "/" + cityDocId; + } +} diff --git a/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java b/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java new file mode 100644 index 0000000000..eb58244297 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java @@ -0,0 +1,106 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.firestore.beam; + +import com.google.api.core.ApiFuture; +import com.google.cloud.firestore.CollectionReference; +import com.google.cloud.firestore.Firestore; +import com.google.cloud.firestore.FirestoreOptions; +import com.google.cloud.firestore.QueryDocumentSnapshot; +import com.google.cloud.firestore.QuerySnapshot; +import com.google.common.truth.Truth; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.List; +import java.util.concurrent.ExecutionException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ExampleFirestoreWriteReadTest { + private ByteArrayOutputStream bout; + private static String projectId; + private static Firestore firestore; + + @BeforeClass + public static void setUp() { + projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); + firestore = FirestoreOptions.getDefaultInstance().getService(); + } + + @Before + public void setUpStream() { + bout = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bout)); + } + + @AfterClass + public static void tearDown() { + deleteCollection(firestore.collection("cities-collection"), 1); + } + + @Test + public void testWriteAndRead() throws ExecutionException, InterruptedException { + String[] args = { + "--project=" + projectId, "--region=us-central1", "--numWorkers=1", "--maxNumWorkers=1" + }; + // write the data to Firestore + ExampleFirestoreBeamWrite.main(args); + + ApiFuture apiFuture = firestore.collection("cities-collection").get(); + QuerySnapshot querySnapshot = apiFuture.get(); + List documents = querySnapshot.getDocuments(); + Truth.assertThat(documents).hasSize(2); + + // filter and read it back + ExampleFirestoreBeamRead.main(args); + + String output = bout.toString(); + Truth.assertThat(output).contains("/documents/cities-collection/NYC"); + } + + private static String requireEnv(String varName) { + String value = System.getenv(varName); + Assert.assertNotNull( + String.format("Environment variable '%s' is required to perform these tests.", varName), + value); + return value; + } + + // from: https://firebase.google.com/docs/firestore/manage-data/delete-data#collections + private static void deleteCollection(CollectionReference collection, int batchSize) { + try { + // retrieve a small batch of documents to avoid out-of-memory errors + ApiFuture future = collection.limit(batchSize).get(); + int deleted = 0; + // future.get() blocks on document retrieval + List documents = future.get().getDocuments(); + for (QueryDocumentSnapshot document : documents) { + document.getReference().delete(); + ++deleted; + } + if (deleted >= batchSize) { + // retrieve and delete another batch + deleteCollection(collection, batchSize); + } + } catch (Exception e) { + System.err.println("Error deleting collection : " + e.getMessage()); + } + } +} From 6e0d2288402a23b1c1753e6c68f0edbb2a9c19be Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 16 Jun 2022 20:05:19 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a6e3695a9a..970c84fb8f 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:25.3.0') +implementation platform('com.google.cloud:libraries-bom:25.4.0') implementation 'com.google.cloud:google-cloud-firestore' ``` From 8137a9687a3e13fa069d9ef712df47e90ad68eb4 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Wed, 22 Jun 2022 11:15:51 -0400 Subject: [PATCH 3/7] randomize collection id --- .../firestore/beam/ExampleFirestoreBeamRead.java | 3 +-- .../firestore/beam/ExampleFirestoreBeamWrite.java | 3 +-- .../beam/ExampleFirestoreWriteReadTest.java | 13 ++++++++----- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java index f5766081e1..69f9fc173a 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java @@ -36,14 +36,13 @@ import org.apache.beam.sdk.values.PCollection; public class ExampleFirestoreBeamRead { - public static void main(String[] args) { + public static void runRead(String[] args, String collectionId) { FirestoreOptions firestoreOptions = FirestoreOptions.getDefaultInstance(); PipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); Pipeline pipeline = Pipeline.create(options); - String collectionId = "cities-collection"; RpcQosOptions rpcQosOptions = RpcQosOptions.newBuilder() .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java index bd69e3437b..61c1b1f53d 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java @@ -31,12 +31,11 @@ public class ExampleFirestoreBeamWrite { private static final FirestoreOptions FIRESTORE_OPTIONS = FirestoreOptions.getDefaultInstance(); - public static void main(String[] args) { + public static void runWrite(String[] args, String collectionId) { PipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); Pipeline pipeline = Pipeline.create(options); - String collectionId = "cities-collection"; RpcQosOptions rpcQosOptions = RpcQosOptions.newBuilder() .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) diff --git a/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java b/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java index eb58244297..bd94fcf817 100644 --- a/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java +++ b/samples/snippets/src/test/java/com/example/firestore/beam/ExampleFirestoreWriteReadTest.java @@ -26,6 +26,7 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutionException; import org.junit.AfterClass; import org.junit.Assert; @@ -37,9 +38,11 @@ public class ExampleFirestoreWriteReadTest { private ByteArrayOutputStream bout; private static String projectId; private static Firestore firestore; + private static String collectionId; @BeforeClass public static void setUp() { + collectionId = "cities-collection-" + UUID.randomUUID().toString().substring(0, 10); projectId = requireEnv("GOOGLE_CLOUD_PROJECT"); firestore = FirestoreOptions.getDefaultInstance().getService(); } @@ -52,7 +55,7 @@ public void setUpStream() { @AfterClass public static void tearDown() { - deleteCollection(firestore.collection("cities-collection"), 1); + deleteCollection(firestore.collection(collectionId), 1); } @Test @@ -61,18 +64,18 @@ public void testWriteAndRead() throws ExecutionException, InterruptedException { "--project=" + projectId, "--region=us-central1", "--numWorkers=1", "--maxNumWorkers=1" }; // write the data to Firestore - ExampleFirestoreBeamWrite.main(args); + ExampleFirestoreBeamWrite.runWrite(args, collectionId); - ApiFuture apiFuture = firestore.collection("cities-collection").get(); + ApiFuture apiFuture = firestore.collection(collectionId).get(); QuerySnapshot querySnapshot = apiFuture.get(); List documents = querySnapshot.getDocuments(); Truth.assertThat(documents).hasSize(2); // filter and read it back - ExampleFirestoreBeamRead.main(args); + ExampleFirestoreBeamRead.runRead(args, collectionId); String output = bout.toString(); - Truth.assertThat(output).contains("/documents/cities-collection/NYC"); + Truth.assertThat(output).contains("/documents/" + collectionId + "/NYC"); } private static String requireEnv(String varName) { From 49bbbe15eb0cf39cd4f75a5718d2f7caea77cf8c Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Wed, 22 Jun 2022 11:38:15 -0400 Subject: [PATCH 4/7] review feedback --- samples/install-without-bom/pom.xml | 2 +- samples/native-image-sample/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- .../beam/ExampleFirestoreBeamRead.java | 48 ++++++++++--------- .../beam/ExampleFirestoreBeamWrite.java | 8 +++- 6 files changed, 35 insertions(+), 29 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 16afb8d6ee..8c4125c7ec 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -20,7 +20,7 @@ 1.8 1.8 - 2.38.0 + 2.39.0 UTF-8 diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 3001c9bd60..5c953c1fa5 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -23,7 +23,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> 1.8 1.8 - 2.38.0 + 2.39.0 UTF-8 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 486d4d6d16..b11bcaf000 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -21,7 +21,7 @@ 1.8 1.8 UTF-8 - 2.38.0 + 2.39.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index b7525f9e37..61f9a718de 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -21,7 +21,7 @@ 1.8 1.8 UTF-8 - 2.38.0 + 2.39.0 diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java index 69f9fc173a..9b706e0bad 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java @@ -51,11 +51,12 @@ public static void runRead(String[] args, String collectionId) { pipeline .apply(Create.of(collectionId)) .apply( - new ReadDocumentsQuery( + new FilterDocumentsQuery( firestoreOptions.getProjectId(), firestoreOptions.getDatabaseId())) .apply(FirestoreIO.v1().read().runQuery().withRpcQosOptions(rpcQosOptions).build()) .apply( ParDo.of( + // transform each document to its name new DoFn() { @ProcessElement public void processElement(ProcessContext c) { @@ -64,6 +65,7 @@ public void processElement(ProcessContext c) { })) .apply( ParDo.of( + // print the document name new DoFn() { @ProcessElement public void processElement(ProcessContext c) { @@ -74,13 +76,13 @@ public void processElement(ProcessContext c) { pipeline.run().waitUntilFinish(); } - private static final class ReadDocumentsQuery + private static final class FilterDocumentsQuery extends PTransform, PCollection> { private final String projectId; private final String databaseId; - public ReadDocumentsQuery(String projectId, String databaseId) { + public FilterDocumentsQuery(String projectId, String databaseId) { this.projectId = projectId; this.databaseId = databaseId; } @@ -90,33 +92,33 @@ public PCollection expand(PCollection input) { return input.apply( ParDo.of( new DoFn() { - @ProcessElement public void processElement(ProcessContext c) { + // select from collection "cities-collection-" + StructuredQuery.CollectionSelector collection = + StructuredQuery.CollectionSelector.newBuilder() + .setCollectionId(Objects.requireNonNull(c.element())) + .build(); + // filter where country is equal to USA + StructuredQuery.Filter countryFilter = + StructuredQuery.Filter.newBuilder() + .setFieldFilter( + StructuredQuery.FieldFilter.newBuilder() + .setField( + StructuredQuery.FieldReference.newBuilder() + .setFieldPath("country") + .build()) + .setValue(Value.newBuilder().setStringValue("USA").build()) + .setOp(StructuredQuery.FieldFilter.Operator.EQUAL)) + .buildPartial(); + RunQueryRequest runQueryRequest = RunQueryRequest.newBuilder() .setParent(DocumentRootName.format(projectId, databaseId)) .setStructuredQuery( StructuredQuery.newBuilder() - .addFrom( - StructuredQuery.CollectionSelector.newBuilder() - .setCollectionId(Objects.requireNonNull(c.element())) - .build()) - .setWhere( - StructuredQuery.Filter.newBuilder() - .setFieldFilter( - StructuredQuery.FieldFilter.newBuilder() - .setField( - StructuredQuery.FieldReference.newBuilder() - .setFieldPath("country") - .build()) - .setValue( - Value.newBuilder() - .setStringValue("USA") - .build()) - .setOp( - StructuredQuery.FieldFilter.Operator.EQUAL)) - .buildPartial()) + .addFrom(collection) + .setWhere(countryFilter) .build()) .build(); c.output(runQueryRequest); diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java index 61c1b1f53d..55b79345ad 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java @@ -32,6 +32,7 @@ public class ExampleFirestoreBeamWrite { private static final FirestoreOptions FIRESTORE_OPTIONS = FirestoreOptions.getDefaultInstance(); public static void runWrite(String[] args, String collectionId) { + // create pipeline options from the passed in arguments PipelineOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(PipelineOptions.class); Pipeline pipeline = Pipeline.create(options); @@ -41,12 +42,13 @@ public static void runWrite(String[] args, String collectionId) { .withHintMaxNumWorkers(options.as(DataflowPipelineOptions.class).getMaxNumWorkers()) .build(); + // create some writes Write write1 = Write.newBuilder() .setUpdate( Document.newBuilder() // resolves to - // projects//databases//documents/cities/NYC + // projects//databases//documents//NYC .setName(createDocumentName(collectionId, "NYC")) .putFields("name", Value.newBuilder().setStringValue("New York City").build()) .putFields("state", Value.newBuilder().setStringValue("New York").build()) @@ -58,17 +60,19 @@ public static void runWrite(String[] args, String collectionId) { .setUpdate( Document.newBuilder() // resolves to - // projects//databases//documents/cities/TOK + // projects//databases//documents//TOK .setName(createDocumentName(collectionId, "TOK")) .putFields("name", Value.newBuilder().setStringValue("Tokyo").build()) .putFields("country", Value.newBuilder().setStringValue("Japan").build()) .putFields("capital", Value.newBuilder().setBooleanValue(true).build())) .build(); + // batch write the data pipeline .apply(Create.of(write1, write2)) .apply(FirestoreIO.v1().write().batchWrite().withRpcQosOptions(rpcQosOptions).build()); + // run the pipeline pipeline.run().waitUntilFinish(); } From 71e791afd20cd1725634416ca952ffc08f3bd238 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Wed, 22 Jun 2022 12:40:09 -0400 Subject: [PATCH 5/7] add README.md --- .../java/com/example/firestore/beam/README.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 samples/snippets/src/main/java/com/example/firestore/beam/README.md diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/README.md b/samples/snippets/src/main/java/com/example/firestore/beam/README.md new file mode 100644 index 0000000000..5694a5f449 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/firestore/beam/README.md @@ -0,0 +1,35 @@ +# Firestore Beam Samples + +## Running instructions +These examples will show you how to read and write data from/into a Firestore database using the Firestore Beam connector. + +1. Set up the following environment variables +``` +GOOGLE_CLOUD_PROJECT= +REGION= +TEMP_LOCATION=gs:///temp/ +NUM_WORKERS= +MAX_NUM_WORKERS= +``` +2. Run the command to write example "cities" data: +``` +mvn compile exec:java -Dexec.mainClass=ExampleFirestoreWrite \ +"-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT \ +--runner=DataflowRunner \ +--project=$GOOGLE_CLOUD_PROJECT \ +--region=$REGION \ +--tempLocation=$TEMP_LOCATION \ +--numWorkers=$NUM_WORKERS \ +--maxNumWorkers=$MAX_NUM_WORKERS" +``` + +3. Run the command to filter and read the data you wrote: +``` +mvn compile exec:java -Dexec.mainClass=ExampleFirestoreRead \ +"-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT \ +--runner=DataflowRunner \ +--region=$REGION \ +--tempLocation=$TEMP_LOCATION \ +--numWorkers=$NUM_WORKERS \ +--maxNumWorkers=$MAX_NUM_WORKERS" +``` \ No newline at end of file From d7296937e62f0870c4f177682d3fa183dfd2c0a1 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Wed, 22 Jun 2022 12:44:15 -0400 Subject: [PATCH 6/7] fix main class --- .../example/firestore/beam/ExampleFirestoreBeamRead.java | 6 ++++++ .../example/firestore/beam/ExampleFirestoreBeamWrite.java | 5 +++++ .../src/main/java/com/example/firestore/beam/README.md | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java index 9b706e0bad..35b824b2a6 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamRead.java @@ -23,6 +23,7 @@ import com.google.firestore.v1.StructuredQuery; import com.google.firestore.v1.Value; import java.util.Objects; +import java.util.UUID; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; @@ -36,6 +37,11 @@ import org.apache.beam.sdk.values.PCollection; public class ExampleFirestoreBeamRead { + + public static void main(String[] args) { + runRead(args, "cities-collection-" + UUID.randomUUID().toString().substring(0, 10)); + } + public static void runRead(String[] args, String collectionId) { FirestoreOptions firestoreOptions = FirestoreOptions.getDefaultInstance(); diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java index 55b79345ad..13fb64fefb 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java @@ -20,6 +20,7 @@ import com.google.firestore.v1.Document; import com.google.firestore.v1.Value; import com.google.firestore.v1.Write; +import java.util.UUID; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.gcp.firestore.FirestoreIO; @@ -31,6 +32,10 @@ public class ExampleFirestoreBeamWrite { private static final FirestoreOptions FIRESTORE_OPTIONS = FirestoreOptions.getDefaultInstance(); + public static void main(String[] args) { + runWrite(args, "cities-collection-" + UUID.randomUUID().toString().substring(0, 10)); + } + public static void runWrite(String[] args, String collectionId) { // create pipeline options from the passed in arguments PipelineOptions options = diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/README.md b/samples/snippets/src/main/java/com/example/firestore/beam/README.md index 5694a5f449..33b10f9392 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/README.md +++ b/samples/snippets/src/main/java/com/example/firestore/beam/README.md @@ -13,7 +13,7 @@ MAX_NUM_WORKERS= ``` 2. Run the command to write example "cities" data: ``` -mvn compile exec:java -Dexec.mainClass=ExampleFirestoreWrite \ +mvn compile exec:java -Dexec.mainClass=ExampleFirestoreBeamWrite \ "-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT \ --runner=DataflowRunner \ --project=$GOOGLE_CLOUD_PROJECT \ @@ -25,7 +25,7 @@ mvn compile exec:java -Dexec.mainClass=ExampleFirestoreWrite \ 3. Run the command to filter and read the data you wrote: ``` -mvn compile exec:java -Dexec.mainClass=ExampleFirestoreRead \ +mvn compile exec:java -Dexec.mainClass=ExampleFirestoreBeamRead \ "-Dexec.args=--project=$GOOGLE_CLOUD_PROJECT \ --runner=DataflowRunner \ --region=$REGION \ From 4b2e5c60299515b926179114f2e7867256040b42 Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Wed, 22 Jun 2022 12:55:36 -0400 Subject: [PATCH 7/7] fix checkstyle --- .../com/example/firestore/beam/ExampleFirestoreBeamWrite.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java index 13fb64fefb..a4921c57e8 100644 --- a/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java +++ b/samples/snippets/src/main/java/com/example/firestore/beam/ExampleFirestoreBeamWrite.java @@ -53,7 +53,7 @@ public static void runWrite(String[] args, String collectionId) { .setUpdate( Document.newBuilder() // resolves to - // projects//databases//documents//NYC + // projects//databases//documents//NYC .setName(createDocumentName(collectionId, "NYC")) .putFields("name", Value.newBuilder().setStringValue("New York City").build()) .putFields("state", Value.newBuilder().setStringValue("New York").build()) @@ -65,7 +65,7 @@ public static void runWrite(String[] args, String collectionId) { .setUpdate( Document.newBuilder() // resolves to - // projects//databases//documents//TOK + // projects//databases//documents//TOK .setName(createDocumentName(collectionId, "TOK")) .putFields("name", Value.newBuilder().setStringValue("Tokyo").build()) .putFields("country", Value.newBuilder().setStringValue("Japan").build())