diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2aee0146b5..7c4d9ac88e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,8 +4,9 @@ # For syntax help see: # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax -# The @googleapis/bigtable-dpe is the default owner for changes in this repo -**/*.java @googleapis/bigtable-dpe +# The @googleapis/api-bigtable is the default owner for changes in this repo +* @googleapis/yoshi-java @googleapis/api-bigtable +**/*.java @googleapis/api-bigtable # The java-samples-reviewers team is the default owner for samples changes samples/**/*.java @googleapis/java-samples-reviewers diff --git a/.github/readme/synth.py b/.github/readme/synth.py new file mode 100644 index 0000000000..7b48cc28d3 --- /dev/null +++ b/.github/readme/synth.py @@ -0,0 +1,19 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated the README for this library.""" + +from synthtool.languages import java + +java.custom_templates(["java_library/README.md"]) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 0000000000..6bddd18eac --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,49 @@ + +# Whether or not rebase-merging is enabled on this repository. +# Defaults to `true` +rebaseMergeAllowed: false + +# Whether or not squash-merging is enabled on this repository. +# Defaults to `true` +squashMergeAllowed: true + +# Whether or not PRs are merged with a merge commit on this repository. +# Defaults to `false` +mergeCommitAllowed: false + +# Rules for master branch protection +branchProtectionRules: +# Identifies the protection rule pattern. Name of the branch to be protected. +# Defaults to `master` +- pattern: master + # Can admins overwrite branch protection. + # Defaults to `true` + isAdminEnforced: true + # Number of approving reviews required to update matching branches. + # Defaults to `1` + requiredApprovingReviewCount: 1 + # Are reviews from code owners required to update matching branches. + # Defaults to `false` + requiresCodeOwnerReviews: true + # Require up to date branches + requiresStrictStatusChecks: false + # List of required status check contexts that must pass for commits to be accepted to matching branches. + requiredStatusCheckContexts: + - "dependencies (8)" + - "dependencies (11)" + - "linkage-monitor" + - "lint" + - "clirr" + - "units (7)" + - "units (8)" + - "units (11)" + - "Kokoro - Test: Integration" + - "cla/google" +# List of explicit permissions to add (additive only) +permissionRules: +- team: yoshi-admins + permission: admin +- team: yoshi-java-admins + permission: admin +- team: yoshi-java + permission: push \ No newline at end of file diff --git a/.github/workflows/auto-release.yaml b/.github/workflows/auto-release.yaml new file mode 100644 index 0000000000..bc1554aecb --- /dev/null +++ b/.github/workflows/auto-release.yaml @@ -0,0 +1,88 @@ +on: + pull_request: +name: auto-release +jobs: + approve: + runs-on: ubuntu-latest + if: contains(github.head_ref, 'release-v') + steps: + - uses: actions/github-script@v3.0.0 + with: + github-token: ${{secrets.YOSHI_APPROVER_TOKEN}} + debug: true + script: | + // only approve PRs from release-please[bot] + if (context.payload.pull_request.user.login !== "release-please[bot]") { + return; + } + + // only approve PRs like "chore: release " + if ( !context.payload.pull_request.title.startsWith("chore: release") ) { + return; + } + + // only approve PRs with pom.xml and versions.txt changes + const filesPromise = github.pulls.listFiles.endpoint({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }); + const changed_files = await github.paginate(filesPromise) + + if ( changed_files.length < 1 ) { + console.log( "Not proceeding since PR is empty!" ) + return; + } + + if ( !changed_files.some(v => v.filename.includes("pom")) || !changed_files.some(v => v.filename.includes("versions.txt")) ) { + console.log( "PR file changes do not have pom.xml or versions.txt -- something is wrong. PTAL!" ) + return; + } + + // trigger auto-release when + // 1) it is a SNAPSHOT release (auto-generated post regular release) + // 2) there are dependency updates only + // 3) there are no open dependency update PRs in this repo (to avoid multiple releases) + if ( + context.payload.pull_request.body.includes("Fix") || + context.payload.pull_request.body.includes("Build") || + context.payload.pull_request.body.includes("Documentation") || + context.payload.pull_request.body.includes("BREAKING CHANGES") || + context.payload.pull_request.body.includes("Features") + ) { + console.log( "Not auto-releasing since it is not a dependency-update-only release." ); + return; + } + + const promise = github.pulls.list.endpoint({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open' + }); + const open_pulls = await github.paginate(promise) + + if ( open_pulls.length > 1 && !context.payload.pull_request.title.includes("SNAPSHOT") ) { + for ( const pull of open_pulls ) { + if ( pull.title.startsWith("deps: update dependency") ) { + console.log( "Not auto-releasing yet since there are dependency update PRs open in this repo." ); + return; + } + } + } + + // approve release PR + await github.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Rubber stamped release!', + pull_number: context.payload.pull_request.number, + event: 'APPROVE' + }); + + // attach kokoro:force-run and automerge labels + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + labels: ['kokoro:force-run', 'automerge'] + }); diff --git a/.github/workflows/samples.yaml b/.github/workflows/samples.yaml index a1d5007306..c46230a78c 100644 --- a/.github/workflows/samples.yaml +++ b/.github/workflows/samples.yaml @@ -2,7 +2,7 @@ on: pull_request: name: samples jobs: - lint: + checkstyle: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.kokoro/continuous/readme.cfg b/.kokoro/continuous/readme.cfg new file mode 100644 index 0000000000..5b59996afa --- /dev/null +++ b/.kokoro/continuous/readme.cfg @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/java-bigtable/.kokoro/readme.sh" +} + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.log" + } +} + +# The github token is stored here. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + # TODO(theacodes): remove this after secrets have globally propagated + backend_type: FASTCONFIGPUSH + } + } +} + +# Common env vars for all repositories and builds. +env_vars: { + key: "GITHUB_USER" + value: "yoshi-automation" +} +env_vars: { + key: "GITHUB_EMAIL" + value: "yoshi-automation@google.com" +} diff --git a/.kokoro/readme.sh b/.kokoro/readme.sh new file mode 100644 index 0000000000..b586093890 --- /dev/null +++ b/.kokoro/readme.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +cd ${KOKORO_ARTIFACTS_DIR}/github/java-bigtable + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Kokoro exposes this as a file, but the scripts expect just a plain variable. +export GITHUB_TOKEN=$(cat ${KOKORO_KEYSTORE_DIR}/73713_yoshi-automation-github-key) + +# Setup git credentials +echo "https://${GITHUB_TOKEN}:@github.com" >> ~/.git-credentials +git config --global credential.helper 'store --file ~/.git-credentials' + +python3.6 -m pip install git+https://github.com/googleapis/synthtool.git#egg=gcp-synthtool +python3.6 -m autosynth.synth \ + --repository=googleapis/java-bigtable \ + --synth-file-name=.github/readme/synth.py \ + --metadata-path=.github/readme/synth.metadata \ + --pr-title="chore: regenerate README" \ + --branch-suffix="readme" \ No newline at end of file diff --git a/.repo-metadata.json b/.repo-metadata.json index 9d4c364693..238d7852fc 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -9,6 +9,6 @@ "repo": "googleapis/java-bigtable", "repo_short": "java-bigtable", "distribution_name": "com.google.cloud:google-cloud-bigtable", - "codeowner_team": "@googleapis/bigtable-dpe", + "codeowner_team": "@googleapis/api-bigtable", "api_id": "bigtable.googleapis.com" -} \ No newline at end of file +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b5c872aa6..bb8588e785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [1.16.0](https://www.github.com/googleapis/java-bigtable/compare/v1.15.0...v1.16.0) (2020-10-06) + + +### Features + +* add keepalive changes in java client library ([#409](https://www.github.com/googleapis/java-bigtable/issues/409)) ([edbcde1](https://www.github.com/googleapis/java-bigtable/commit/edbcde1a5b22317319803cb57401252aac6d580d)) +* include User agent ([#404](https://www.github.com/googleapis/java-bigtable/issues/404)) ([0cd71b5](https://www.github.com/googleapis/java-bigtable/commit/0cd71b59305cd7a336c223faff68402a8bee4639)) + + +### Bug Fixes + +* **test:** Clean up tests ([#439](https://www.github.com/googleapis/java-bigtable/issues/439)) ([c5c881b](https://www.github.com/googleapis/java-bigtable/commit/c5c881bb956860a393c2a7f34d0d790a23d270af)) +* Add documentation to bulkReadRows that each batch will process t… ([#410](https://www.github.com/googleapis/java-bigtable/issues/410)) ([71dc8e3](https://www.github.com/googleapis/java-bigtable/commit/71dc8e3419fbb49d48bb7a3fd984d24e24661c81)) +* Filters should be serializable ([#397](https://www.github.com/googleapis/java-bigtable/issues/397)) ([57edfde](https://www.github.com/googleapis/java-bigtable/commit/57edfde5eace28d50ec777e14589c9747616f0a8)) +* RowCells are not actually serializeable ([#407](https://www.github.com/googleapis/java-bigtable/issues/407)) ([39e17cc](https://www.github.com/googleapis/java-bigtable/commit/39e17cc17c438f2d665875f9ff2b2cdf984e37b7)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-conformance-tests to v0.0.12 ([#415](https://www.github.com/googleapis/java-bigtable/issues/415)) ([7b3713a](https://www.github.com/googleapis/java-bigtable/commit/7b3713a8935b2f0b1ca57d189e9e0363506b28a1)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.10.0 ([#428](https://www.github.com/googleapis/java-bigtable/issues/428)) ([373032e](https://www.github.com/googleapis/java-bigtable/commit/373032e93b838cbc0ccab7dc45cd61b3b973542a)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.10.1 ([#443](https://www.github.com/googleapis/java-bigtable/issues/443)) ([939fefa](https://www.github.com/googleapis/java-bigtable/commit/939fefa819d09885489d9faeedadc74ee2b0e1b9)) +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.9.1 ([#427](https://www.github.com/googleapis/java-bigtable/issues/427)) ([5175e28](https://www.github.com/googleapis/java-bigtable/commit/5175e28f5ce69f6fb3ed14131c1cfd26dbc47bb9)) + ## [1.15.0](https://www.github.com/googleapis/java-bigtable/compare/v1.14.0...v1.15.0) (2020-09-01) diff --git a/README.md b/README.md index 90549e0575..22f3636595 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,16 @@ If you are using Maven, add this to your pom.xml file com.google.cloud google-cloud-bigtable - 1.15.0 + 1.16.0 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-bigtable:1.15.0' +compile 'com.google.cloud:google-cloud-bigtable:1.16.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "1.15.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "1.16.0" ``` [//]: # ({x-version-update-end}) diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 7bb0444ad0..addc96ab77 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom com.google.cloud @@ -72,32 +72,32 @@ com.google.cloud google-cloud-bigtable - 1.15.0 + 1.16.0 com.google.cloud google-cloud-bigtable-emulator - 0.124.0 + 0.125.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.15.0 + 1.16.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.15.0 + 1.16.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 1.15.0 + 1.16.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.15.0 + 1.16.0 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index 3c12565a0c..fee20379f6 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom @@ -79,7 +79,7 @@ com.google.cloud google-cloud-shared-dependencies - 0.9.0 + 0.10.1 pom import diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index feed747c3d..166d242889 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.124.0 + 0.125.0 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.15.0 + 1.16.0 scm:git:git@github.com:googleapis/java-bigtable.git @@ -80,14 +80,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom import diff --git a/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java b/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java index 9c2fa49b38..3cfbc981ab 100644 --- a/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java +++ b/google-cloud-bigtable-emulator/src/main/java/com/google/cloud/bigtable/emulator/v2/Emulator.java @@ -42,7 +42,6 @@ */ @BetaApi("Surface for Bigtable emulator is not yet stable") public class Emulator { - private static final Logger LOGGER = Logger.getLogger(Emulator.class.getName()); private final Path executable; @@ -54,6 +53,9 @@ public class Emulator { private ManagedChannel dataChannel; private ManagedChannel adminChannel; + public static Emulator createFromPath(Path path) { + return new Emulator(path); + } /** * Create a new instance of emulator. The emulator will use the bundled binaries in this jar. * Please note that the emulator is created in a stopped state, please use {@link #start()} after @@ -177,7 +179,13 @@ public synchronized ManagedChannel getDataChannel() { } if (dataChannel == null) { - dataChannel = newChannelBuilder(port).maxInboundMessageSize(256 * 1024 * 1024).build(); + dataChannel = + newChannelBuilder(port) + .maxInboundMessageSize(256 * 1024 * 1024) + .keepAliveTimeout(10, TimeUnit.SECONDS) + .keepAliveTime(10, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .build(); } return dataChannel; } diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index 0ba7a1d644..d967fe0e00 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 1.15.0 + 1.16.0 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,7 +12,7 @@ com.google.cloud google-cloud-bigtable-parent - 1.15.0 + 1.16.0 google-cloud-bigtable @@ -36,14 +36,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom import diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java index 0fffcc54fe..5d0350b9cf 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java @@ -76,7 +76,7 @@ * // One instance per application. * BigtableTableAdminClient client = BigtableTableAdminClient.create("[PROJECT]", "[INSTANCE]"); * - * CreateTable request = + * CreateTableRequest request = * CreateTableRequest.of("my-table") * .addFamily("cf1") * .addFamily("cf2", GCRULES.maxVersions(10)) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java index e249684244..04e1b15987 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClient.java @@ -57,7 +57,7 @@ * // One instance per application. * BigtableDataClient client = BigtableDataClient.create("[PROJECT]", "[INSTANCE]") * - * for(Row row : client.readRows(Query.create("[TABLE]")) { + * for(Row row : client.readRows(Query.create("[TABLE]"))) { * // Do something with row * } * @@ -1077,9 +1077,13 @@ public Batcher newBulkMutationBatcher(@Nonnull String ta } /** - * Reads rows for given tableId in a batch. If the row does not exist, the value will be null. - * This operation should be called with in a single thread. The returned Batcher instance is not - * threadsafe, it can only be used from single thread. + * Reads rows for given tableId in a batch. If the row does not exist, the value will be null. The + * returned Batcher instance is not threadsafe, it can only be used from a single thread. + * + *

Performance notice: The ReadRows protocol requires that rows are sent in ascending key + * order, which means that the keys are processed sequentially on the server-side, so batching + * allows improving throughput but not latency. Lower latencies can be achieved by sending smaller + * requests concurrently. * *

Sample Code: * @@ -1113,8 +1117,13 @@ public Batcher newBulkReadRowsBatcher(String tableId) { /** * Reads rows for given tableId and filter criteria in a batch. If the row does not exist, the - * value will be null. This operation should be called with in a single thread. The returned - * Batcher instance is not threadsafe, it can only be used from single thread. + * value will be null. The returned Batcher instance is not threadsafe, it can only be used from a + * single thread. + * + *

Performance notice: The ReadRows protocol requires that rows are sent in ascending key + * order, which means that the keys are processed sequentially on the server-side, so batching + * allows improving throughput but not latency. Lower latencies can be achieved by sending smaller + * requests concurrently. * *

Sample Code: * diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index 8dd0fa6d97..3b07eeaf23 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.logging.Logger; import javax.annotation.Nonnull; +import org.threeten.bp.Duration; /** * Settings class to configure an instance of {@link BigtableDataClient}. @@ -122,6 +123,10 @@ public ManagedChannelBuilder apply(ManagedChannelBuilder input) { return input.usePlaintext(); } }) + .setKeepAliveTime(Duration.ofSeconds(10)) // sends ping in this interval + .setKeepAliveTimeout( + Duration.ofSeconds(10)) // wait this long before considering the connection dead + .setKeepAliveWithoutCalls(true) // sends ping without active streams .build()); LOGGER.info("Connecting to the Bigtable emulator at " + hostname + ":" + port); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Filters.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Filters.java index bd310cd2c5..4c82608545 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Filters.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Filters.java @@ -25,6 +25,9 @@ import com.google.cloud.bigtable.data.v2.models.Range.AbstractTimestampRange; import com.google.common.base.Preconditions; import com.google.protobuf.ByteString; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.Serializable; import javax.annotation.Nonnull; @@ -200,7 +203,19 @@ public Filter label(@Nonnull String label) { // Implementations of target specific filters. /** DSL for adding filters to a chain. */ public static final class ChainFilter implements Filter { - private RowFilter.Chain.Builder builder; + private static final long serialVersionUID = -6756759448656768478L; + private transient RowFilter.Chain.Builder builder; + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + s.writeObject(builder.build()); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + RowFilter.Chain chain = (RowFilter.Chain) s.readObject(); + this.builder = chain.toBuilder(); + } private ChainFilter() { this.builder = RowFilter.Chain.newBuilder(); @@ -241,7 +256,19 @@ public ChainFilter clone() { /** DSL for adding filters to the interleave list. */ public static final class InterleaveFilter implements Filter { - private RowFilter.Interleave.Builder builder; + private static final long serialVersionUID = -6356151037337889421L; + private transient RowFilter.Interleave.Builder builder; + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + s.writeObject(builder.build()); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + RowFilter.Interleave interleave = (RowFilter.Interleave) s.readObject(); + this.builder = interleave.toBuilder(); + } private InterleaveFilter() { builder = RowFilter.Interleave.newBuilder(); @@ -281,7 +308,19 @@ public InterleaveFilter clone() { /** DSL for configuring a conditional filter. */ public static final class ConditionFilter implements Filter { - private RowFilter.Condition.Builder builder; + private static final long serialVersionUID = -2720899822014446776L; + private transient RowFilter.Condition.Builder builder; + + private void writeObject(ObjectOutputStream s) throws IOException { + s.defaultWriteObject(); + s.writeObject(builder.build()); + } + + private void readObject(ObjectInputStream s) throws IOException, ClassNotFoundException { + s.defaultReadObject(); + RowFilter.Condition condition = (RowFilter.Condition) s.readObject(); + this.builder = condition.toBuilder(); + } private ConditionFilter(@Nonnull Filter predicate) { Preconditions.checkNotNull(predicate); @@ -323,7 +362,9 @@ public ConditionFilter clone() { } } - public static final class KeyFilter { + public static final class KeyFilter implements Serializable { + private static final long serialVersionUID = 5137765114285539458L; + private KeyFilter() {} /** @@ -383,7 +424,9 @@ public Filter sample(double probability) { } } - public static final class FamilyFilter { + public static final class FamilyFilter implements Serializable { + private static final long serialVersionUID = -4470936841191831553L; + private FamilyFilter() {} /** @@ -405,7 +448,9 @@ public Filter exactMatch(@Nonnull String value) { } } - public static final class QualifierFilter { + public static final class QualifierFilter implements Serializable { + private static final long serialVersionUID = -1274850022909506559L; + private QualifierFilter() {} /** @@ -459,7 +504,8 @@ public QualifierRangeFilter rangeWithinFamily(@Nonnull String family) { /** Matches only cells from columns within the given range. */ public static final class QualifierRangeFilter - extends AbstractByteStringRange implements Filter, Serializable { + extends AbstractByteStringRange implements Filter { + private static final long serialVersionUID = -1909319911147913630L; private final String family; private QualifierRangeFilter(String family) { @@ -505,7 +551,9 @@ public QualifierRangeFilter clone() { } } - public static final class TimestampFilter { + public static final class TimestampFilter implements Serializable { + private static final long serialVersionUID = 5284219722591464991L; + private TimestampFilter() {} /** @@ -529,7 +577,9 @@ public TimestampRangeFilter exact(Long exactTimestamp) { /** Matches only cells with microsecond timestamps within the given range. */ public static final class TimestampRangeFilter - extends AbstractTimestampRange implements Filter, Serializable { + extends AbstractTimestampRange implements Filter { + private static final long serialVersionUID = 8410980338603335276L; + private TimestampRangeFilter() {} @InternalApi @@ -571,7 +621,9 @@ public TimestampRangeFilter clone() { } } - public static final class ValueFilter { + public static final class ValueFilter implements Serializable { + private static final long serialVersionUID = 6722715229238811179L; + private ValueFilter() {} /** @@ -628,7 +680,9 @@ public Filter strip() { /** Matches only cells with values that fall within the given value range. */ public static final class ValueRangeFilter extends AbstractByteStringRange - implements Filter, Serializable { + implements Filter { + private static final long serialVersionUID = -2452360677825047088L; + private ValueRangeFilter() {} @InternalApi @@ -668,7 +722,9 @@ public ValueRangeFilter clone() { } } - public static final class OffsetFilter { + public static final class OffsetFilter implements Serializable { + private static final long serialVersionUID = 3228791236971884041L; + private OffsetFilter() {} /** @@ -681,7 +737,9 @@ public Filter cellsPerRow(int count) { } } - public static final class LimitFilter { + public static final class LimitFilter implements Serializable { + private static final long serialVersionUID = -794915549003008940L; + private LimitFilter() {} /** @@ -705,6 +763,7 @@ public Filter cellsPerColumn(int count) { } private static final class SimpleFilter implements Filter { + private static final long serialVersionUID = 3595911451325189833L; private final RowFilter proto; private SimpleFilter(@Nonnull RowFilter proto) { @@ -729,7 +788,7 @@ public SimpleFilter clone() { } @InternalExtensionOnly - public interface Filter extends Cloneable { + public interface Filter extends Cloneable, Serializable { @InternalApi RowFilter toProto(); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowCell.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowCell.java index 473c560b83..e4f3c635d3 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowCell.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowCell.java @@ -20,6 +20,7 @@ import com.google.auto.value.AutoValue; import com.google.cloud.bigtable.data.v2.internal.ByteStringComparator; import com.google.common.collect.ComparisonChain; +import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; import java.io.Serializable; import java.util.Comparator; @@ -62,7 +63,13 @@ public static RowCell create( long timestamp, @Nonnull List labels, @Nonnull ByteString value) { - return new AutoValue_RowCell(family, qualifier, timestamp, value, labels); + // Ensure that the list is serializable and optimize for the common case + if (labels.isEmpty()) { + labels = ImmutableList.of(); + } else { + labels = ImmutableList.copyOf(labels); + } + return new AutoValue_RowCell(family, qualifier, timestamp, value, ImmutableList.copyOf(labels)); } /** The cell's family */ diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java index d843265d1e..360141083e 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java @@ -20,9 +20,11 @@ import com.google.api.gax.batching.BatchingSettings; import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.FixedHeaderProvider; import com.google.api.gax.rpc.ServerStreamingCallSettings; import com.google.api.gax.rpc.StatusCode.Code; import com.google.api.gax.rpc.StubSettings; @@ -38,8 +40,10 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsBatchingDescriptor; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; import javax.annotation.Nonnull; @@ -82,8 +86,7 @@ public class EnhancedBigtableStubSettings extends StubSettings IDEMPOTENT_RETRY_CODES = ImmutableSet.of(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); @@ -238,6 +241,10 @@ public static InstantiatingGrpcChannelProvider.Builder defaultGrpcTransportProvi return BigtableStubSettings.defaultGrpcTransportProviderBuilder() .setPoolSize(getDefaultChannelPoolSize()) .setMaxInboundMessageSize(MAX_MESSAGE_SIZE) + .setKeepAliveTime(Duration.ofSeconds(10)) // sends ping in this interval + .setKeepAliveTimeout( + Duration.ofSeconds(10)) // wait this long before considering the connection dead + .setKeepAliveWithoutCalls(true) // sends ping without active streams // TODO(weiranf): Set this to true by default once DirectPath goes to public beta .setAttemptDirectPath(isDirectPathEnabled()); } @@ -530,8 +537,19 @@ private Builder() { setTransportChannelProvider(defaultTransportChannelProvider()); setStreamWatchdogCheckInterval(baseDefaults.getStreamWatchdogCheckInterval()); setStreamWatchdogProvider(baseDefaults.getStreamWatchdogProvider()); - setInternalHeaderProvider( - BigtableStubSettings.defaultApiClientHeaderProviderBuilder().build()); + + // Inject the UserAgent in addition to api-client header + Map headers = + ImmutableMap.builder() + .putAll( + BigtableStubSettings.defaultApiClientHeaderProviderBuilder().build().getHeaders()) + // GrpcHeaderInterceptor treats the `user-agent` as a magic string + .put( + "user-agent", + "bigtable-java/" + + GaxProperties.getLibraryVersion(EnhancedBigtableStubSettings.class)) + .build(); + setInternalHeaderProvider(FixedHeaderProvider.create(headers)); // Per-method settings using baseSettings for defaults. readRowsSettings = ServerStreamingCallSettings.newBuilder(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminSettingsTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminSettingsTest.java index 6fa6f3e4a7..c6ccf9557b 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminSettingsTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminSettingsTest.java @@ -21,8 +21,11 @@ import com.google.api.gax.rpc.StatusCode.Code; import java.io.IOException; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.mockito.Mockito; +@RunWith(JUnit4.class) public class BigtableInstanceAdminSettingsTest { @Test public void testProjectName() throws Exception { diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTest.java index 2cda42b3ea..c265514325 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTest.java @@ -71,18 +71,22 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.mockito.Matchers; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; -import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; import org.mockito.stubbing.Answer; import org.threeten.bp.Instant; -@RunWith(MockitoJUnitRunner.class) +@RunWith(JUnit4.class) public class BigtableTableAdminClientTest { + @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); private static final String PROJECT_ID = "my-project"; private static final String INSTANCE_ID = "my-instance"; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminSettingsTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminSettingsTest.java index 3a16dd236d..77ce733ce0 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminSettingsTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminSettingsTest.java @@ -21,8 +21,11 @@ import com.google.api.gax.rpc.StatusCode.Code; import java.io.IOException; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.mockito.Mockito; +@RunWith(JUnit4.class) public class BigtableTableAdminSettingsTest { @Test diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java index 1e1bc84648..4d17bbf0e9 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java @@ -47,9 +47,12 @@ import java.util.concurrent.TimeoutException; import java.util.logging.Logger; import org.junit.*; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.threeten.bp.Duration; import org.threeten.bp.Instant; +@RunWith(JUnit4.class) public class BigtableBackupIT { private static final Logger LOGGER = Logger.getLogger(BigtableBackupIT.class.getName()); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java index f31654334d..9b41444c7f 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java @@ -40,8 +40,11 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.threeten.bp.Instant; +@RunWith(JUnit4.class) public class BigtableInstanceAdminClientIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java index 333fa875ce..2dc64b6407 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java @@ -46,8 +46,11 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; import org.threeten.bp.Duration; +@RunWith(JUnit4.class) public class BigtableTableAdminClientIT { @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); @Rule public final TestName testNameRule = new TestName(); diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/FiltersTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/FiltersTest.java index ad79025258..e5fcd133f5 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/FiltersTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/FiltersTest.java @@ -17,6 +17,8 @@ import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assert.fail; import com.google.bigtable.v2.ColumnRange; import com.google.bigtable.v2.RowFilter; @@ -26,6 +28,15 @@ import com.google.bigtable.v2.TimestampRange; import com.google.bigtable.v2.ValueRange; import com.google.protobuf.ByteString; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -528,4 +539,213 @@ public void labelTest() { assertThat(actualFilter).isEqualTo(expectedFilter); } + + @Test + public void serializationTest() throws InvocationTargetException, IllegalAccessException { + // checks that the all objects returned by the all methods of the Filters class + // can be serialized/deserialized. + + for (Method m : Filters.class.getDeclaredMethods()) { + String name = m.getName(); + if (Modifier.isPublic(m.getModifiers())) { + switch (name) { + case "condition": + checkSerialization( + name, + FILTERS + .condition( + FILTERS + .chain() + .filter(FILTERS.qualifier().exactMatch("data_plan_10gb")) + .filter(FILTERS.value().exactMatch("true"))) + .then(FILTERS.label("passed-filter")) + .otherwise(FILTERS.label("filtered-out"))); + break; + case "label": + checkSerialization(name, FILTERS.label("label")); + break; + case "fromProto": + checkSerialization(name, FILTERS.label("label").toProto()); + break; + default: + checkSerialization(name, m.invoke(FILTERS)); + } + } + } + } + + private static void checkSerialization(String name, Object filter) { + try { + Object deserialized = serializeDeserialize(filter); + checkClassDeclaresSerialVersionUid(filter.getClass()); + if (filter instanceof Filters.Filter) { + checkFilters(name, (Filters.Filter) filter, (Filters.Filter) deserialized); + } else if (filter instanceof RowFilter) { + assertWithMessage("'" + name + "' deserialized filter differs") + .that(filter) + .isEqualTo(deserialized); + } else { + Class cls = filter.getClass(); + checkClassDoesNotContainNonStaticFields(cls, cls.getFields()); + checkClassDoesNotContainNonStaticFields(cls, cls.getDeclaredFields()); + checkSpawnedFilters(name, cls, filter, deserialized); + } + } catch (IOException | ClassNotFoundException e) { + fail(name + ": " + e); + } + } + + private static void checkFilters( + String name, Filters.Filter original, Filters.Filter deserialized) { + RowFilter protoBefore = ((Filters.Filter) original).toProto(); + RowFilter protoAfter = ((Filters.Filter) deserialized).toProto(); + assertWithMessage("'" + name + "' filter protoBuf mismatches after deserialization") + .that(protoBefore) + .isEqualTo(protoAfter); + } + + private static void checkSpawnedFilters( + String name, Class cls, Object original, Object deserialized) { + + int numberOfMethods = 0; + for (Method m : cls.getDeclaredMethods()) { + if (Modifier.isPublic(m.getModifiers())) { + numberOfMethods++; + } + } + ByteString re = ByteString.copyFromUtf8("some\\[0\\-9\\]regex"); + + switch (name) { + case "family": + { + Filters.FamilyFilter f1 = (Filters.FamilyFilter) original; + Filters.FamilyFilter f2 = (Filters.FamilyFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(2); + checkFilters(name + "/exactMatch", f1.exactMatch("abc"), f2.exactMatch("abc")); + checkFilters(name + "/regex", f1.regex("*"), f2.regex("*")); + + break; + } + case "qualifier": + { + Filters.QualifierFilter f1 = (Filters.QualifierFilter) original; + Filters.QualifierFilter f2 = (Filters.QualifierFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(5); + checkFilters(name + "/exactMatch", f1.exactMatch("abc"), f2.exactMatch("abc")); + checkFilters(name + "/exactMatch(ByteString)", f1.exactMatch(re), f2.exactMatch(re)); + checkFilters(name + "/regex", f1.regex("*"), f2.regex("*")); + checkFilters(name + "/regex(ByteString)", f1.regex(re), f2.regex(re)); + checkFilters( + name + "/rangeWithinFamily", + f1.rangeWithinFamily("family"), + f2.rangeWithinFamily("family")); + + break; + } + case "limit": + { + Filters.LimitFilter f1 = (Filters.LimitFilter) original; + Filters.LimitFilter f2 = (Filters.LimitFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(2); + checkFilters( + name + "/cellsPerColumn", f1.cellsPerColumn(100500), f2.cellsPerColumn(100500)); + checkFilters(name + "/cellsPerRow", f1.cellsPerRow(-10), f2.cellsPerRow(-10)); + + break; + } + case "value": + { + Filters.ValueFilter f1 = (Filters.ValueFilter) original; + Filters.ValueFilter f2 = (Filters.ValueFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(6); + checkFilters(name + "/exactMatch", f1.exactMatch("x"), f2.exactMatch("x")); + checkFilters(name + "/exactMatch(ByteString)", f1.exactMatch(re), f2.exactMatch(re)); + checkFilters(name + "/range", f1.range(), f2.range()); + checkFilters(name + "/regex", f1.regex("*"), f2.regex("*")); + checkFilters(name + "/regex(ByteString)", f1.regex(re), f2.regex(re)); + checkFilters(name + "/strip", f1.strip(), f2.strip()); + + break; + } + case "offset": + { + Filters.OffsetFilter f1 = (Filters.OffsetFilter) original; + Filters.OffsetFilter f2 = (Filters.OffsetFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(1); + checkFilters(name + "/cellsPerRow", f1.cellsPerRow(100500), f2.cellsPerRow(100500)); + + break; + } + case "key": + { + Filters.KeyFilter f1 = (Filters.KeyFilter) original; + Filters.KeyFilter f2 = (Filters.KeyFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(5); + checkFilters(name + "/exactMatch", f1.exactMatch("a"), f2.exactMatch("a")); + checkFilters(name + "/exactMatch(ByteString)", f1.exactMatch(re), f2.exactMatch(re)); + checkFilters(name + "/regex", f1.regex("a"), f2.regex("a")); + checkFilters(name + "/regex(ByteString)", f1.regex(re), f2.regex(re)); + checkFilters(name + "/sample", f1.sample(0.1), f2.sample(0.1)); + + break; + } + case "timestamp": + { + Filters.TimestampFilter f1 = (Filters.TimestampFilter) original; + Filters.TimestampFilter f2 = (Filters.TimestampFilter) deserialized; + + assertThat(numberOfMethods).isEqualTo(2); + checkFilters(name + "/exact", f1.exact(100500L), f2.exact(100500L)); + checkFilters(name + "/range", f1.range(), f2.range()); + + break; + } + default: + fail("Untested filter: " + name); + } + } + + private static void checkClassDeclaresSerialVersionUid(Class cls) { + String uid = "serialVersionUID"; + for (Field field : cls.getDeclaredFields()) { + if (field.getName() == uid) { + int modifiers = field.getModifiers(); + assertWithMessage(field + " is not static").that(Modifier.isStatic(modifiers)).isTrue(); + assertWithMessage(field + " is not final").that(Modifier.isFinal(modifiers)).isTrue(); + assertWithMessage(field + " is not private").that(Modifier.isPrivate(modifiers)).isTrue(); + assertWithMessage(field + " must be long") + .that(field.getType().getSimpleName()) + .isEqualTo("long"); + return; + } + } + fail(cls + " does not declare serialVersionUID"); + } + + private static void checkClassDoesNotContainNonStaticFields(Class cls, Field[] fields) { + for (Field field : fields) { + assertWithMessage(cls + " has a non-static field '" + field + "'") + .that(Modifier.isStatic(field.getModifiers())) + .isTrue(); + } + } + + private static Object serializeDeserialize(Object obj) + throws IOException, ClassNotFoundException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ObjectOutputStream outStream = new ObjectOutputStream(bos)) { + outStream.writeObject(obj); + } + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + try (ObjectInputStream inStream = new ObjectInputStream(bis)) { + return inStream.readObject(); + } + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java index b9e6f9e7ba..b0a8be33c9 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java @@ -28,7 +28,10 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +@RunWith(JUnit4.class) public class ReadModifyWriteRowTest { private static final String PROJECT_ID = "fake-project"; private static final String INSTANCE_ID = "fake-instance"; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowCellTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowCellTest.java index 96c78c2c33..f98e01e785 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowCellTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowCellTest.java @@ -19,6 +19,14 @@ import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; +import com.google.protobuf.LazyStringArrayList; +import com.google.protobuf.UnmodifiableLazyStringList; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import org.junit.Test; @@ -98,4 +106,46 @@ public void compareTest() { RowCell.create("family1", col1, timestamp2, labels1, value1))) .isEqualTo(1); } + + @Test + public void testSerialization() throws IOException, ClassNotFoundException { + LazyStringArrayList lazyList = new LazyStringArrayList(); + lazyList.add("lazy"); + lazyList.add("very lazy"); + List[] labelLists = { + Arrays.asList("str1", "str2", "str3"), + ImmutableList.of("string1", "string2"), + new UnmodifiableLazyStringList(lazyList), + new UnmodifiableLazyStringList(LazyStringArrayList.EMPTY) + }; + + for (int i = 0; i < labelLists.length; i++) { + String family = "family_" + i; + ByteString col = ByteString.copyFromUtf8("col_" + i); + long timestamp = 1000L * (i + 1); + List labels = labelLists[i]; + ByteString value = ByteString.copyFromUtf8("value_" + i); + RowCell cell = RowCell.create(family, col, timestamp, labels, value); + RowCell deserialized = (RowCell) serializeDeserialize(cell); + + assertThat(cell.getFamily()).isEqualTo(deserialized.getFamily()); + assertThat(cell.getQualifier()).isEqualTo(deserialized.getQualifier()); + assertThat(cell.getTimestamp()).isEqualTo(deserialized.getTimestamp()); + assertThat(cell.getLabels()).isEqualTo(deserialized.getLabels()); + assertThat(cell.getValue()).isEqualTo(deserialized.getValue()); + } + } + + private static Object serializeDeserialize(Object obj) + throws IOException, ClassNotFoundException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try (ObjectOutputStream outStream = new ObjectOutputStream(bos)) { + outStream.writeObject(obj); + } + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + try (ObjectInputStream inStream = new ObjectInputStream(bis)) { + return inStream.readObject(); + } + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java index b823930fb6..b9c5c96167 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubTest.java @@ -33,8 +33,13 @@ import com.google.protobuf.ByteString; import com.google.protobuf.BytesValue; import com.google.protobuf.StringValue; +import io.grpc.Metadata; import io.grpc.Server; import io.grpc.ServerBuilder; +import io.grpc.ServerCall; +import io.grpc.ServerCall.Listener; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; import io.grpc.stub.StreamObserver; import java.io.IOException; import java.net.ServerSocket; @@ -56,6 +61,7 @@ public class EnhancedBigtableStubTest { private static final String APP_PROFILE_ID = "app-profile-id"; private Server server; + private MetadataInterceptor metadataInterceptor; private FakeDataService fakeDataService; private EnhancedBigtableStubSettings defaultSettings; private EnhancedBigtableStub enhancedBigtableStub; @@ -66,8 +72,13 @@ public void setUp() throws IOException, IllegalAccessException, InstantiationExc try (ServerSocket ss = new ServerSocket(0)) { port = ss.getLocalPort(); } + metadataInterceptor = new MetadataInterceptor(); fakeDataService = new FakeDataService(); - server = ServerBuilder.forPort(port).addService(fakeDataService).build(); + server = + ServerBuilder.forPort(port) + .intercept(metadataInterceptor) + .addService(fakeDataService) + .build(); server.start(); defaultSettings = @@ -137,12 +148,39 @@ public void testChannelPrimerConfigured() throws IOException { } } + @Test + public void testUserAgent() throws InterruptedException { + ServerStreamingCallable streamingCallable = + enhancedBigtableStub.createReadRowsCallable(new DefaultRowAdapter()); + + Query request = Query.create("table-id").rowKey("row-key"); + streamingCallable.call(request).iterator().next(); + + assertThat(metadataInterceptor.headers).hasSize(1); + Metadata metadata = metadataInterceptor.headers.take(); + assertThat(metadata.get(Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER))) + .contains("bigtable-java/"); + } + + private static class MetadataInterceptor implements ServerInterceptor { + final BlockingQueue headers = Queues.newLinkedBlockingDeque(); + + @Override + public Listener interceptCall( + ServerCall serverCall, + Metadata metadata, + ServerCallHandler serverCallHandler) { + headers.add(metadata); + return serverCallHandler.startCall(serverCall, metadata); + } + } + private static class FakeDataService extends BigtableGrpc.BigtableImplBase { - final BlockingQueue requests = Queues.newLinkedBlockingDeque(); + final BlockingQueue requests = Queues.newLinkedBlockingDeque(); @SuppressWarnings("unchecked") - T popLastRequest() throws InterruptedException { - return (T) requests.poll(1, TimeUnit.SECONDS); + ReadRowsRequest popLastRequest() throws InterruptedException { + return requests.poll(1, TimeUnit.SECONDS); } @Override diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java index f3e97584e3..d1740fbfcd 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java @@ -22,8 +22,12 @@ import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.emulator.v2.Emulator; +import com.google.common.base.Strings; +import java.nio.file.Paths; public class EmulatorEnv extends AbstractTestEnv { + private static final String EMULATOR_OVERRIDE_PROPERTY_NAME = "bigtable.emulator-path"; + private static final String PROJECT_ID = "fake-project"; private static final String INSTANCE_ID = "fake-instance"; private static final String TABLE_ID = "default-table"; @@ -42,7 +46,12 @@ private EmulatorEnv() {} @Override void start() throws Exception { - emulator = Emulator.createBundled(); + String overridePath = System.getProperty(EMULATOR_OVERRIDE_PROPERTY_NAME); + if (!Strings.isNullOrEmpty(overridePath)) { + emulator = Emulator.createFromPath(Paths.get(overridePath)); + } else { + emulator = Emulator.createBundled(); + } emulator.start(); dataSettings = diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index a27712ee6a..eff3d8ce9a 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 1.15.0 + 1.16.0 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.15.0 + 1.16.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index bb8015da65..1be1cf84e9 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 1.15.0 + 1.16.0 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.15.0 + 1.16.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom import diff --git a/pom.xml b/pom.xml index ed134153c6..b2fecd978b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 1.15.0 + 1.16.0 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable @@ -162,7 +162,7 @@ com.google.cloud google-cloud-conformance-tests - 0.0.11 + 0.0.12 + 1.16.0 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 1.15.0 + 1.16.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index a84a85e002..4c3c3371b6 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 1.15.0 + 1.16.0 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 1.15.0 + 1.16.0 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 1.15.0 + 1.16.0 pom import com.google.cloud google-cloud-bigtable-bom - 1.15.0 + 1.16.0 pom import diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index f1b8e7e961..d6bd01bd61 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -14,7 +14,7 @@ com.google.cloud.samples shared-configuration - 1.0.18 + 1.0.21 @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigtable - 1.14.0 + 1.15.0 diff --git a/samples/pom.xml b/samples/pom.xml index ac04bb8af3..1e76b96358 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -18,7 +18,7 @@ com.google.cloud.samples shared-configuration - 1.0.18 + 1.0.21 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index bb697bfcdc..dc81eb9d44 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -14,7 +14,7 @@ com.google.cloud.samples shared-configuration - 1.0.18 + 1.0.21 @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 1.15.0 + 1.16.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 0e34671195..dcfbdd8db0 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -14,7 +14,7 @@ com.google.cloud.samples shared-configuration - 1.0.18 + 1.0.21 @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 10.0.0 + 12.0.0 pom import diff --git a/synth.metadata b/synth.metadata index 4d2ddc577b..62814f52a7 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigtable.git", - "sha": "a56a0f8c9caf675b68d02587b042e1feeb261ccb" + "sha": "000987fa784809a4f269203e604b9184099ca8ce" } }, { @@ -19,7 +19,7 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "019c7168faa0e56619f792693a8acdb30d6de19b" + "sha": "8a7a3021fe97aa0a3641db642fe2b767f1c8110f" } } ], @@ -48,8 +48,10 @@ ".github/ISSUE_TEMPLATE/feature_request.md", ".github/ISSUE_TEMPLATE/support_request.md", ".github/PULL_REQUEST_TEMPLATE.md", + ".github/readme/synth.py", ".github/release-please.yml", ".github/trusted-contribution.yml", + ".github/workflows/auto-release.yaml", ".github/workflows/ci.yaml", ".github/workflows/samples.yaml", ".kokoro/build.bat", @@ -59,6 +61,7 @@ ".kokoro/common.sh", ".kokoro/continuous/common.cfg", ".kokoro/continuous/java8.cfg", + ".kokoro/continuous/readme.cfg", ".kokoro/dependencies.sh", ".kokoro/linkage-monitor.sh", ".kokoro/nightly/common.cfg", @@ -78,6 +81,7 @@ ".kokoro/presubmit/java8.cfg", ".kokoro/presubmit/linkage-monitor.cfg", ".kokoro/presubmit/lint.cfg", + ".kokoro/readme.sh", ".kokoro/release/bump_snapshot.cfg", ".kokoro/release/common.cfg", ".kokoro/release/common.sh", diff --git a/versions.txt b/versions.txt index 3918eab106..6cbd7fef61 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:1.15.0:1.15.0 -grpc-google-cloud-bigtable-admin-v2:1.15.0:1.15.0 -grpc-google-cloud-bigtable-v2:1.15.0:1.15.0 -proto-google-cloud-bigtable-admin-v2:1.15.0:1.15.0 -proto-google-cloud-bigtable-v2:1.15.0:1.15.0 -google-cloud-bigtable-emulator:0.124.0:0.124.0 +google-cloud-bigtable:1.16.0:1.16.0 +grpc-google-cloud-bigtable-admin-v2:1.16.0:1.16.0 +grpc-google-cloud-bigtable-v2:1.16.0:1.16.0 +proto-google-cloud-bigtable-admin-v2:1.16.0:1.16.0 +proto-google-cloud-bigtable-v2:1.16.0:1.16.0 +google-cloud-bigtable-emulator:0.125.0:0.125.0