From d3ba37cbf7c19a113da9f40cf7441622017a8888 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 21 Aug 2019 15:22:40 -0400 Subject: [PATCH 1/4] Bigtable: allow integration test helpers to be shared by data & admin apis --- .../scripts/setup-test-table.sh | 2 +- .../data/v2/it/BulkMutationBatcherIT.java | 11 ++- .../bigtable/data/v2/it/CheckAndMutateIT.java | 5 +- .../bigtable/data/v2/it/MutateRowIT.java | 5 +- .../cloud/bigtable/data/v2/it/ReadIT.java | 5 +- .../data/v2/it/ReadModifyWriteIT.java | 5 +- .../bigtable/data/v2/it/SampleRowsIT.java | 7 +- .../bigtable/data/v2/it/env/TestEnv.java | 41 ---------- .../test_helpers/env/AbstractTestEnv.java | 77 +++++++++++++++++++ .../env/DirectPathEnv.java | 53 +++---------- .../it => test_helpers}/env/EmulatorEnv.java | 16 ++-- .../v2/it => test_helpers}/env/ProdEnv.java | 53 +++---------- .../it => test_helpers}/env/TestEnvRule.java | 12 ++- 13 files changed, 137 insertions(+), 155 deletions(-) delete mode 100644 google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java create mode 100644 google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java rename google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/{data/v2/it => test_helpers}/env/DirectPathEnv.java (76%) rename google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/{data/v2/it => test_helpers}/env/EmulatorEnv.java (88%) rename google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/{data/v2/it => test_helpers}/env/ProdEnv.java (64%) rename google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/{data/v2/it => test_helpers}/env/TestEnvRule.java (90%) diff --git a/google-cloud-clients/google-cloud-bigtable/scripts/setup-test-table.sh b/google-cloud-clients/google-cloud-bigtable/scripts/setup-test-table.sh index 15b564a9d0b2..6321c1154cb4 100755 --- a/google-cloud-clients/google-cloud-bigtable/scripts/setup-test-table.sh +++ b/google-cloud-clients/google-cloud-bigtable/scripts/setup-test-table.sh @@ -43,6 +43,6 @@ for ADMIN_HOST in "${ADMIN_HOSTS[@]}"; do # Ensure that the family exists if ! call_cbt ls "${TABLE_ID}" | grep -q "^$FAMILY\b"; then call_cbt createfamily "${TABLE_ID}" "${FAMILY}" - call_cbt setgcpolicy "${TABLE_ID}" "${FAMILY}" maxversions=1 + call_cbt setgcpolicy "${TABLE_ID}" "${FAMILY}" maxversions=1 maxage=1h fi done diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java index 97d42978ed9d..076e9af2d83b 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BulkMutationBatcherIT.java @@ -17,16 +17,17 @@ import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.BulkMutationBatcher; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.models.RowCell; import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.common.collect.Lists; import com.google.common.truth.Truth; import com.google.protobuf.ByteString; import java.util.List; +import java.util.UUID; import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,12 +42,14 @@ public void test() throws Exception { BigtableDataClient client = testEnvRule.env().getDataClient(); String tableId = testEnvRule.env().getTableId(); String family = testEnvRule.env().getFamilyId(); - String rowPrefix = testEnvRule.env().getRowPrefix(); + String rowPrefix = UUID.randomUUID() + "-"; + long timestamp = System.currentTimeMillis() * 1_000; try (BulkMutationBatcher batcher = client.newBulkMutationBatcher()) { for (int i = 0; i < 10; i++) { batcher.add( - RowMutation.create(tableId, rowPrefix + "-" + i).setCell(family, "q", 10_000, "value")); + RowMutation.create(tableId, rowPrefix + "-" + i) + .setCell(family, "q", timestamp, "value")); } } @@ -59,7 +62,7 @@ public void test() throws Exception { RowCell.create( family, ByteString.copyFromUtf8("q"), - 10_000, + timestamp, Lists.newArrayList(), ByteString.copyFromUtf8("value"))))); } diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java index 53f1a8997dd1..5f532846903e 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/CheckAndMutateIT.java @@ -18,13 +18,14 @@ import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS; import static com.google.common.truth.Truth.assertThat; -import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation; import com.google.cloud.bigtable.data.v2.models.Mutation; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.protobuf.ByteString; +import java.util.UUID; import java.util.concurrent.TimeUnit; import org.junit.ClassRule; import org.junit.Test; @@ -38,8 +39,8 @@ public class CheckAndMutateIT { @Test public void test() throws Exception { String tableId = testEnvRule.env().getTableId(); - String rowKey = testEnvRule.env().getRowPrefix(); String familyId = testEnvRule.env().getFamilyId(); + String rowKey = UUID.randomUUID().toString(); testEnvRule .env() diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java index 545028e14a58..720b6f84f08c 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/MutateRowIT.java @@ -17,11 +17,12 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.protobuf.ByteString; +import java.util.UUID; import java.util.concurrent.TimeUnit; import org.junit.ClassRule; import org.junit.Test; @@ -34,7 +35,7 @@ public class MutateRowIT { @Test public void test() throws Exception { - String rowKey = testEnvRule.env().getRowPrefix() + "testA"; + String rowKey = UUID.randomUUID().toString(); String familyId = testEnvRule.env().getFamilyId(); testEnvRule diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 6f7db9abce54..0e4a5f37aee9 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -23,17 +23,18 @@ import com.google.api.core.SettableApiFuture; import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.StreamController; -import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.models.RowCell; import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -53,7 +54,7 @@ public class ReadIT { @Before public void setUp() { - prefix = testEnvRule.env().getRowPrefix(); + prefix = UUID.randomUUID().toString(); } @Test diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java index 692027db5446..1bf6e137aab9 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadModifyWriteIT.java @@ -17,10 +17,11 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.ReadModifyWriteRow; import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.protobuf.ByteString; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -37,7 +38,7 @@ public class ReadModifyWriteIT { public void test() throws InterruptedException, ExecutionException, TimeoutException { String tableId = testEnvRule.env().getTableId(); String family = testEnvRule.env().getFamilyId(); - String rowKey = testEnvRule.env().getRowPrefix(); + String rowKey = UUID.randomUUID().toString(); Row row = testEnvRule diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java index 20bba81ea4ae..cb0624350905 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/SampleRowsIT.java @@ -20,11 +20,12 @@ import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.it.env.TestEnvRule; import com.google.cloud.bigtable.data.v2.models.KeyOffset; import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.common.collect.Lists; import java.util.List; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -40,14 +41,14 @@ public class SampleRowsIT { @Test public void test() throws InterruptedException, ExecutionException, TimeoutException { BigtableDataClient client = testEnvRule.env().getDataClient(); + String rowPrefix = UUID.randomUUID().toString(); // Create some data so that sample row keys has something to show List> futures = Lists.newArrayList(); for (int i = 0; i < 10; i++) { ApiFuture future = client.mutateRowAsync( - RowMutation.create( - testEnvRule.env().getTableId(), testEnvRule.env().getRowPrefix() + "-" + i) + RowMutation.create(testEnvRule.env().getTableId(), rowPrefix + "-" + i) .setCell(testEnvRule.env().getFamilyId(), "", "value")); futures.add(future); } diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java deleted file mode 100644 index 69c70efc3189..000000000000 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnv.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.data.v2.it.env; - -import com.google.cloud.bigtable.data.v2.BigtableDataClient; - -/** - * Defines the interface of a target environment. - * - *

This allows for integration tests to run against either production or an emulator. - */ -public interface TestEnv { - void start() throws Exception; - - void stop() throws Exception; - - BigtableDataClient getDataClient(); - - String getProjectId(); - - String getInstanceId(); - - String getTableId(); - - String getFamilyId(); - - String getRowPrefix(); -} diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java new file mode 100644 index 000000000000..da2e01d58137 --- /dev/null +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java @@ -0,0 +1,77 @@ +/* + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.test_helpers.env; + +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import org.threeten.bp.Instant; +import org.threeten.bp.temporal.ChronoUnit; + +/** + * Defines the interface of a target environment. + * + *

This allows for integration tests to run against either production or an emulator. + */ +public abstract class AbstractTestEnv { + private static final String PREFIX = "temp-"; + + public abstract void start() throws Exception; + + public abstract void stop() throws Exception; + + public abstract BigtableDataClient getDataClient(); + + public abstract BigtableTableAdminClient getTableAdminClient(); + + public abstract String getProjectId(); + + public abstract String getInstanceId(); + + public abstract String getTableId(); + + public String getFamilyId() { + return "cf"; + } + + public String generateTableId(String suffix) { + return newPrefix() + "-" + suffix; + } + + private static String newPrefix() { + return newPrefix(Instant.now()); + } + + private static String newPrefix(Instant instant) { + return String.format(PREFIX + "015%d", instant.getEpochSecond()); + } + + void cleanUpStale() { + cleanupStaleTables(); + } + + private void cleanupStaleTables() { + String stalePrefix = newPrefix(Instant.now().minus(1, ChronoUnit.DAYS)); + + for (String tableId : getTableAdminClient().listTables()) { + if (!tableId.startsWith(PREFIX)) { + continue; + } + if (stalePrefix.compareTo(tableId) > 0) { + getTableAdminClient().deleteTable(tableId); + } + } + } +} diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/DirectPathEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java similarity index 76% rename from google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/DirectPathEnv.java rename to google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java index cf3356ee1f30..806fedb2e9ca 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/DirectPathEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java @@ -13,24 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.data.v2.it.env; +package com.google.cloud.bigtable.test_helpers.env; -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.data.v2.BigtableDataClient; import com.google.cloud.bigtable.data.v2.BigtableDataSettings; -import com.google.cloud.bigtable.data.v2.models.Query; -import com.google.cloud.bigtable.data.v2.models.Row; -import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; -import com.google.common.collect.Lists; import java.io.IOException; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** * Test environment that uses an existing bigtable table in the directpath environment. @@ -48,7 +37,7 @@ *

  • {@code bigtable.table} * */ -public class DirectPathEnv implements TestEnv { +class DirectPathEnv extends AbstractTestEnv { // TODO(igorbernstein): move direct path conditional logic to gax private static final String DIRECT_PATH_ENV_VAR = "GOOGLE_CLOUD_ENABLE_DIRECT_PATH"; private static final String DIRECT_PATH_END_POINT = @@ -62,10 +51,9 @@ public class DirectPathEnv implements TestEnv { private final String projectId; private final String instanceId; private final String tableId; - private static final String FAMILY_ID = "cf"; - private String rowPrefix; private BigtableDataClient dataClient; + private BigtableTableAdminClient tableAdminClient; static DirectPathEnv create() { return new DirectPathEnv( @@ -81,7 +69,6 @@ public DirectPathEnv( this.projectId = projectId; this.instanceId = instanceId; this.tableId = tableId; - this.rowPrefix = UUID.randomUUID() + "-"; } @Override @@ -105,12 +92,13 @@ public void start() throws IOException { } dataClient = BigtableDataClient.create(settingsBuilder.build()); + tableAdminClient = BigtableTableAdminClient.create(projectId, instanceId); } @Override public void stop() throws Exception { - deleteRows(); dataClient.close(); + tableAdminClient.close(); } @Override @@ -118,6 +106,11 @@ public BigtableDataClient getDataClient() { return dataClient; } + @Override + public BigtableTableAdminClient getTableAdminClient() { + return tableAdminClient; + } + @Override public String getProjectId() { return projectId; @@ -133,30 +126,6 @@ public String getTableId() { return tableId; } - @Override - public String getFamilyId() { - return FAMILY_ID; - } - - @Override - public String getRowPrefix() { - return rowPrefix; - } - - private void deleteRows() throws InterruptedException, ExecutionException, TimeoutException { - Query query = Query.create(tableId).prefix(rowPrefix); - - List> futures = Lists.newArrayList(); - ServerStream rows = dataClient.readRows(query); - for (Row row : rows) { - ApiFuture future = - dataClient.mutateRowAsync(RowMutation.create(tableId, row.getKey()).deleteRow()); - futures.add(future); - } - - ApiFutures.allAsList(futures).get(10, TimeUnit.MINUTES); - } - private static String getRequiredProperty(String prop) { String value = System.getProperty(prop); if (value == null || value.isEmpty()) { diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java similarity index 88% rename from google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java rename to google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java index d6abb9ca5ae3..f25d6d8fc415 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/EmulatorEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.data.v2.it.env; +package com.google.cloud.bigtable.test_helpers.env; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; @@ -22,11 +22,10 @@ import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.emulator.v2.Emulator; -public class EmulatorEnv implements TestEnv { +class EmulatorEnv extends AbstractTestEnv { private static final String PROJECT_ID = "fake-project"; private static final String INSTANCE_ID = "fake-instance"; private static final String TABLE_ID = "default-table"; - private static final String FAMILY_ID = "cf"; private Emulator emulator; private BigtableTableAdminClient tableAdminClient; @@ -44,7 +43,7 @@ public void start() throws Exception { BigtableDataClient.create( BigtableDataSettings.newBuilderForEmulator(emulator.getPort()).build()); - tableAdminClient.createTable(CreateTableRequest.of(TABLE_ID).addFamily(FAMILY_ID)); + tableAdminClient.createTable(CreateTableRequest.of(TABLE_ID).addFamily(getFamilyId())); } @Override @@ -69,18 +68,13 @@ public String getTableId() { return TABLE_ID; } - @Override - public String getRowPrefix() { - return "fake-"; - } - @Override public BigtableDataClient getDataClient() { return dataClient; } @Override - public String getFamilyId() { - return FAMILY_ID; + public BigtableTableAdminClient getTableAdminClient() { + return tableAdminClient; } } diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java similarity index 64% rename from google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java rename to google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java index 6f3f333350dd..0de1e2f60bd4 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/ProdEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java @@ -13,22 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.data.v2.it.env; +package com.google.cloud.bigtable.test_helpers.env; -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.api.gax.rpc.ServerStream; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.data.v2.BigtableDataClient; -import com.google.cloud.bigtable.data.v2.models.Query; -import com.google.cloud.bigtable.data.v2.models.Row; -import com.google.cloud.bigtable.data.v2.models.RowMutation; -import com.google.common.collect.Lists; import java.io.IOException; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** * Test environment that uses an existing bigtable table. The table must have a pre-existing family @@ -40,7 +29,7 @@ *
  • {@code bigtable.table} * */ -public class ProdEnv implements TestEnv { +class ProdEnv extends AbstractTestEnv { private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; private static final String TABLE_PROPERTY_NAME = "bigtable.table"; @@ -48,10 +37,9 @@ public class ProdEnv implements TestEnv { private final String projectId; private final String instanceId; private final String tableId; - private static final String FAMILY_ID = "cf"; - private String rowPrefix; private BigtableDataClient dataClient; + private BigtableTableAdminClient tableAdminClient; static ProdEnv fromSystemProperties() { return new ProdEnv( @@ -64,18 +52,18 @@ public ProdEnv(String projectId, String instanceId, String tableId) { this.projectId = projectId; this.instanceId = instanceId; this.tableId = tableId; - this.rowPrefix = UUID.randomUUID() + "-"; } @Override public void start() throws IOException { dataClient = BigtableDataClient.create(projectId, instanceId); + tableAdminClient = BigtableTableAdminClient.create(projectId, instanceId); } @Override public void stop() throws Exception { - deleteRows(); dataClient.close(); + tableAdminClient.close(); } @Override @@ -83,6 +71,11 @@ public BigtableDataClient getDataClient() { return dataClient; } + @Override + public BigtableTableAdminClient getTableAdminClient() { + return tableAdminClient; + } + @Override public String getProjectId() { return projectId; @@ -98,30 +91,6 @@ public String getTableId() { return tableId; } - @Override - public String getFamilyId() { - return FAMILY_ID; - } - - @Override - public String getRowPrefix() { - return rowPrefix; - } - - private void deleteRows() throws InterruptedException, ExecutionException, TimeoutException { - Query query = Query.create(tableId).prefix(rowPrefix); - - List> futures = Lists.newArrayList(); - ServerStream rows = dataClient.readRows(query); - for (Row row : rows) { - ApiFuture future = - dataClient.mutateRowAsync(RowMutation.create(tableId, row.getKey()).deleteRow()); - futures.add(future); - } - - ApiFutures.allAsList(futures).get(10, TimeUnit.MINUTES); - } - private static String getRequiredProperty(String prop) { String value = System.getProperty(prop); if (value == null || value.isEmpty()) { diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java similarity index 90% rename from google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java rename to google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java index b2f7ca40e567..dc03ec91f7ce 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/env/TestEnvRule.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.google.cloud.bigtable.data.v2.it.env; +package com.google.cloud.bigtable.test_helpers.env; import java.util.logging.Level; import java.util.logging.Logger; @@ -41,7 +41,7 @@ public class TestEnvRule extends ExternalResource { private static final String ENV_PROPERTY = "bigtable.env"; - private TestEnv testEnv; + private AbstractTestEnv testEnv; @Override protected void before() throws Throwable { @@ -68,6 +68,12 @@ protected void before() throws Throwable { @Override protected void after() { + try { + env().cleanUpStale(); + } catch (Exception e) { + LOGGER.log(Level.WARNING, "Failed to cleanup environment", e); + } + try { testEnv.stop(); } catch (Exception e) { @@ -76,7 +82,7 @@ protected void after() { testEnv = null; } - public TestEnv env() { + public AbstractTestEnv env() { return testEnv; } } From 16cfada6e3b58e2892e71a35f50a1f67617e1c5c Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 22 Aug 2019 12:18:38 -0400 Subject: [PATCH 2/4] Allow test environment to be inspected --- .../bigtable/test_helpers/env/AbstractTestEnv.java | 4 ++-- .../bigtable/test_helpers/env/DirectPathEnv.java | 8 ++++---- .../cloud/bigtable/test_helpers/env/EmulatorEnv.java | 12 +++++++++--- .../cloud/bigtable/test_helpers/env/ProdEnv.java | 6 +++--- .../cloud/bigtable/test_helpers/env/TestEnvRule.java | 2 +- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java index da2e01d58137..3e959e56365c 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/AbstractTestEnv.java @@ -28,9 +28,9 @@ public abstract class AbstractTestEnv { private static final String PREFIX = "temp-"; - public abstract void start() throws Exception; + abstract void start() throws Exception; - public abstract void stop() throws Exception; + abstract void stop() throws Exception; public abstract BigtableDataClient getDataClient(); diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java index 806fedb2e9ca..143f61adc59a 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/DirectPathEnv.java @@ -37,7 +37,7 @@ *
  • {@code bigtable.table} * */ -class DirectPathEnv extends AbstractTestEnv { +public class DirectPathEnv extends AbstractTestEnv { // TODO(igorbernstein): move direct path conditional logic to gax private static final String DIRECT_PATH_ENV_VAR = "GOOGLE_CLOUD_ENABLE_DIRECT_PATH"; private static final String DIRECT_PATH_END_POINT = @@ -63,7 +63,7 @@ static DirectPathEnv create() { getRequiredProperty(TABLE_PROPERTY_NAME)); } - public DirectPathEnv( + private DirectPathEnv( boolean directPathEnabled, String projectId, String instanceId, String tableId) { this.directPathEnabled = directPathEnabled; this.projectId = projectId; @@ -72,7 +72,7 @@ public DirectPathEnv( } @Override - public void start() throws IOException { + void start() throws IOException { BigtableDataSettings.Builder settingsBuilder = BigtableDataSettings.newBuilder().setProjectId(projectId).setInstanceId(instanceId); @@ -96,7 +96,7 @@ public void start() throws IOException { } @Override - public void stop() throws Exception { + void stop() throws Exception { dataClient.close(); tableAdminClient.close(); } diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java index f25d6d8fc415..da5287ce13ba 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/EmulatorEnv.java @@ -22,7 +22,7 @@ import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.emulator.v2.Emulator; -class EmulatorEnv extends AbstractTestEnv { +public class EmulatorEnv extends AbstractTestEnv { private static final String PROJECT_ID = "fake-project"; private static final String INSTANCE_ID = "fake-instance"; private static final String TABLE_ID = "default-table"; @@ -31,8 +31,14 @@ class EmulatorEnv extends AbstractTestEnv { private BigtableTableAdminClient tableAdminClient; private BigtableDataClient dataClient; + public static EmulatorEnv createBundled() { + return new EmulatorEnv(); + } + + private EmulatorEnv() {} + @Override - public void start() throws Exception { + void start() throws Exception { emulator = Emulator.createBundled(); emulator.start(); @@ -47,7 +53,7 @@ public void start() throws Exception { } @Override - public void stop() throws Exception { + void stop() throws Exception { tableAdminClient.close(); dataClient.close(); emulator.stop(); diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java index 0de1e2f60bd4..daecf9df7b56 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/ProdEnv.java @@ -48,20 +48,20 @@ static ProdEnv fromSystemProperties() { getRequiredProperty(TABLE_PROPERTY_NAME)); } - public ProdEnv(String projectId, String instanceId, String tableId) { + private ProdEnv(String projectId, String instanceId, String tableId) { this.projectId = projectId; this.instanceId = instanceId; this.tableId = tableId; } @Override - public void start() throws IOException { + void start() throws IOException { dataClient = BigtableDataClient.create(projectId, instanceId); tableAdminClient = BigtableTableAdminClient.create(projectId, instanceId); } @Override - public void stop() throws Exception { + void stop() throws Exception { dataClient.close(); tableAdminClient.close(); } diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java index dc03ec91f7ce..081357c80ce4 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java @@ -49,7 +49,7 @@ protected void before() throws Throwable { switch (env) { case "emulator": - testEnv = new EmulatorEnv(); + testEnv = EmulatorEnv.createBundled(); break; case "prod": testEnv = ProdEnv.fromSystemProperties(); From cddc1bd1fa8208690e924bf07159a0a8b2756780 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 22 Aug 2019 12:19:20 -0400 Subject: [PATCH 3/4] Run most of table admin tests in emulator --- .../google-cloud-bigtable/pom.xml | 1 + .../v2/it/BigtableTableAdminClientIT.java | 269 ++++++------------ 2 files changed, 95 insertions(+), 175 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigtable/pom.xml b/google-cloud-clients/google-cloud-bigtable/pom.xml index 585fc36b7904..fcaef8e9c031 100644 --- a/google-cloud-clients/google-cloud-bigtable/pom.xml +++ b/google-cloud-clients/google-cloud-bigtable/pom.xml @@ -105,6 +105,7 @@ emulator com.google.cloud.bigtable.data.v2.it.** + com.google.cloud.bigtable.admin.v2.it.** target/failsafe-reports/failsafe-summary-emulator-it.xml diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java index 4097f98ce870..074f19608373 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java @@ -16,6 +16,7 @@ package com.google.cloud.bigtable.admin.v2.it; import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES; +import static com.google.common.truth.TruthJUnit.assume; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -30,85 +31,48 @@ import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule; import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest; import com.google.cloud.bigtable.admin.v2.models.Table; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; +import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv; +import com.google.cloud.bigtable.test_helpers.env.TestEnvRule; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; -import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; -import org.junit.AfterClass; -import org.junit.AssumptionViolatedException; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; +import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; import org.threeten.bp.Duration; public class BigtableTableAdminClientIT { - private static final Logger LOGGER = Logger.getLogger(BigtableTableAdminClientIT.class.getName()); + @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule(); + @Rule public final TestName testNameRule = new TestName(); - private static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; - private static List missingProperties; + private BigtableTableAdminClient tableAdmin; + private String tableId; + private boolean skipTableDelete; - private static BigtableTableAdminClient tableAdmin; - private static String prefix; - - @BeforeClass - public static void createClient() throws IOException { - missingProperties = Lists.newArrayList(); - - String targetProject = System.getProperty(PROJECT_PROPERTY_NAME); - if (targetProject == null) { - missingProperties.add(PROJECT_PROPERTY_NAME); - } - - String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME); - if (targetInstance == null) { - missingProperties.add(INSTANCE_PROPERTY_NAME); - } - - if (!missingProperties.isEmpty()) { - LOGGER.warning("Missing properties: " + Joiner.on(",").join(missingProperties)); - return; - } - - tableAdmin = BigtableTableAdminClient.create(targetProject, targetInstance); - - // Setup a prefix to avoid collisions between concurrent test runs - prefix = String.format("020%d", System.currentTimeMillis()); - - // Cleanup old tables, under normal circumstances this will do nothing - String stalePrefix = - String.format("020%d", System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1)); - for (String tableId : tableAdmin.listTables()) { - if (stalePrefix.compareTo(tableId) > 0) { - tableAdmin.deleteTable(tableId); - } - } + @Before + public void setUp() { + tableAdmin = testEnvRule.env().getTableAdminClient(); + tableId = testEnvRule.env().generateTableId(testNameRule.getMethodName()); } - @AfterClass - public static void closeClient() { - if (tableAdmin != null) { - tableAdmin.close(); - } - } - - @Before - public void setup() { - if (tableAdmin == null) { - throw new AssumptionViolatedException( - "Required properties are not set, skipping integration tests."); + @After + public void tearDown() { + if (!skipTableDelete) { + testEnvRule.env().getTableAdminClient().deleteTable(tableId); } } @Test public void createTable() { - String tableId = getTableId("adminCreateTest"); + assume() + .withMessage("Emulator doesn't return proper responses for CreateTable") + .that(testEnvRule.env()) + .isNotInstanceOf(EmulatorEnv.class); + CreateTableRequest createTableReq = CreateTableRequest.of(tableId) .addFamily("cf1") @@ -116,153 +80,108 @@ public void createTable() { .addSplit(ByteString.copyFromUtf8("b")) .addSplit(ByteString.copyFromUtf8("q")); - try { - Table tableResponse = tableAdmin.createTable(createTableReq); - assertNotNull(tableResponse); - assertEquals(tableId, tableResponse.getId()); + Table tableResponse = tableAdmin.createTable(createTableReq); + assertEquals(tableId, tableResponse.getId()); - Map columnFamilyById = Maps.newHashMap(); - for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) { - columnFamilyById.put(columnFamily.getId(), columnFamily); - } - assertEquals(2, tableResponse.getColumnFamilies().size()); - assertFalse(columnFamilyById.get("cf1").hasGCRule()); - assertTrue(columnFamilyById.get("cf2").hasGCRule()); - assertEquals(10, ((VersionRule) columnFamilyById.get("cf2").getGCRule()).getMaxVersions()); - } finally { - tableAdmin.deleteTable(tableId); + Map columnFamilyById = Maps.newHashMap(); + for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) { + columnFamilyById.put(columnFamily.getId(), columnFamily); } + assertEquals(2, tableResponse.getColumnFamilies().size()); + assertFalse(columnFamilyById.get("cf1").hasGCRule()); + assertTrue(columnFamilyById.get("cf2").hasGCRule()); + assertEquals(10, ((VersionRule) columnFamilyById.get("cf2").getGCRule()).getMaxVersions()); } @Test public void modifyFamilies() { - String tableId = getTableId("adminModifyFamTest"); - ModifyColumnFamiliesRequest modifyFamiliesReq = ModifyColumnFamiliesRequest.of(tableId); - - modifyFamiliesReq - .addFamily("mf1") - .addFamily("mf2", GCRULES.maxAge(Duration.ofSeconds(1000, 20000))) - .updateFamily( - "mf1", - GCRULES - .union() - .rule(GCRULES.maxAge(Duration.ofSeconds(100))) - .rule(GCRULES.maxVersions(1))) - .addFamily( - "mf3", - GCRULES - .intersection() - .rule(GCRULES.maxAge(Duration.ofSeconds(2000))) - .rule(GCRULES.maxVersions(10))) - .addFamily("mf4", GCRULES.intersection().rule(GCRULES.maxAge(Duration.ofSeconds(360)))) - .addFamily("mf5") - .addFamily("mf6") - .dropFamily("mf5") - .dropFamily("mf6") - .addFamily("mf7"); - - try { - tableAdmin.createTable(CreateTableRequest.of(tableId)); - Table tableResponse = tableAdmin.modifyFamilies(modifyFamiliesReq); + tableAdmin.createTable(CreateTableRequest.of(tableId)); - Map columnFamilyById = Maps.newHashMap(); - for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) { - columnFamilyById.put(columnFamily.getId(), columnFamily); - } - assertEquals(5, columnFamilyById.size()); - assertNotNull(columnFamilyById.get("mf1")); - assertNotNull(columnFamilyById.get("mf2")); - assertEquals(2, ((UnionRule) columnFamilyById.get("mf1").getGCRule()).getRulesList().size()); - assertEquals( - 1000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getSeconds()); - assertEquals( - 20000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getNano()); - assertEquals( - 2, ((IntersectionRule) columnFamilyById.get("mf3").getGCRule()).getRulesList().size()); - assertEquals( - 360, ((DurationRule) columnFamilyById.get("mf4").getGCRule()).getMaxAge().getSeconds()); - assertNotNull(columnFamilyById.get("mf7")); - } finally { - tableAdmin.deleteTable(tableId); + ModifyColumnFamiliesRequest modifyFamiliesReq = + ModifyColumnFamiliesRequest.of(tableId) + .addFamily("mf1") + .addFamily("mf2", GCRULES.maxAge(Duration.ofSeconds(1000, 20000))) + .updateFamily( + "mf1", + GCRULES + .union() + .rule(GCRULES.maxAge(Duration.ofSeconds(100))) + .rule(GCRULES.maxVersions(1))) + .addFamily( + "mf3", + GCRULES + .intersection() + .rule(GCRULES.maxAge(Duration.ofSeconds(2000))) + .rule(GCRULES.maxVersions(10))) + .addFamily("mf4", GCRULES.intersection().rule(GCRULES.maxAge(Duration.ofSeconds(360)))) + .addFamily("mf5") + .addFamily("mf6") + .dropFamily("mf5") + .dropFamily("mf6") + .addFamily("mf7"); + + Table tableResponse = tableAdmin.modifyFamilies(modifyFamiliesReq); + + Map columnFamilyById = Maps.newHashMap(); + for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) { + columnFamilyById.put(columnFamily.getId(), columnFamily); } + assertEquals(5, columnFamilyById.size()); + assertNotNull(columnFamilyById.get("mf1")); + assertNotNull(columnFamilyById.get("mf2")); + assertEquals(2, ((UnionRule) columnFamilyById.get("mf1").getGCRule()).getRulesList().size()); + assertEquals( + 1000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getSeconds()); + assertEquals( + 20000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getNano()); + assertEquals( + 2, ((IntersectionRule) columnFamilyById.get("mf3").getGCRule()).getRulesList().size()); + assertEquals( + 360, ((DurationRule) columnFamilyById.get("mf4").getGCRule()).getMaxAge().getSeconds()); + assertNotNull(columnFamilyById.get("mf7")); } @Test public void deleteTable() { - String tableId = getTableId("adminDeleteTest"); tableAdmin.createTable(CreateTableRequest.of(tableId)); tableAdmin.deleteTable(tableId); + skipTableDelete = true; } @Test public void getTable() { - String tableId = getTableId("adminGetTest"); - - try { - tableAdmin.createTable(CreateTableRequest.of(tableId)); - Table tableResponse = tableAdmin.getTable(tableId); - assertNotNull(tableResponse); - assertEquals(tableId, tableResponse.getId()); - } finally { - tableAdmin.deleteTable(tableId); - } + tableAdmin.createTable(CreateTableRequest.of(tableId)); + Table tableResponse = tableAdmin.getTable(tableId); + assertNotNull(tableResponse); + assertEquals(tableId, tableResponse.getId()); } @Test public void listTables() { - String tableId = getTableId("adminListTest"); - - try { - tableAdmin.createTable(CreateTableRequest.of(tableId)); - List tables = tableAdmin.listTables(); - assertNotNull(tables); - assertFalse("List tables did not return any tables", tables.isEmpty()); - } finally { - tableAdmin.deleteTable(tableId); - } + tableAdmin.createTable(CreateTableRequest.of(tableId)); + List tables = tableAdmin.listTables(); + assertNotNull(tables); + assertFalse("List tables did not return any tables", tables.isEmpty()); } @Test public void listTablesAsync() throws Exception { - String tableId = getTableId("adminListAsyncTest"); - - try { - tableAdmin.createTable(CreateTableRequest.of(tableId)); - List tables = tableAdmin.listTablesAsync().get(); - assertNotNull(tables); - assertFalse("List tables did not return any tables", tables.isEmpty()); - } finally { - tableAdmin.deleteTable(tableId); - } + tableAdmin.createTable(CreateTableRequest.of(tableId)); + List tables = tableAdmin.listTablesAsync().get(); + assertNotNull(tables); + assertFalse("List tables did not return any tables", tables.isEmpty()); } - @Ignore @Test public void dropRowRange() { - String tableId = getTableId("adminDropRowrangeTest"); - - try { - tableAdmin.createTable(CreateTableRequest.of(tableId)); - tableAdmin.dropRowRange(tableId, "rowPrefix"); - tableAdmin.dropAllRows(tableId); - } finally { - tableAdmin.deleteTable(tableId); - } + tableAdmin.createTable(CreateTableRequest.of(tableId)); + tableAdmin.dropRowRange(tableId, "rowPrefix"); + tableAdmin.dropAllRows(tableId); } @Test public void awaitReplication() { - String tableId = getTableId("adminConsistencyTest"); - - try { - tableAdmin.createTable(CreateTableRequest.of(tableId)); - tableAdmin.awaitReplication(tableId); - } finally { - tableAdmin.deleteTable(tableId); - } - } - - private static String getTableId(String name) { - return prefix + "-" + name; + tableAdmin.createTable(CreateTableRequest.of(tableId)); + tableAdmin.awaitReplication(tableId); } } From 56cdd56b14c098a1e14730e4195908d66dc77653 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 22 Aug 2019 12:29:44 -0400 Subject: [PATCH 4/4] fix cleanup --- .../bigtable/admin/v2/it/BigtableTableAdminClientIT.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java index 074f19608373..142a3c8cfe01 100644 --- a/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java +++ b/google-cloud-clients/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; @@ -51,7 +52,6 @@ public class BigtableTableAdminClientIT { private BigtableTableAdminClient tableAdmin; private String tableId; - private boolean skipTableDelete; @Before public void setUp() { @@ -61,8 +61,10 @@ public void setUp() { @After public void tearDown() { - if (!skipTableDelete) { + try { testEnvRule.env().getTableAdminClient().deleteTable(tableId); + } catch (NotFoundException e) { + // table was deleted in test or was never created. Carry on } } @@ -145,7 +147,6 @@ public void modifyFamilies() { public void deleteTable() { tableAdmin.createTable(CreateTableRequest.of(tableId)); tableAdmin.deleteTable(tableId); - skipTableDelete = true; } @Test