Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Bigtable: Refactor TableAdmin tests to run against the emulator #5795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
igorbernstein2 merged 5 commits into from
Aug 26, 2019
Merged

Bigtable: Refactor TableAdmin tests to run against the emulator #5795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d3ba37c
Bigtable: allow integration test helpers to be shared by data & admin…
igorbernstein2 Aug 21, 2019
16cfada
Allow test environment to be inspected
igorbernstein2 Aug 22, 2019
cddc1bd
Run most of table admin tests in emulator
igorbernstein2 Aug 22, 2019
56cdd56
fix cleanup
igorbernstein2 Aug 22, 2019
0d7a5f0
Merge remote-tracking branch 'origin/master' into table-admin-emulator
igorbernstein2 Aug 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions google-cloud-clients/google-cloud-bigtable/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@
<bigtable.env>emulator</bigtable.env>
</systemPropertyVariables>
<test>com.google.cloud.bigtable.data.v2.it.**</test>
<test>com.google.cloud.bigtable.admin.v2.it.**</test>
<summaryFile>target/failsafe-reports/failsafe-summary-emulator-it.xml</summaryFile>
</configuration>
</execution>
Expand Down
2 changes: 1 addition & 1 deletion google-cloud-clients/google-cloud-bigtable/scripts/setup-test-table.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ for ADMIN_HOST in "${ADMIN_HOSTS[@]}"; do
# Ensure that the family exists
if ! call_cbt ls "${TABLE_ID}" | grep -q "^$FAMILY\b"; then
call_cbt createfamily "${TABLE_ID}" "${FAMILY}"
call_cbt setgcpolicy "${TABLE_ID}" "${FAMILY}" maxversions=1
call_cbt setgcpolicy "${TABLE_ID}" "${FAMILY}" maxversions=1 maxage=1h
fi
done
268 changes: 95 additions & 173 deletions ...table/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@
package com.google.cloud.bigtable.admin.v2.it;

import static com.google.cloud.bigtable.admin.v2.models.GCRules.GCRULES;
import static com.google.common.truth.TruthJUnit.assume;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import com.google.api.gax.rpc.NotFoundException;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.models.ColumnFamily;
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
Expand All @@ -30,237 +32,157 @@
import com.google.cloud.bigtable.admin.v2.models.GCRules.VersionRule;
import com.google.cloud.bigtable.admin.v2.models.ModifyColumnFamiliesRequest;
import com.google.cloud.bigtable.admin.v2.models.Table;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
import com.google.common.collect.Maps;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import org.junit.AfterClass;
import org.junit.AssumptionViolatedException;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.threeten.bp.Duration;

public class BigtableTableAdminClientIT {
private static final Logger LOGGER = Logger.getLogger(BigtableTableAdminClientIT.class.getName());
@ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
@Rule public final TestName testNameRule = new TestName();

private static final String PROJECT_PROPERTY_NAME = "bigtable.project";
private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance";
private static List<String> missingProperties;
private BigtableTableAdminClient tableAdmin;
private String tableId;

private static BigtableTableAdminClient tableAdmin;
private static String prefix;

@BeforeClass
public static void createClient() throws IOException {
missingProperties = Lists.newArrayList();

String targetProject = System.getProperty(PROJECT_PROPERTY_NAME);
if (targetProject == null) {
missingProperties.add(PROJECT_PROPERTY_NAME);
}

String targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME);
if (targetInstance == null) {
missingProperties.add(INSTANCE_PROPERTY_NAME);
}

if (!missingProperties.isEmpty()) {
LOGGER.warning("Missing properties: " + Joiner.on(",").join(missingProperties));
return;
}

tableAdmin = BigtableTableAdminClient.create(targetProject, targetInstance);

// Setup a prefix to avoid collisions between concurrent test runs
prefix = String.format("020%d", System.currentTimeMillis());

// Cleanup old tables, under normal circumstances this will do nothing
String stalePrefix =
String.format("020%d", System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1));
for (String tableId : tableAdmin.listTables()) {
if (stalePrefix.compareTo(tableId) > 0) {
tableAdmin.deleteTable(tableId);
}
}
}

@AfterClass
public static void closeClient() {
if (tableAdmin != null) {
tableAdmin.close();
}
@Before
public void setUp() {
tableAdmin = testEnvRule.env().getTableAdminClient();
tableId = testEnvRule.env().generateTableId(testNameRule.getMethodName());
}

@Before
public void setup() {
if (tableAdmin == null) {
throw new AssumptionViolatedException(
"Required properties are not set, skipping integration tests.");
@After
public void tearDown() {
try {
testEnvRule.env().getTableAdminClient().deleteTable(tableId);
} catch (NotFoundException e) {
// table was deleted in test or was never created. Carry on
}
}

@Test
public void createTable() {
String tableId = getTableId("adminCreateTest");
assume()
.withMessage("Emulator doesn't return proper responses for CreateTable")
.that(testEnvRule.env())
.isNotInstanceOf(EmulatorEnv.class);

CreateTableRequest createTableReq =
CreateTableRequest.of(tableId)
.addFamily("cf1")
.addFamily("cf2", GCRULES.maxVersions(10))
.addSplit(ByteString.copyFromUtf8("b"))
.addSplit(ByteString.copyFromUtf8("q"));

try {
Table tableResponse = tableAdmin.createTable(createTableReq);
assertNotNull(tableResponse);
assertEquals(tableId, tableResponse.getId());
Table tableResponse = tableAdmin.createTable(createTableReq);
assertEquals(tableId, tableResponse.getId());

Map<String, ColumnFamily> columnFamilyById = Maps.newHashMap();
for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) {
columnFamilyById.put(columnFamily.getId(), columnFamily);
}
assertEquals(2, tableResponse.getColumnFamilies().size());
assertFalse(columnFamilyById.get("cf1").hasGCRule());
assertTrue(columnFamilyById.get("cf2").hasGCRule());
assertEquals(10, ((VersionRule) columnFamilyById.get("cf2").getGCRule()).getMaxVersions());
} finally {
tableAdmin.deleteTable(tableId);
Map<String, ColumnFamily> columnFamilyById = Maps.newHashMap();
for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) {
columnFamilyById.put(columnFamily.getId(), columnFamily);
}
assertEquals(2, tableResponse.getColumnFamilies().size());
assertFalse(columnFamilyById.get("cf1").hasGCRule());
assertTrue(columnFamilyById.get("cf2").hasGCRule());
assertEquals(10, ((VersionRule) columnFamilyById.get("cf2").getGCRule()).getMaxVersions());
}

@Test
public void modifyFamilies() {
String tableId = getTableId("adminModifyFamTest");
ModifyColumnFamiliesRequest modifyFamiliesReq = ModifyColumnFamiliesRequest.of(tableId);

modifyFamiliesReq
.addFamily("mf1")
.addFamily("mf2", GCRULES.maxAge(Duration.ofSeconds(1000, 20000)))
.updateFamily(
"mf1",
GCRULES
.union()
.rule(GCRULES.maxAge(Duration.ofSeconds(100)))
.rule(GCRULES.maxVersions(1)))
.addFamily(
"mf3",
GCRULES
.intersection()
.rule(GCRULES.maxAge(Duration.ofSeconds(2000)))
.rule(GCRULES.maxVersions(10)))
.addFamily("mf4", GCRULES.intersection().rule(GCRULES.maxAge(Duration.ofSeconds(360))))
.addFamily("mf5")
.addFamily("mf6")
.dropFamily("mf5")
.dropFamily("mf6")
.addFamily("mf7");

try {
tableAdmin.createTable(CreateTableRequest.of(tableId));
Table tableResponse = tableAdmin.modifyFamilies(modifyFamiliesReq);
tableAdmin.createTable(CreateTableRequest.of(tableId));

Map<String, ColumnFamily> columnFamilyById = Maps.newHashMap();
for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) {
columnFamilyById.put(columnFamily.getId(), columnFamily);
}
assertEquals(5, columnFamilyById.size());
assertNotNull(columnFamilyById.get("mf1"));
assertNotNull(columnFamilyById.get("mf2"));
assertEquals(2, ((UnionRule) columnFamilyById.get("mf1").getGCRule()).getRulesList().size());
assertEquals(
1000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getSeconds());
assertEquals(
20000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getNano());
assertEquals(
2, ((IntersectionRule) columnFamilyById.get("mf3").getGCRule()).getRulesList().size());
assertEquals(
360, ((DurationRule) columnFamilyById.get("mf4").getGCRule()).getMaxAge().getSeconds());
assertNotNull(columnFamilyById.get("mf7"));
} finally {
tableAdmin.deleteTable(tableId);
ModifyColumnFamiliesRequest modifyFamiliesReq =
ModifyColumnFamiliesRequest.of(tableId)
.addFamily("mf1")
.addFamily("mf2", GCRULES.maxAge(Duration.ofSeconds(1000, 20000)))
.updateFamily(
"mf1",
GCRULES
.union()
.rule(GCRULES.maxAge(Duration.ofSeconds(100)))
.rule(GCRULES.maxVersions(1)))
.addFamily(
"mf3",
GCRULES
.intersection()
.rule(GCRULES.maxAge(Duration.ofSeconds(2000)))
.rule(GCRULES.maxVersions(10)))
.addFamily("mf4", GCRULES.intersection().rule(GCRULES.maxAge(Duration.ofSeconds(360))))
.addFamily("mf5")
.addFamily("mf6")
.dropFamily("mf5")
.dropFamily("mf6")
.addFamily("mf7");

Table tableResponse = tableAdmin.modifyFamilies(modifyFamiliesReq);

Map<String, ColumnFamily> columnFamilyById = Maps.newHashMap();
for (ColumnFamily columnFamily : tableResponse.getColumnFamilies()) {
columnFamilyById.put(columnFamily.getId(), columnFamily);
}
assertEquals(5, columnFamilyById.size());
assertNotNull(columnFamilyById.get("mf1"));
assertNotNull(columnFamilyById.get("mf2"));
assertEquals(2, ((UnionRule) columnFamilyById.get("mf1").getGCRule()).getRulesList().size());
assertEquals(
1000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getSeconds());
assertEquals(
20000, ((DurationRule) columnFamilyById.get("mf2").getGCRule()).getMaxAge().getNano());
assertEquals(
2, ((IntersectionRule) columnFamilyById.get("mf3").getGCRule()).getRulesList().size());
assertEquals(
360, ((DurationRule) columnFamilyById.get("mf4").getGCRule()).getMaxAge().getSeconds());
assertNotNull(columnFamilyById.get("mf7"));
}

@Test
public void deleteTable() {
String tableId = getTableId("adminDeleteTest");
tableAdmin.createTable(CreateTableRequest.of(tableId));
tableAdmin.deleteTable(tableId);
}

@Test
public void getTable() {
String tableId = getTableId("adminGetTest");

try {
tableAdmin.createTable(CreateTableRequest.of(tableId));
Table tableResponse = tableAdmin.getTable(tableId);
assertNotNull(tableResponse);
assertEquals(tableId, tableResponse.getId());
} finally {
tableAdmin.deleteTable(tableId);
}
tableAdmin.createTable(CreateTableRequest.of(tableId));
Table tableResponse = tableAdmin.getTable(tableId);
assertNotNull(tableResponse);
assertEquals(tableId, tableResponse.getId());
}

@Test
public void listTables() {
String tableId = getTableId("adminListTest");

try {
tableAdmin.createTable(CreateTableRequest.of(tableId));
List<String> tables = tableAdmin.listTables();
assertNotNull(tables);
assertFalse("List tables did not return any tables", tables.isEmpty());
} finally {
tableAdmin.deleteTable(tableId);
}
tableAdmin.createTable(CreateTableRequest.of(tableId));
List<String> tables = tableAdmin.listTables();
assertNotNull(tables);
assertFalse("List tables did not return any tables", tables.isEmpty());
}

@Test
public void listTablesAsync() throws Exception {
String tableId = getTableId("adminListAsyncTest");

try {
tableAdmin.createTable(CreateTableRequest.of(tableId));
List<String> tables = tableAdmin.listTablesAsync().get();
assertNotNull(tables);
assertFalse("List tables did not return any tables", tables.isEmpty());
} finally {
tableAdmin.deleteTable(tableId);
}
tableAdmin.createTable(CreateTableRequest.of(tableId));
List<String> tables = tableAdmin.listTablesAsync().get();
assertNotNull(tables);
assertFalse("List tables did not return any tables", tables.isEmpty());
}

@Test
public void dropRowRange() {
String tableId = getTableId("adminDropRowrangeTest");

try {
tableAdmin.createTable(CreateTableRequest.of(tableId));
tableAdmin.dropRowRange(tableId, "rowPrefix");
tableAdmin.dropAllRows(tableId);
} finally {
tableAdmin.deleteTable(tableId);
}
tableAdmin.createTable(CreateTableRequest.of(tableId));
tableAdmin.dropRowRange(tableId, "rowPrefix");
tableAdmin.dropAllRows(tableId);
}

@Test
public void awaitReplication() {
String tableId = getTableId("adminConsistencyTest");

try {
tableAdmin.createTable(CreateTableRequest.of(tableId));
tableAdmin.awaitReplication(tableId);
} finally {
tableAdmin.deleteTable(tableId);
}
}

private static String getTableId(String name) {
return prefix + "-" + name;
tableAdmin.createTable(CreateTableRequest.of(tableId));
tableAdmin.awaitReplication(tableId);
}
}
Loading