Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Refactored value provider #2025

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rahulKQL wants to merge 3 commits into from
Closed

Refactored value provider #2025

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
286b782
Refactored CloudBigtableScan
rahulKQL Dec 10, 2018
155825c
Added RequestWithScanValueProvider
rahulKQL Dec 12, 2018
dfa8dc6
adding BigtableExtendedScan initial impl
rahulKQL Dec 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 33 additions & 10 deletions ...ase-beam/src/main/java/com/google/cloud/bigtable/beam/CloudBigtableScanConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.beam;

import com.google.cloud.bigtable.hbase.BigtableExtendedScan;
import java.io.Serializable;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -271,16 +272,19 @@ Builder withTableId(ValueProvider<String> tableId) {
*/
@Override
public CloudBigtableScanConfiguration build() {
RowSet rowSet = null;
if (request == null) {
ReadHooks readHooks = new DefaultReadHooks();
if (scan == null) {
scan = new Scan();
}
ReadRowsRequest.Builder builder = Adapters.SCAN_ADAPTER.adapt(scan, readHooks);
request = StaticValueProvider.of(readHooks.applyPreSendHook(builder.build()));
BigtableExtendedScan extendedScan = new BigtableExtendedScan();
extendedScan.addRange(scan.getStartRow(), scan.getStopRow());
extendedScan.setMaxVersions(scan.getMaxVersions());
extendedScan.setFilter(scan.getFilter());
rowSet = extendedScan.getRowSet();
}
return new CloudBigtableScanConfiguration(projectId, instanceId, tableId,
request, additionalConfiguration);
request, rowSet, additionalConfiguration);
}
}

Expand All @@ -295,23 +299,38 @@ private static class RequestWithTableNameValueProvider
private final ValueProvider<String> projectId;
private final ValueProvider<String> instanceId;
private final ValueProvider<String> tableId;
private final ValueProvider<ReadRowsRequest> request;
ValueProvider<ReadRowsRequest> request;
private final RowSet rowSet;
private ReadRowsRequest cachedRequest;

RequestWithTableNameValueProvider(
ValueProvider<String> projectId,
ValueProvider<String> instanceId,
ValueProvider<String> tableId,
ValueProvider<ReadRowsRequest> request) {
RowSet rowSet) {
this.projectId = projectId;
this.instanceId = instanceId;
this.tableId = tableId;
this.request = request;
this.rowSet = rowSet;
}

@Override
public ReadRowsRequest get() {
if (cachedRequest == null) {
if (request == null) {
ReadHooks readHooks = new DefaultReadHooks();
BigtableExtendedScan scan = new BigtableExtendedScan();
if (rowSet != null) {
for (RowRange range : rowSet.getRowRangesList()) {
scan.addRange(range);
}
for (ByteString rowKey : rowSet.getRowKeysList()) {
scan.addRowKey(ByteStringer.extract(rowKey));
}
}
ReadRowsRequest.Builder builder = Adapters.SCAN_ADAPTER.adapt(scan, readHooks);
request = StaticValueProvider.of(readHooks.applyPreSendHook(builder.build()));
}
if (request.get().getTableName().isEmpty()) {
BigtableInstanceName bigtableInstanceName =
new BigtableInstanceName(projectId.get(), instanceId.get());
Expand All @@ -328,8 +347,7 @@ public ReadRowsRequest get() {
public boolean isAccessible() {
return projectId.isAccessible()
&& instanceId.isAccessible()
&& tableId.isAccessible()
&& request.isAccessible();
&& tableId.isAccessible();
}

@Override
Expand All @@ -348,16 +366,21 @@ public String toString() {
* @param instanceId The instance ID.
* @param tableId The table to connect to in the instance.
* @param request The {@link ReadRowsRequest} that will be used to filter the table.
* @Param rowSet The {@link RowSet} it may contain {@link RowRange} from {@link Scan}.
* @param additionalConfiguration A {@link Map} with additional connection configuration.
*/
protected CloudBigtableScanConfiguration(
ValueProvider<String> projectId,
ValueProvider<String> instanceId,
ValueProvider<String> tableId,
ValueProvider<ReadRowsRequest> request,
RowSet rowSet,
Map<String, ValueProvider<String>> additionalConfiguration) {
super(projectId, instanceId, tableId, additionalConfiguration);
this.request = new RequestWithTableNameValueProvider(projectId, instanceId, tableId, request);
if (rowSet != null) {
request = new RequestWithTableNameValueProvider(projectId, instanceId, tableId, rowSet);
}
this.request = request;
}

/**
Expand Down
20 changes: 20 additions & 0 deletions ...beam/src/test/java/com/google/cloud/bigtable/beam/CloudBigtableScanConfigurationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.beam;

import java.util.Arrays;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.hadoop.hbase.client.Scan;
Expand Down Expand Up @@ -138,4 +139,23 @@ public void testRegularAndRuntimeParametersAreEqualWithRequest() {
.build();
Assert.assertEquals(withRegularParameters, withRuntimeParameters);
}

@Test
public void testRowKeys(){
byte[] PREFIX = "PRE".getBytes();
Scan scan = new Scan()
.setStartRow(START_ROW)
.setStopRow(STOP_ROW)
.setRowPrefixFilter(PREFIX);
CloudBigtableScanConfiguration scanConfiguration =
new CloudBigtableScanConfiguration.Builder()
.withTableId(StaticValueProvider.of(TABLE))
.withProjectId(StaticValueProvider.of(PROJECT))
.withInstanceId(StaticValueProvider.of(INSTANCE))
.withScan(scan)
.withConfiguration("somekey", StaticValueProvider.of("somevalue"))
.build();
Assert.assertTrue(Arrays.equals(PREFIX, scanConfiguration.getStartRow()));
Assert.assertFalse(Arrays.equals(PREFIX, scanConfiguration.getStopRow()));
}
}