Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Feb 5, 2025. It is now read-only.

Commit fdcf829

Browse files
rahulKQLrahulKQL
authored
chore: part 2 of replacing BigtableOptions to BigtableHBaseSettings (googleapis#2431)
* chore: part 2 of replacing BigtableOptions to BigtableHBaseSettings This change updated `BigtableBufferedMutator`, `BigtableBufferedMutatorHelper`, `HBaseRequestAdapter` constructor to accept `BigtableHBaseSetting` instead of `BigtableOptions`. * chore: address feedback comments - included `BatchExecutor.java` due to dep over settings. - fixed unit test settings initialization - updated config properties to retriesWithoutTimestamp getter
1 parent 1e93c92 commit fdcf829

File tree

14 files changed

+109
-106
lines changed

14 files changed

+109
-106
lines changed

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/AbstractBigtableTable.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -696,7 +696,11 @@ static String makeGenericExceptionMessage(
696696
*/
697697
protected synchronized BatchExecutor getBatchExecutor() {
698698
if (batchExecutor == null) {
699-
batchExecutor = new BatchExecutor(bigtableConnection.getSession(), hbaseAdapter);
699+
batchExecutor =
700+
new BatchExecutor(
701+
bigtableConnection.getSession(),
702+
bigtableConnection.getBigtableHBaseSettings(),
703+
hbaseAdapter);
700704
}
701705
return batchExecutor;
702706
}

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BatchExecutor.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@
2020
import com.google.api.core.ApiFutures;
2121
import com.google.api.core.InternalApi;
2222
import com.google.api.core.SettableApiFuture;
23-
import com.google.cloud.bigtable.config.BigtableOptions;
2423
import com.google.cloud.bigtable.grpc.BigtableSession;
2524
import com.google.cloud.bigtable.grpc.async.BulkRead;
2625
import com.google.cloud.bigtable.grpc.scanner.FlatRow;
2726
import com.google.cloud.bigtable.hbase.adapters.Adapters;
2827
import com.google.cloud.bigtable.hbase.adapters.HBaseRequestAdapter;
2928
import com.google.cloud.bigtable.hbase.util.Logger;
29+
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
3030
import com.google.cloud.bigtable.metrics.BigtableClientMetrics;
3131
import com.google.cloud.bigtable.metrics.BigtableClientMetrics.MetricLevel;
3232
import com.google.cloud.bigtable.metrics.Timer;
@@ -128,7 +128,7 @@ public final void onFailure(Throwable throwable) {
128128
}
129129
}
130130

131-
protected final BigtableOptions options;
131+
protected final BigtableHBaseSettings settings;
132132
protected final HBaseRequestAdapter requestAdapter;
133133
protected final Timer batchTimer = BigtableClientMetrics.timer(MetricLevel.Info, "batch.latency");
134134
// Once the IBigtableDataClient interface is implemented, this will be removed.
@@ -142,16 +142,15 @@ public final void onFailure(Throwable throwable) {
142142
* @param requestAdapter a {@link com.google.cloud.bigtable.hbase.adapters.HBaseRequestAdapter}
143143
* object.
144144
*/
145-
public BatchExecutor(BigtableSession session, HBaseRequestAdapter requestAdapter) {
145+
// TODO(rahulkql): This is an intermediate state, once BigtableWrapper is implemented we can
146+
// fetch settings directly from that(similar to BigtableSession.
147+
public BatchExecutor(
148+
BigtableSession session, BigtableHBaseSettings settings, HBaseRequestAdapter requestAdapter) {
146149
this.requestAdapter = requestAdapter;
147-
this.options = session.getOptions();
150+
this.settings = settings;
148151
this.bulkRead = session.createBulkRead(requestAdapter.getBigtableTableName());
149152
this.bufferedMutatorHelper =
150-
new BigtableBufferedMutatorHelper(
151-
requestAdapter,
152-
null, // configuration isn't passed in, but also isn't used in
153-
// BigtableBufferedMutatorHelper
154-
session);
153+
new BigtableBufferedMutatorHelper(requestAdapter, settings, session);
155154
}
156155

157156
/**
@@ -281,7 +280,7 @@ public <R> void batchCallback(
281280
} catch (ExecutionException e) {
282281
problemActions.add(actions.get(i));
283282
problems.add(e.getCause());
284-
hosts.add(options.getDataHost());
283+
hosts.add(settings.getDataHost());
285284
}
286285
}
287286
if (problems.size() > 0) {

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutator.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.cloud.bigtable.grpc.BigtableSession;
2323
import com.google.cloud.bigtable.hbase.adapters.HBaseRequestAdapter;
2424
import com.google.cloud.bigtable.hbase.util.Logger;
25+
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
2526
import com.google.common.util.concurrent.MoreExecutors;
2627
import java.io.IOException;
2728
import java.util.ArrayList;
@@ -64,18 +65,18 @@ private static class MutationException {
6465
* Constructor for BigtableBufferedMutator.
6566
*
6667
* @param adapter Converts HBase objects to Bigtable protos
67-
* @param configuration For Additional configuration. TODO: move this to options
68+
* @param settings For bigtable settings
6869
* @param listener Handles exceptions. By default, it just throws the exception.
6970
* @param session a {@link BigtableSession} object.
7071
*/
7172
public BigtableBufferedMutator(
7273
HBaseRequestAdapter adapter,
73-
Configuration configuration,
74+
BigtableHBaseSettings settings,
7475
BigtableSession session,
7576
BufferedMutator.ExceptionListener listener) {
76-
helper = new BigtableBufferedMutatorHelper(adapter, configuration, session);
77+
helper = new BigtableBufferedMutatorHelper(adapter, settings, session);
7778
this.listener = listener;
78-
this.host = session.getOptions().getDataHost();
79+
this.host = settings.getDataHost();
7980
}
8081

8182
/** {@inheritDoc} */

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutatorHelper.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818
import com.google.api.core.ApiFuture;
1919
import com.google.api.core.ApiFutures;
2020
import com.google.api.core.InternalApi;
21-
import com.google.cloud.bigtable.config.BigtableOptions;
2221
import com.google.cloud.bigtable.core.IBigtableDataClient;
2322
import com.google.cloud.bigtable.core.IBulkMutation;
2423
import com.google.cloud.bigtable.grpc.BigtableSession;
2524
import com.google.cloud.bigtable.grpc.BigtableTableName;
2625
import com.google.cloud.bigtable.hbase.adapters.HBaseRequestAdapter;
2726
import com.google.cloud.bigtable.hbase.util.Logger;
2827
import com.google.cloud.bigtable.hbase.util.OperationAccountant;
28+
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
2929
import com.google.cloud.bigtable.util.ApiFutureUtil;
3030
import java.io.IOException;
3131
import java.util.ArrayList;
@@ -53,8 +53,6 @@ public class BigtableBufferedMutatorHelper {
5353
/** Constant <code>LOG</code> */
5454
protected static final Logger LOG = new Logger(BigtableBufferedMutatorHelper.class);
5555

56-
private final Configuration configuration;
57-
5856
/**
5957
* Makes sure that mutations and flushes are safe to proceed. Ensures that while the mutator is
6058
* closing, there will be no additional writes.
@@ -69,21 +67,20 @@ public class BigtableBufferedMutatorHelper {
6967
private final HBaseRequestAdapter adapter;
7068
private final IBulkMutation bulkMutation;
7169
private final IBigtableDataClient dataClient;
72-
private final BigtableOptions options;
70+
private final BigtableHBaseSettings settings;
7371
private final OperationAccountant operationAccountant;
7472

7573
/**
7674
* Constructor for BigtableBufferedMutator.
7775
*
7876
* @param adapter Converts HBase objects to Bigtable protos
79-
* @param configuration For Additional configuration. TODO: move this to options
77+
* @param settings For bigtable settings
8078
* @param session a {@link BigtableSession} object.
8179
*/
8280
public BigtableBufferedMutatorHelper(
83-
HBaseRequestAdapter adapter, Configuration configuration, BigtableSession session) {
81+
HBaseRequestAdapter adapter, BigtableHBaseSettings settings, BigtableSession session) {
8482
this.adapter = adapter;
85-
this.configuration = configuration;
86-
this.options = session.getOptions();
83+
this.settings = settings;
8784
BigtableTableName tableName = this.adapter.getBigtableTableName();
8885
this.bulkMutation = session.createBulkMutationWrapper(tableName);
8986
this.dataClient = session.getDataClientWrapper();
@@ -121,15 +118,15 @@ public void sendUnsent() {
121118
}
122119

123120
public Configuration getConfiguration() {
124-
return this.configuration;
121+
return this.settings.getConfiguration();
125122
}
126123

127124
public TableName getName() {
128125
return this.adapter.getTableName();
129126
}
130127

131128
public long getWriteBufferSize() {
132-
return this.options.getBulkOptions().getMaxMemory();
129+
return this.settings.getBatchingMaxRequestSize();
133130
}
134131

135132
public List<ApiFuture<?>> mutate(List<? extends Mutation> mutations) {

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/Adapters.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
package com.google.cloud.bigtable.hbase.adapters;
1717

1818
import com.google.api.core.InternalApi;
19-
import com.google.cloud.bigtable.config.BigtableOptions;
2019
import com.google.cloud.bigtable.grpc.scanner.FlatRow;
2120
import com.google.cloud.bigtable.hbase.adapters.filters.BigtableWhileMatchResultScannerAdapter;
2221
import com.google.cloud.bigtable.hbase.adapters.filters.FilterAdapter;
@@ -26,6 +25,7 @@
2625
import com.google.cloud.bigtable.hbase.adapters.read.RowAdapter;
2726
import com.google.cloud.bigtable.hbase.adapters.read.RowRangeAdapter;
2827
import com.google.cloud.bigtable.hbase.adapters.read.ScanAdapter;
28+
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
2929
import org.apache.hadoop.conf.Configuration;
3030
import org.apache.hadoop.hbase.client.Append;
3131
import org.apache.hadoop.hbase.client.Increment;
@@ -80,12 +80,12 @@ public static HBaseMutationAdapter createMutationsAdapter(PutAdapter putAdapter)
8080
/**
8181
* createPutAdapter.
8282
*
83-
* @param config a {@link org.apache.hadoop.conf.Configuration} object.
84-
* @param options a {@link com.google.cloud.bigtable.config.BigtableOptions} object.
83+
* @param settings a {@link org.apache.hadoop.conf.Configuration} object.
8584
* @return a {@link com.google.cloud.bigtable.hbase.adapters.PutAdapter} object.
8685
*/
87-
public static PutAdapter createPutAdapter(Configuration config, BigtableOptions options) {
88-
boolean setClientTimestamp = !options.getRetryOptions().allowRetriesWithoutTimestamp();
86+
public static PutAdapter createPutAdapter(BigtableHBaseSettings settings) {
87+
Configuration config = settings.getConfiguration();
88+
boolean setClientTimestamp = !settings.isRetriesWithoutTimestampAllowed();
8989
return new PutAdapter(config.getInt("hbase.client.keyvalue.maxsize", -1), setClientTimestamp);
9090
}
9191

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/adapters/HBaseRequestAdapter.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
package com.google.cloud.bigtable.hbase.adapters;
1717

1818
import com.google.api.core.InternalApi;
19-
import com.google.cloud.bigtable.config.BigtableOptions;
19+
import com.google.cloud.bigtable.data.v2.internal.NameUtil;
2020
import com.google.cloud.bigtable.data.v2.models.Mutation;
2121
import com.google.cloud.bigtable.data.v2.models.MutationApi;
2222
import com.google.cloud.bigtable.data.v2.models.Query;
@@ -26,9 +26,9 @@
2626
import com.google.cloud.bigtable.grpc.BigtableTableName;
2727
import com.google.cloud.bigtable.hbase.adapters.read.DefaultReadHooks;
2828
import com.google.cloud.bigtable.hbase.adapters.read.ReadHooks;
29+
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
2930
import com.google.common.annotations.VisibleForTesting;
3031
import com.google.protobuf.ByteString;
31-
import org.apache.hadoop.conf.Configuration;
3232
import org.apache.hadoop.hbase.TableName;
3333
import org.apache.hadoop.hbase.client.Append;
3434
import org.apache.hadoop.hbase.client.Delete;
@@ -53,8 +53,8 @@ public static class MutationAdapters {
5353
protected final HBaseMutationAdapter hbaseMutationAdapter;
5454
protected final RowMutationsAdapter rowMutationsAdapter;
5555

56-
public MutationAdapters(BigtableOptions options, Configuration config) {
57-
this(Adapters.createPutAdapter(config, options));
56+
public MutationAdapters(BigtableHBaseSettings settings) {
57+
this(Adapters.createPutAdapter(settings));
5858
}
5959

6060
@VisibleForTesting
@@ -76,26 +76,29 @@ public MutationAdapters withServerSideTimestamps() {
7676
/**
7777
* Constructor for HBaseRequestAdapter.
7878
*
79-
* @param options a {@link com.google.cloud.bigtable.config.BigtableOptions} object.
79+
* @param settings a {@link BigtableHBaseSettings} object.
8080
* @param tableName a {@link org.apache.hadoop.hbase.TableName} object.
81-
* @param config a {@link org.apache.hadoop.conf.Configuration} object.
8281
*/
83-
public HBaseRequestAdapter(BigtableOptions options, TableName tableName, Configuration config) {
84-
this(options, tableName, new MutationAdapters(options, config));
82+
public HBaseRequestAdapter(BigtableHBaseSettings settings, TableName tableName) {
83+
this(settings, tableName, new MutationAdapters(settings));
8584
}
8685

8786
/**
8887
* Constructor for HBaseRequestAdapter.
8988
*
90-
* @param options a {@link BigtableOptions} object.
89+
* @param settings a {@link BigtableHBaseSettings} object.
9190
* @param tableName a {@link TableName} object.
9291
* @param mutationAdapters a {@link MutationAdapters} object.
9392
*/
9493
public HBaseRequestAdapter(
95-
BigtableOptions options, TableName tableName, MutationAdapters mutationAdapters) {
94+
BigtableHBaseSettings settings, TableName tableName, MutationAdapters mutationAdapters) {
9695
this(
9796
tableName,
98-
options.getInstanceName().toTableName(tableName.getQualifierAsString()),
97+
new BigtableTableName(
98+
NameUtil.formatTableName(
99+
settings.getProjectId(),
100+
settings.getInstanceId(),
101+
tableName.getQualifierAsString())),
99102
mutationAdapters);
100103
}
101104

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableConnection.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,19 +145,18 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I
145145

146146
HBaseRequestAdapter adapter = createAdapter(tableName);
147147
ExceptionListener listener = params.getListener();
148-
return new BigtableBufferedMutator(adapter, getConfiguration(), session, listener);
148+
return new BigtableBufferedMutator(adapter, settings, session, listener);
149149
}
150150

151151
public HBaseRequestAdapter createAdapter(TableName tableName) {
152152
if (mutationAdapters == null) {
153153
synchronized (this) {
154154
if (mutationAdapters == null) {
155-
mutationAdapters =
156-
new HBaseRequestAdapter.MutationAdapters(getOptions(), getConfiguration());
155+
mutationAdapters = new HBaseRequestAdapter.MutationAdapters(settings);
157156
}
158157
}
159158
}
160-
return new HBaseRequestAdapter(getOptions(), tableName, mutationAdapters);
159+
return new HBaseRequestAdapter(settings, tableName, mutationAdapters);
161160
}
162161

163162
/** {@inheritDoc} */

‎bigtable-client-core-parent/bigtable-hbase/src/test/java/com/google/cloud/bigtable/hbase/PutMicroBenchmark.java‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,19 @@
1515
*/
1616
package com.google.cloud.bigtable.hbase;
1717

18+
import static com.google.cloud.bigtable.hbase.BigtableOptionsFactory.CUSTOM_USER_AGENT_KEY;
19+
1820
import com.google.bigtable.v2.MutateRowRequest;
1921
import com.google.bigtable.v2.MutateRowResponse;
20-
import com.google.cloud.bigtable.config.BigtableOptions;
2122
import com.google.cloud.bigtable.data.v2.internal.RequestContext;
2223
import com.google.cloud.bigtable.grpc.BigtableDataClient;
2324
import com.google.cloud.bigtable.grpc.BigtableDataGrpcClient;
2425
import com.google.cloud.bigtable.grpc.BigtableSession;
2526
import com.google.cloud.bigtable.grpc.BigtableSessionSharedThreadPools;
2627
import com.google.cloud.bigtable.grpc.io.ChannelPool;
2728
import com.google.cloud.bigtable.hbase.adapters.HBaseRequestAdapter;
29+
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
30+
import com.google.cloud.bigtable.hbase.wrappers.classic.BigtableHBaseClassicSettings;
2831
import io.grpc.CallOptions;
2932
import io.grpc.ClientCall;
3033
import io.grpc.ManagedChannel;
@@ -41,33 +44,30 @@
4144
import org.apache.hadoop.hbase.client.Put;
4245
import org.apache.hadoop.hbase.util.Bytes;
4346

47+
// TODO(rahulkql): If possible move this class to bigtable-1.x-benchmark.
4448
public class PutMicroBenchmark {
4549
static final int NUM_CELLS = 10;
4650
private static final byte[] COLUMN_FAMILY = Bytes.toBytes("test_family");
4751
private static final int REAL_CHANNEL_PUT_COUNT = 100;
4852
private static final int FAKE_CHANNEL_PUT_COUNT = 100_000;
4953
private static final int VALUE_SIZE = 100;
50-
private static BigtableOptions options;
54+
private static BigtableHBaseSettings settings;
5155
private static RequestContext requestContext;
5256

5357
public static void main(String[] args) throws Exception {
5458
String projectId = args.length > 0 ? args[0] : "project";
5559
String instanceId = args.length > 1 ? args[1] : "instanceId";
5660
String tableId = args.length > 2 ? args[2] : "table";
5761

58-
options =
59-
BigtableOptions.builder()
60-
.setProjectId(projectId)
61-
.setInstanceId(instanceId)
62-
.setUserAgent("put_microbenchmark")
63-
.build();
62+
Configuration configuration = BigtableConfiguration.configure(projectId, instanceId);
63+
configuration.set(CUSTOM_USER_AGENT_KEY, "put_microbenchmark");
64+
settings = BigtableHBaseSettings.create(configuration);
65+
6466
boolean useRealConnection = args.length >= 2;
6567
int putCount = useRealConnection ? REAL_CHANNEL_PUT_COUNT : FAKE_CHANNEL_PUT_COUNT;
6668
HBaseRequestAdapter hbaseAdapter =
67-
new HBaseRequestAdapter(options, TableName.valueOf(tableId), new Configuration(false));
68-
requestContext =
69-
RequestContext.create(
70-
options.getProjectId(), options.getInstanceId(), options.getAppProfileId());
69+
new HBaseRequestAdapter(settings, TableName.valueOf(tableId));
70+
requestContext = RequestContext.create(settings.getProjectId(), settings.getInstanceId(), "");
7171

7272
testCreatePuts(10_000);
7373

@@ -82,7 +82,8 @@ public static void main(String[] args) throws Exception {
8282
protected static ManagedChannel getChannelPool(final boolean useRealConnection)
8383
throws IOException, GeneralSecurityException {
8484
if (useRealConnection) {
85-
return BigtableSession.createChannelPool(options.getDataHost(), options);
85+
return BigtableSession.createChannelPool(
86+
settings.getDataHost(), ((BigtableHBaseClassicSettings) settings).getBigtableOptions());
8687
} else {
8788
return new ChannelPool(createFakeChannels(), 1);
8889
}
@@ -129,7 +130,9 @@ protected static void run(
129130
throws InterruptedException {
130131
final BigtableDataClient client =
131132
new BigtableDataGrpcClient(
132-
cp, BigtableSessionSharedThreadPools.getInstance().getRetryExecutor(), options);
133+
cp,
134+
BigtableSessionSharedThreadPools.getInstance().getRetryExecutor(),
135+
((BigtableHBaseClassicSettings) settings).getBigtableOptions());
133136

134137
Runnable r1 =
135138
new Runnable() {

0 commit comments

Comments
 (0)