Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit d64a598

Browse files
billyjacobsonleahecole
billyjacobson
and
leahecole
authored
Cleanup bigtable python examples (GoogleCloudPlatform#2692)
* Cleanup bigtable python: Use new row types for mutations Update bigtable version in requirements Delete table after tests * Change bigtable cluster variable to bigtable instance for consistency Create and delete quickstart table during test * Fixing step size for metric scaler Create unique tables for quickstart tests * Creating fixtures for quickstart tests Fixing hb quickstart test output * Fix quickstart extra delete table Update happybase to use direct row * Use clearer instance names for tests Create unique instances for metric scaler tests * Linting * remove core dep Co-authored-by: Leah E. Cole <[email protected]>
1 parent 4fb6a39 commit d64a598

File tree

20 files changed

+127
-41
lines changed

20 files changed

+127
-41
lines changed

‎bigtable/hello/main.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def main(project_id, instance_id, table_id):
7575
#
7676
# https://cloud.google.com/bigtable/docs/schema-design
7777
row_key = 'greeting{}'.format(i).encode()
78-
row = table.row(row_key)
78+
row = table.direct_row(row_key)
7979
row.set_cell(column_family_id,
8080
column,
8181
value,

‎bigtable/hello/main_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,16 @@
1818
from main import main
1919

2020
PROJECT = os.environ['GCLOUD_PROJECT']
21-
BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
22-
TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}'
21+
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
22+
TABLE_NAME_FORMAT = 'hello-world-test-{}'
2323
TABLE_NAME_RANGE = 10000
2424

2525

2626
def test_main(capsys):
2727
table_name = TABLE_NAME_FORMAT.format(
2828
random.randrange(TABLE_NAME_RANGE))
2929

30-
main(PROJECT, BIGTABLE_CLUSTER, table_name)
30+
main(PROJECT, BIGTABLE_INSTANCE, table_name)
3131

3232
out, _ = capsys.readouterr()
3333
assert 'Creating the {} table.'.format(table_name) in out

‎bigtable/hello/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
google-cloud-bigtable==1.2.0
1+
google-cloud-bigtable==1.2.1
22
google-cloud-core==1.1.0

‎bigtable/hello_happybase/main_test.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
from main import main
1919

2020
PROJECT = os.environ['GCLOUD_PROJECT']
21-
BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
22-
TABLE_NAME_FORMAT = 'hello_happybase-system-tests-{}'
21+
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
22+
TABLE_NAME_FORMAT = 'hello-world-hb-test-{}'
2323
TABLE_NAME_RANGE = 10000
2424

2525

@@ -28,7 +28,7 @@ def test_main(capsys):
2828
random.randrange(TABLE_NAME_RANGE))
2929
main(
3030
PROJECT,
31-
BIGTABLE_CLUSTER,
31+
BIGTABLE_INSTANCE,
3232
table_name)
3333

3434
out, _ = capsys.readouterr()

‎bigtable/instanceadmin/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
google-cloud-bigtable==1.2.0
1+
google-cloud-bigtable==1.2.1

‎bigtable/metricscaler/metricscaler_test.py

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,25 @@
1717
import os
1818
import time
1919

20+
import random
21+
22+
import pytest
2023
from google.cloud import bigtable
24+
from google.cloud.bigtable import enums
2125
from mock import patch
2226

2327
from metricscaler import get_cpu_load
2428
from metricscaler import main
2529
from metricscaler import scale_bigtable
2630

27-
# tests assume instance and cluster have the same ID
28-
BIGTABLE_INSTANCE = os.environ['BIGTABLE_CLUSTER']
31+
PROJECT = os.environ['GCLOUD_PROJECT']
32+
BIGTABLE_ZONE = os.environ['BIGTABLE_ZONE']
2933
SIZE_CHANGE_STEP = 3
34+
INSTANCE_ID_FORMAT = 'metric-scale-test-{}'
35+
INSTANCE_ID_RANGE = 10000
36+
BIGTABLE_INSTANCE = INSTANCE_ID_FORMAT.format(
37+
random.randrange(INSTANCE_ID_RANGE))
38+
3039

3140
# System tests to verify API calls succeed
3241

@@ -35,8 +44,33 @@ def test_get_cpu_load():
3544
assert float(get_cpu_load()) > 0.0
3645

3746

38-
def test_scale_bigtable():
47+
@pytest.fixture()
48+
def instance():
49+
cluster_id = BIGTABLE_INSTANCE
50+
51+
client = bigtable.Client(project=PROJECT, admin=True)
52+
53+
serve_nodes = 3
54+
storage_type = enums.StorageType.SSD
55+
production = enums.Instance.Type.PRODUCTION
56+
labels = {'prod-label': 'prod-label'}
57+
instance = client.instance(BIGTABLE_INSTANCE, instance_type=production,
58+
labels=labels)
59+
60+
if not instance.exists():
61+
cluster = instance.cluster(cluster_id, location_id=BIGTABLE_ZONE,
62+
serve_nodes=serve_nodes,
63+
default_storage_type=storage_type)
64+
instance.create(clusters=[cluster])
65+
66+
yield
67+
68+
instance.delete()
69+
70+
71+
def test_scale_bigtable(instance):
3972
bigtable_client = bigtable.Client(admin=True)
73+
4074
instance = bigtable_client.instance(BIGTABLE_INSTANCE)
4175
instance.reload()
4276

‎bigtable/metricscaler/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
google-cloud-bigtable==1.2.0
1+
google-cloud-bigtable==1.2.1
22
google-cloud-monitoring==0.34.0

‎bigtable/quickstart/main_test.py

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,41 @@
1313
# limitations under the License.
1414

1515
import os
16+
import random
17+
import pytest
1618

1719
from main import main
20+
from google.cloud import bigtable
1821

1922
PROJECT = os.environ['GCLOUD_PROJECT']
20-
BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
21-
TABLE_NAME = 'my-table'
23+
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
24+
TABLE_ID_FORMAT = 'quickstart-test-{}'
25+
TABLE_ID_RANGE = 10000
2226

2327

24-
def test_main(capsys):
25-
main(PROJECT, BIGTABLE_CLUSTER, TABLE_NAME)
28+
@pytest.fixture()
29+
def table():
30+
table_id = TABLE_ID_FORMAT.format(
31+
random.randrange(TABLE_ID_RANGE))
32+
client = bigtable.Client(project=PROJECT, admin=True)
33+
instance = client.instance(BIGTABLE_INSTANCE)
34+
table = instance.table(table_id)
35+
column_family_id = 'cf1'
36+
column_families = {column_family_id: None}
37+
table.create(column_families=column_families)
38+
39+
row = table.direct_row("r1")
40+
row.set_cell(column_family_id, "c1", "test-value")
41+
row.commit()
42+
43+
yield table_id
44+
45+
table.delete()
46+
47+
48+
def test_main(capsys, table):
49+
table_id = table
50+
main(PROJECT, BIGTABLE_INSTANCE, table_id)
2651

2752
out, _ = capsys.readouterr()
2853
assert 'Row key: r1\nData: test-value\n' in out

‎bigtable/quickstart/requirements.txt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1 @@
1-
google-cloud-bigtable==1.0.0
2-
google-cloud-core==1.0.3
1+
google-cloud-bigtable==1.2.1

‎bigtable/quickstart_happybase/main.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
# limitations under the License.
1616
# [START bigtable_quickstart_happybase]
1717
import argparse
18-
import json
1918

2019
from google.cloud import bigtable
2120
from google.cloud import happybase
@@ -37,9 +36,10 @@ def main(project_id="project-id", instance_id="instance-id",
3736

3837
key = 'r1'
3938
row = table.row(key.encode('utf-8'))
40-
value = {k.decode("utf-8"): v.decode("utf-8") for k, v in row.items()}
41-
print('Row key: {}\nData: {}'.format(key, json.dumps(value, indent=4,
42-
sort_keys=True)))
39+
40+
column = 'cf1:c1'.encode('utf-8')
41+
value = row[column].decode('utf-8')
42+
print('Row key: {}\nData: {}'.format(key, value))
4343

4444
finally:
4545
connection.close()

‎bigtable/quickstart_happybase/main_test.py

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,41 @@
1414

1515
import os
1616

17+
import random
18+
19+
import pytest
20+
from google.cloud import bigtable
1721
from main import main
1822

1923
PROJECT = os.environ['GCLOUD_PROJECT']
20-
BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
21-
TABLE_NAME = 'my-table'
24+
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
25+
TABLE_ID_FORMAT = 'quickstart-hb-test-{}'
26+
TABLE_ID_RANGE = 10000
27+
28+
29+
@pytest.fixture()
30+
def table():
31+
table_id = TABLE_ID_FORMAT.format(
32+
random.randrange(TABLE_ID_RANGE))
33+
client = bigtable.Client(project=PROJECT, admin=True)
34+
instance = client.instance(BIGTABLE_INSTANCE)
35+
table = instance.table(table_id)
36+
column_family_id = 'cf1'
37+
column_families = {column_family_id: None}
38+
table.create(column_families=column_families)
39+
40+
row = table.direct_row("r1")
41+
row.set_cell(column_family_id, "c1", "test-value")
42+
row.commit()
43+
44+
yield table_id
45+
46+
table.delete()
2247

2348

24-
def test_main(capsys):
25-
main(PROJECT, BIGTABLE_CLUSTER, TABLE_NAME)
49+
def test_main(capsys, table):
50+
table_id = table
51+
main(PROJECT, BIGTABLE_INSTANCE, table_id)
2652

2753
out, _ = capsys.readouterr()
28-
assert '"cf1:c1": "test-value"' in out
54+
assert 'Row key: r1\nData: test-value\n' in out

‎bigtable/snippets/writes/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
google-cloud-bigtable==1.2.0
1+
google-cloud-bigtable==1.2.1

‎bigtable/snippets/writes/write_batch.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ def write_batch(project_id, instance_id, table_id):
2626
timestamp = datetime.datetime.utcnow()
2727
column_family_id = "stats_summary"
2828

29-
rows = [table.row("tablet#a0b81f74#20190501"),
30-
table.row("tablet#a0b81f74#20190502")]
29+
rows = [table.direct_row("tablet#a0b81f74#20190501"),
30+
table.direct_row("tablet#a0b81f74#20190502")]
3131

3232
rows[0].set_cell(column_family_id,
3333
"connected_wifi",

‎bigtable/snippets/writes/write_conditionally.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ def write_conditional(project_id, instance_id, table_id):
3333
filters=[row_filters.FamilyNameRegexFilter(column_family_id),
3434
row_filters.ColumnQualifierRegexFilter('os_build'),
3535
row_filters.ValueRegexFilter("PQ2A\\..*")])
36-
row = table.row(row_key, filter_=row_filter)
36+
row = table.conditional_row(row_key, filter_=row_filter)
3737
row.set_cell(column_family_id,
3838
"os_name",
3939
"android",

‎bigtable/snippets/writes/write_increment.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ def write_increment(project_id, instance_id, table_id):
2424
column_family_id = "stats_summary"
2525

2626
row_key = "phone#4c410523#20190501"
27-
row = table.row(row_key, append=True)
27+
row = table.append_row(row_key)
2828

2929
# Decrement the connected_wifi value by 1.
3030
row.increment_cell_value(column_family_id, "connected_wifi", -1)

‎bigtable/snippets/writes/write_simple.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ def write_simple(project_id, instance_id, table_id):
2727

2828
row_key = "phone#4c410523#20190501"
2929

30-
row = table.row(row_key)
30+
row = table.direct_row(row_key)
3131
row.set_cell(column_family_id,
3232
"connected_cell",
3333
1,

‎bigtable/snippets/writes/writes_test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
from .write_simple import write_simple
2525

2626
PROJECT = os.environ['GCLOUD_PROJECT']
27-
BIGTABLE_INSTANCE = os.environ['BIGTABLE_CLUSTER']
27+
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
2828
TABLE_ID_PREFIX = 'mobile-time-series-{}'
2929

3030

‎bigtable/tableadmin/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
google-cloud-bigtable==1.2.0
1+
google-cloud-bigtable==1.2.1

‎bigtable/tableadmin/tableadmin_test.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,16 @@
2121
from tableadmin import run_table_operations
2222

2323
PROJECT = os.environ['GCLOUD_PROJECT']
24-
BIGTABLE_CLUSTER = os.environ['BIGTABLE_CLUSTER']
25-
TABLE_NAME_FORMAT = 'hello-bigtable-system-tests-{}'
24+
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
25+
TABLE_NAME_FORMAT = 'tableadmin-test-{}'
2626
TABLE_NAME_RANGE = 10000
2727

2828

2929
def test_run_table_operations(capsys):
3030
table_name = TABLE_NAME_FORMAT.format(
3131
random.randrange(TABLE_NAME_RANGE))
3232

33-
run_table_operations(PROJECT, BIGTABLE_CLUSTER, table_name)
33+
run_table_operations(PROJECT, BIGTABLE_INSTANCE, table_name)
3434
out, _ = capsys.readouterr()
3535

3636
assert 'Creating the ' + table_name + ' table.' in out
@@ -50,13 +50,15 @@ def test_run_table_operations(capsys):
5050
assert 'Delete a column family cf2...' in out
5151
assert 'Column family cf2 deleted successfully.' in out
5252

53+
delete_table(PROJECT, BIGTABLE_INSTANCE, table_name)
54+
5355

5456
def test_delete_table(capsys):
5557
table_name = TABLE_NAME_FORMAT.format(
5658
random.randrange(TABLE_NAME_RANGE))
57-
create_table(PROJECT, BIGTABLE_CLUSTER, table_name)
59+
create_table(PROJECT, BIGTABLE_INSTANCE, table_name)
5860

59-
delete_table(PROJECT, BIGTABLE_CLUSTER, table_name)
61+
delete_table(PROJECT, BIGTABLE_INSTANCE, table_name)
6062
out, _ = capsys.readouterr()
6163

6264
assert 'Table ' + table_name + ' exists.' in out

‎testing/secrets.tar.enc

6 KB
Binary file not shown.

0 commit comments

Comments
 (0)