Better reliability for flaky tests (GoogleCloudPlatform#1960)

gguuss · web-flow · commit c611792bb115 · 2019-01-10T12:27:49.000-08:00
* Increases reliability for flaky tests
* Replaces checks where device or gateway ID could affect test outcome.
diff --git a/iot/api-client/beta-features/gateway/gateway_test.py b/iot/api-client/beta-features/gateway/gateway_test.py
@@ -168,7 +168,6 @@ def test_create_gateway(iot_topic, capsys):
     out, _ = capsys.readouterr()
 
     assert 'Created gateway' in out
-    assert '400' not in out
 
 
 def test_list_gateways(iot_topic, capsys):
@@ -195,7 +194,6 @@ def test_list_gateways(iot_topic, capsys):
     out, _ = capsys.readouterr()
 
     assert 'Gateway ID: {}'.format(gateway_id) in out
-    assert '400' not in out
 
 
 def test_bind_device_to_gateway_and_unbind(iot_topic, capsys):
@@ -233,7 +231,7 @@ def test_bind_device_to_gateway_and_unbind(iot_topic, capsys):
 
     assert 'Device Bound' in out
     assert 'Device unbound' in out
-    assert '400' not in out
+    assert 'HttpError 404' not in out
 
 
 def test_gateway_listen_for_bound_device_configs(iot_topic, capsys):
diff --git a/iot/api-client/manager/manager_test.py b/iot/api-client/manager/manager_test.py
@@ -18,6 +18,7 @@
 
 # Add command receiver for bootstrapping device registry / device for testing
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'mqtt_example'))  # noqa
+from gcp_devrel.testing.flaky import flaky
 from google.cloud import pubsub
 import pytest
 
@@ -278,14 +279,24 @@ def test_add_patch_delete_es256(test_topic, capsys):
             service_account_json, project_id, cloud_region, registry_id)
 
 
+@flaky
 def test_send_command(test_topic, capsys):
     device_id = device_id_template.format('RSA256')
     manager.create_registry(
             service_account_json, project_id, cloud_region, pubsub_topic,
             registry_id)
-    manager.create_rs256_device(
-            service_account_json, project_id, cloud_region, registry_id,
-            device_id, rsa_cert_path)
+
+    exists = False
+    devices = manager.list_devices(
+            service_account_json, project_id, cloud_region, registry_id)
+    for device in devices:
+        if device.get('id') == device_id:
+            exists = True
+
+    if not exists:
+        manager.create_rs256_device(
+                service_account_json, project_id, cloud_region, registry_id,
+                device_id, rsa_cert_path)
 
     # Exercize the functionality
     client = cloudiot_mqtt_example.get_client(
diff --git a/iot/api-client/manager/requirements.txt b/iot/api-client/manager/requirements.txt
@@ -1,4 +1,6 @@
 cryptography==2.4.2
+flaky==3.4.0
+gcp-devrel-py-tools==0.0.15
 google-api-python-client==1.7.5
 google-auth-httplib2==0.0.3
 google-auth==1.6.1
diff --git a/iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py b/iot/api-client/mqtt_example/cloudiot_mqtt_example_test.py
@@ -20,6 +20,7 @@
 
 # Add manager for bootstrapping device registry / device for testing
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'manager'))  # noqa
+from gcp_devrel.testing.flaky import flaky
 import manager
 
 import pytest
@@ -178,14 +179,24 @@ def test_config(test_topic, capsys):
     assert '/devices/{}/config'.format(device_id) in out
 
 
+@flaky
 def test_receive_command(capsys):
     device_id = device_id_template.format('RSA256')
     manager.create_registry(
             service_account_json, project_id, cloud_region, pubsub_topic,
             registry_id)
-    manager.create_rs256_device(
-            service_account_json, project_id, cloud_region, registry_id,
-            device_id, rsa_cert_path)
+
+    exists = False
+    devices = manager.list_devices(
+            service_account_json, project_id, cloud_region, registry_id)
+    for device in devices:
+        if device.get('id') == device_id:
+            exists = True
+
+    if not exists:
+        manager.create_rs256_device(
+                service_account_json, project_id, cloud_region, registry_id,
+                device_id, rsa_cert_path)
 
     # Exercize the functionality
     client = cloudiot_mqtt_example.get_client(
@@ -195,7 +206,7 @@ def test_receive_command(capsys):
     client.loop_start()
 
     # Pre-process commands
-    for i in range(1, 3):
+    for i in range(1, 5):
         client.loop()
         time.sleep(1)
 
@@ -204,7 +215,7 @@ def test_receive_command(capsys):
             device_id, 'me want cookies')
 
     # Process commands
-    for i in range(1, 3):
+    for i in range(1, 5):
         client.loop()
         time.sleep(1)
 
diff --git a/iot/api-client/mqtt_example/requirements.txt b/iot/api-client/mqtt_example/requirements.txt
@@ -1,7 +1,9 @@
+cryptography==2.4.2
+flaky==3.4.0
+gcp-devrel-py-tools==0.0.15
 google-api-python-client==1.7.5
 google-auth-httplib2==0.0.3
 google-auth==1.6.1
 google-cloud-pubsub==0.39.0
-cryptography==2.4.2
 pyjwt==1.6.4
 paho-mqtt==1.4.0
