fix(healthcare): add more retries for mitigating flakes (GoogleCloudPlatform#5237)

Takashi Matsuo · web-flow · commit 9c8afcab4c81 · 2021-01-14T15:02:03.000-08:00
* fix(healthcare): add more retries for mitigating flakes fixes GoogleCloudPlatform#5211 fixes GoogleCloudPlatform#5210 fixes GoogleCloudPlatform#5209 fixes GoogleCloudPlatform#5205 * fix(healthcare): longer retry timeout, one more retry
diff --git a/healthcare/api-client/v1/datasets/datasets_test.py b/healthcare/api-client/v1/datasets/datasets_test.py
@@ -137,8 +137,16 @@ def test_CRUD_dataset(capsys, crud_dataset_id):
         cloud_region,
         crud_dataset_id)
 
-    datasets.get_dataset(
-        project_id, cloud_region, crud_dataset_id)
+    @retry(
+        wait_exponential_multiplier=1000,
+        wait_exponential_max=10000,
+        stop_max_attempt_number=10,
+        retry_on_exception=retry_if_server_exception)
+    def get_dataset():
+        datasets.get_dataset(
+            project_id, cloud_region, crud_dataset_id)
+
+    get_dataset()
 
     datasets.list_datasets(
         project_id, cloud_region)
@@ -156,11 +164,19 @@ def test_CRUD_dataset(capsys, crud_dataset_id):
 
 
 def test_patch_dataset(capsys, test_dataset):
-    datasets.patch_dataset(
-        project_id,
-        cloud_region,
-        dataset_id,
-        time_zone)
+    # To mitigate the flake with HttpErrors.
+    @retry(
+        wait_exponential_multiplier=1000,
+        wait_exponential_max=10000,
+        stop_max_attempt_number=10,
+        retry_on_exception=retry_if_server_exception)
+    def run_sample():
+        datasets.patch_dataset(
+            project_id,
+            cloud_region,
+            dataset_id,
+            time_zone)
+    run_sample()
 
     out, _ = capsys.readouterr()
 
diff --git a/healthcare/api-client/v1beta1/fhir/fhir_resources_test.py b/healthcare/api-client/v1beta1/fhir/fhir_resources_test.py
@@ -54,9 +54,14 @@ def test_dataset():
 
 @pytest.fixture(scope="module")
 def test_fhir_store():
-    fhir_store = fhir_stores.create_fhir_store(
-        service_account_json, project_id, cloud_region, dataset_id, fhir_store_id
-    )
+    # We see HttpErrors with "dataset not initialized" message.
+    # I think retry will mitigate the flake.
+    @backoff.on_exception(backoff.expo, HTTPError, max_time=120)
+    def create_fhir_store():
+        return fhir_stores.create_fhir_store(
+            service_account_json, project_id, cloud_region, dataset_id, fhir_store_id
+        )
+    fhir_store = create_fhir_store()
 
     yield fhir_store
 
