Merge pull request #644 from datafold/LAB-105

poll more frequently when using --cloud

Author: dlawin

data_diff/cloud/datafold_api.py

@@ -7,7 +7,7 @@
 import pydantic
 import requests
 
-from data_diff.errors import DataDiffDatasourceIdNotFoundError
+from data_diff.errors import DataDiffCloudDiffFailed, DataDiffCloudDiffTimedOut, DataDiffDatasourceIdNotFoundError
 
 from ..utils import getLogger
 
@@ -248,8 +248,8 @@ def create_data_diff(self, payload: TCloudApiDataDiff) -> int:
     def poll_data_diff_results(self, diff_id: int) -> TCloudApiDataDiffSummaryResult:
         summary_results = None
         start_time = time.monotonic()
-        sleep_interval = 5  # starts at 5 sec
-        max_sleep_interval = 30
+        sleep_interval = 3
+        max_sleep_interval = 20
         max_wait_time = 300
 
         diff_url = f"{self.host}/datadiffs/{diff_id}/overview"
@@ -260,13 +260,15 @@ def poll_data_diff_results(self, diff_id: int) -> TCloudApiDataDiffSummaryResult
             if response_json["status"] == "success":
                 summary_results = response_json
             elif response_json["status"] == "failed":
-                raise Exception(f"Diff failed: {str(response_json)}")
+                raise DataDiffCloudDiffFailed(f"Diff failed: {str(response_json)}")
 
             if time.monotonic() - start_time > max_wait_time:
-                raise Exception(f"Timed out waiting for diff results. Please, go to the UI for details: {diff_url}")
+                raise DataDiffCloudDiffTimedOut(
+                    f"Timed out waiting for diff results. Please, go to the UI for details: {diff_url}"
+                )
 
             time.sleep(sleep_interval)
-            sleep_interval = min(sleep_interval * 2, max_sleep_interval)
+            sleep_interval = min(sleep_interval + 1, max_sleep_interval)
 
         return TCloudApiDataDiffSummaryResult.from_orm(summary_results)
 

data_diff/errors.py

@@ -60,3 +60,11 @@ class DataDiffNoDatasourceIdError(Exception):
 
 class DataDiffDatasourceIdNotFoundError(Exception):
     "Raised when using --cloud but the datasource_id is not found for a particular org."
+
+
+class DataDiffCloudDiffFailed(Exception):
+    "Raised when using --cloud and the remote diff fails."
+
+
+class DataDiffCloudDiffTimedOut(Exception):
+    "Raised when using --cloud and the diff did not return finish before the timeout value."