Merge pull request #628 from datafold/LAB-72

use a different endpoint for ds metadata

Author: dlawin

data_diff/cloud/datafold_api.py

@@ -7,6 +7,8 @@
 import pydantic
 import requests
 
+from data_diff.errors import DataDiffDatasourceIdNotFoundError
+
 from ..utils import getLogger
 
 logger = getLogger(__name__)
@@ -207,9 +209,15 @@ def get_data_sources(self) -> List[TCloudApiDataSource]:
         return [TCloudApiDataSource(**item) for item in rv.json()]
 
     def get_data_source(self, data_source_id: int) -> TCloudApiDataSource:
-        rv = self.make_get_request(url=f"api/v1/data_sources/{data_source_id}")
+        rv = self.make_get_request(url=f"api/v1/data_sources")
         rv.raise_for_status()
-        return TCloudApiDataSource(**rv.json())
+        response_json = rv.json()
+        datasource = next((datasource for datasource in response_json if datasource["id"] == data_source_id), None)
+        if not datasource:
+            raise DataDiffDatasourceIdNotFoundError(
+                f"Datasource ID: {data_source_id} was not found in your Datafold account!"
+            )
+        return TCloudApiDataSource(**datasource)
 
     def create_data_source(self, config: TDsConfig) -> TCloudApiDataSource:
         payload = config.dict()

data_diff/errors.py

@@ -56,3 +56,7 @@ class DataDiffNoAPIKeyError(Exception):
 
 class DataDiffNoDatasourceIdError(Exception):
     "Raised when using --cloud but no datasource_id was found in dbt_project.yml"
+
+
+class DataDiffDatasourceIdNotFoundError(Exception):
+    "Raised when using --cloud but the datasource_id is not found for a particular org."