Merge pull request #415 from dbeatty10/dbeatty10/dbt-profiles-dbt-project

Use the same logic as dbt-core for the path for the project and profiles

Author: dlawin

data_diff/__main__.py

@@ -217,15 +217,16 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
 )
 @click.option(
     "--dbt-profiles-dir",
+    envvar="DBT_PROFILES_DIR",
     default=None,
     metavar="PATH",
-    help="Override the default dbt profile location (~/.dbt).",
+    help="Which directory to look in for the profiles.yml file. If not set, we follow the default profiles.yml location for the dbt version being used. Can also be set via the DBT_PROFILES_DIR environment variable.",
 )
 @click.option(
     "--dbt-project-dir",
     default=None,
     metavar="PATH",
-    help="Override the dbt project directory. Otherwise assumed to be the current directory.",
+    help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.",
 )
 def main(conf, run, **kw):
     if kw["table2"] is None and kw["database2"]:

data_diff/dbt.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass
 from packaging.version import parse as parse_version
 from typing import List, Optional, Dict, Tuple
+from pathlib import Path
 
 import requests
 
@@ -31,14 +32,30 @@ def import_dbt():
 from .utils import get_from_dict_with_raise, run_as_daemon, truncate_error
 from . import connect_to_table, diff_tables, Algorithm
 
-RUN_RESULTS_PATH = "/target/run_results.json"
-MANIFEST_PATH = "/target/manifest.json"
-PROJECT_FILE = "/dbt_project.yml"
-PROFILES_FILE = "/profiles.yml"
+RUN_RESULTS_PATH = "target/run_results.json"
+MANIFEST_PATH = "target/manifest.json"
+PROJECT_FILE = "dbt_project.yml"
+PROFILES_FILE = "profiles.yml"
 LOWER_DBT_V = "1.0.0"
 UPPER_DBT_V = "1.4.2"
 
 
+# https://github.com/dbt-labs/dbt-core/blob/c952d44ec5c2506995fbad75320acbae49125d3d/core/dbt/cli/resolvers.py#L6
+def default_project_dir() -> Path:
+    paths = list(Path.cwd().parents)
+    paths.insert(0, Path.cwd())
+    return next((x for x in paths if (x / PROJECT_FILE).exists()), Path.cwd())
+
+
+# https://github.com/dbt-labs/dbt-core/blob/c952d44ec5c2506995fbad75320acbae49125d3d/core/dbt/cli/resolvers.py#L12
+def default_profiles_dir() -> Path:
+    return Path.cwd() if (Path.cwd() / PROFILES_FILE).exists() else Path.home() / ".dbt"
+
+
+def legacy_profiles_dir() -> Path:
+    return Path.home() / ".dbt"
+
+
 @dataclass
 class DiffVars:
     dev_path: List[str]
@@ -251,12 +268,9 @@ def _cloud_diff(diff_vars: DiffVars) -> None:
 
 
 class DbtParser:
-    DEFAULT_PROFILES_DIR = os.path.expanduser("~") + "/.dbt"
-    DEFAULT_PROJECT_DIR = os.getcwd()
-
     def __init__(self, profiles_dir_override: str, project_dir_override: str, is_cloud: bool) -> None:
-        self.profiles_dir = profiles_dir_override or self.DEFAULT_PROFILES_DIR
-        self.project_dir = project_dir_override or self.DEFAULT_PROJECT_DIR
+        self.profiles_dir = Path(profiles_dir_override or default_profiles_dir())
+        self.project_dir = Path(project_dir_override or default_project_dir())
         self.is_cloud = is_cloud
         self.connection = None
         self.project_dict = None
@@ -269,18 +283,21 @@ def get_datadiff_variables(self) -> dict:
         return self.project_dict.get("vars").get("data_diff")
 
     def get_models(self):
-        with open(self.project_dir + RUN_RESULTS_PATH) as run_results:
+        with open(self.project_dir / RUN_RESULTS_PATH) as run_results:
             run_results_dict = json.load(run_results)
             run_results_obj = self.parse_run_results(run_results=run_results_dict)
 
         dbt_version = parse_version(run_results_obj.metadata.dbt_version)
 
+        if dbt_version < parse_version("1.3.0"):
+            self.profiles_dir = legacy_profiles_dir()
+
         if dbt_version < parse_version(LOWER_DBT_V) or dbt_version >= parse_version(UPPER_DBT_V):
             raise Exception(
                 f"Found dbt: v{dbt_version} Expected the dbt project's version to be >= {LOWER_DBT_V} and < {UPPER_DBT_V}"
             )
 
-        with open(self.project_dir + MANIFEST_PATH) as manifest:
+        with open(self.project_dir / MANIFEST_PATH) as manifest:
             manifest_dict = json.load(manifest)
             manifest_obj = self.parse_manifest(manifest=manifest_dict)
 
@@ -296,11 +313,11 @@ def get_primary_keys(self, model):
         return list((x.name for x in model.columns.values() if "primary-key" in x.tags))
 
     def set_project_dict(self):
-        with open(self.project_dir + PROJECT_FILE) as project:
+        with open(self.project_dir / PROJECT_FILE) as project:
             self.project_dict = self.yaml.safe_load(project)
 
     def _get_connection_creds(self) -> Tuple[Dict[str, str], str]:
-        profiles_path = self.profiles_dir + PROFILES_FILE
+        profiles_path = self.profiles_dir / PROFILES_FILE
         with open(profiles_path) as profiles:
             profiles = self.yaml.safe_load(profiles)
 

tests/test_dbt.py

@@ -1,5 +1,6 @@
 import os
 
+from pathlib import Path
 import yaml
 from data_diff.diff_tables import Algorithm
 from .test_cli import run_datadiff_cli
@@ -51,7 +52,7 @@ def test_get_datadiff_variables_empty(self):
     def test_get_models(self, mock_open):
         expected_value = "expected_value"
         mock_self = Mock()
-        mock_self.project_dir = ""
+        mock_self.project_dir = Path()
         mock_run_results = Mock()
         mock_success_result = Mock()
         mock_failed_result = Mock()
@@ -69,47 +70,47 @@ def test_get_models(self, mock_open):
         models = DbtParser.get_models(mock_self)
 
         self.assertEqual(expected_value, models[0])
-        mock_open.assert_any_call(RUN_RESULTS_PATH)
-        mock_open.assert_any_call(MANIFEST_PATH)
+        mock_open.assert_any_call(Path(RUN_RESULTS_PATH))
+        mock_open.assert_any_call(Path(MANIFEST_PATH))
         mock_self.parse_run_results.assert_called_once_with(run_results={})
         mock_self.parse_manifest.assert_called_once_with(manifest={})
 
     @patch("builtins.open", new_callable=mock_open, read_data="{}")
     def test_get_models_bad_lower_dbt_version(self, mock_open):
         mock_self = Mock()
-        mock_self.project_dir = ""
+        mock_self.project_dir = Path()
         mock_run_results = Mock()
         mock_self.parse_run_results.return_value = mock_run_results
         mock_run_results.metadata.dbt_version = "0.19.0"
 
         with self.assertRaises(Exception) as ex:
             DbtParser.get_models(mock_self)
 
-        mock_open.assert_called_once_with(RUN_RESULTS_PATH)
+        mock_open.assert_called_once_with(Path(RUN_RESULTS_PATH))
         mock_self.parse_run_results.assert_called_once_with(run_results={})
         mock_self.parse_manifest.assert_not_called()
         self.assertIn("version to be", ex.exception.args[0])
 
     @patch("builtins.open", new_callable=mock_open, read_data="{}")
     def test_get_models_bad_upper_dbt_version(self, mock_open):
         mock_self = Mock()
-        mock_self.project_dir = ""
+        mock_self.project_dir = Path()
         mock_run_results = Mock()
         mock_self.parse_run_results.return_value = mock_run_results
         mock_run_results.metadata.dbt_version = "1.5.1"
 
         with self.assertRaises(Exception) as ex:
             DbtParser.get_models(mock_self)
 
-        mock_open.assert_called_once_with(RUN_RESULTS_PATH)
+        mock_open.assert_called_once_with(Path(RUN_RESULTS_PATH))
         mock_self.parse_run_results.assert_called_once_with(run_results={})
         mock_self.parse_manifest.assert_not_called()
         self.assertIn("version to be", ex.exception.args[0])
 
     @patch("builtins.open", new_callable=mock_open, read_data="{}")
     def test_get_models_no_success(self, mock_open):
         mock_self = Mock()
-        mock_self.project_dir = ""
+        mock_self.project_dir = Path()
         mock_run_results = Mock()
         mock_success_result = Mock()
         mock_failed_result = Mock()
@@ -126,21 +127,22 @@ def test_get_models_no_success(self, mock_open):
         with self.assertRaises(Exception):
             DbtParser.get_models(mock_self)
 
-        mock_open.assert_any_call(RUN_RESULTS_PATH)
-        mock_open.assert_any_call(MANIFEST_PATH)
+        mock_open.assert_any_call(Path(RUN_RESULTS_PATH))
+        mock_open.assert_any_call(Path(MANIFEST_PATH))
         mock_self.parse_run_results.assert_called_once_with(run_results={})
         mock_self.parse_manifest.assert_called_once_with(manifest={})
 
     @patch("builtins.open", new_callable=mock_open, read_data="key:\n  value")
     def test_set_project_dict(self, mock_open):
         expected_dict = {"key1": "value1"}
         mock_self = Mock()
-        mock_self.project_dir = ""
+
+        mock_self.project_dir = Path()
         mock_self.yaml.safe_load.return_value = expected_dict
         DbtParser.set_project_dict(mock_self)
 
         self.assertEqual(mock_self.project_dict, expected_dict)
-        mock_open.assert_called_once_with(PROJECT_FILE)
+        mock_open.assert_called_once_with(Path(PROJECT_FILE))
 
     def test_set_connection_snowflake_success(self):
         expected_driver = "snowflake"
@@ -222,7 +224,7 @@ def test_get_connection_creds_success(self, mock_open):
         profile = profiles_dict["a_profile"]
         expected_credentials = profiles_dict["a_profile"]["outputs"]["a_target"]
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         mock_self.project_dict = {"profile": "a_profile"}
         mock_self.yaml.safe_load.return_value = profiles_dict
         mock_self.ProfileRenderer().render_data.return_value = profile
@@ -234,7 +236,7 @@ def test_get_connection_creds_success(self, mock_open):
     def test_get_connection_no_matching_profile(self, mock_open):
         profiles_dict = {"a_profile": {}}
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         mock_self.project_dict = {"profile": "wrong_profile"}
         mock_self.yaml.safe_load.return_value = profiles_dict
         profile = profiles_dict["a_profile"]
@@ -252,7 +254,7 @@ def test_get_connection_no_target(self, mock_open):
             }
         }
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         profile = profiles_dict["a_profile"]
         mock_self.ProfileRenderer().render_data.return_value = profile
         mock_self.project_dict = {"profile": "a_profile"}
@@ -269,7 +271,7 @@ def test_get_connection_no_target(self, mock_open):
     def test_get_connection_no_outputs(self, mock_open):
         profiles_dict = {"a_profile": {"target": "a_target"}}
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         mock_self.project_dict = {"profile": "a_profile"}
         profile = profiles_dict["a_profile"]
         mock_self.ProfileRenderer().render_data.return_value = profile
@@ -286,7 +288,7 @@ def test_get_connection_no_credentials(self, mock_open):
             }
         }
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         mock_self.project_dict = {"profile": "a_profile"}
         mock_self.yaml.safe_load.return_value = profiles_dict
         profile = profiles_dict["a_profile"]
@@ -305,7 +307,7 @@ def test_get_connection_no_target_credentials(self, mock_open):
             }
         }
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         mock_self.project_dict = {"profile": "a_profile"}
         profile = profiles_dict["a_profile"]
         mock_self.ProfileRenderer().render_data.return_value = profile
@@ -322,7 +324,7 @@ def test_get_connection_no_type(self, mock_open):
             }
         }
         mock_self = Mock()
-        mock_self.profiles_dir = ""
+        mock_self.profiles_dir = Path()
         mock_self.project_dict = {"profile": "a_profile"}
         mock_self.yaml.safe_load.return_value = profiles_dict
         profile = profiles_dict["a_profile"]