detect conflicts when schema_directory overlaps with paths in schemas_and_paths config

Author: codeforkjeff

README.md

@@ -33,8 +33,10 @@ dbt_sqlite:
       # connect schemas to paths: at least one of these must be 'main'
       schemas_and_paths: 'main=/my_project/data/etl.db;dataset=/my_project/data/dataset_v1.db'
 
-      # directory where all *.db files are attached as schema
-      schema_directory: '/myproject/data/schemas'
+      # directory where all *.db files are attached as schema, using base filename 
+      # as schema name, and where new schema are created. this can overlap with the dirs of
+      # files in schemas_and_paths as long as there's no conflicts.
+      schema_directory: '/myproject/data'
 
       # optional: semi-colon separated list of file paths for SQLite extensions to load.
       # digest.so is needed to provide for snapshots to work; see README
@@ -111,9 +113,8 @@ Install the `pytest-dbt-adapter` package and run the test specs in this reposito
 ```
 pip install pytest-dbt-adapter
 
-# these paths need to exist for tests to write data
+# this path needs to exist for tests to write database file
 mkdir -p /tmp/dbt-sqlite-tests
-mkdir -p /tmp/dbt-sqlite-tests/schemas
 
 pytest test/sqlite.dbtspec
 ```

dbt/adapters/sqlite/connections.py

@@ -49,7 +49,8 @@ def open(cls, connection):
         schemas_and_paths = {}
         for path_entry in credentials.schemas_and_paths.split(";"):
             schema, path = path_entry.split("=", 1)
-            schemas_and_paths[schema] = path
+            # store abs path so we can tell if we've attached the file already
+            schemas_and_paths[schema] = os.path.abspath(path)
 
         try:
             if 'main' in schemas_and_paths:
@@ -72,11 +73,20 @@ def open(cls, connection):
                 attached.append(schema)
 
             for path in glob.glob(os.path.join(credentials.schema_directory, "*.db")):
-                schema = os.path.basename(path)[:-3]
-                if path in schema:
-                    raise DatabaseException(
-                        f"cannot attach schema '{schema}': defined in profile but also exists in schema_directory: {path}")
-                cursor.execute(f"attach '{path}' as '{schema}'")
+                abs_path = os.path.abspath(path)
+
+                # if file was already attached from being defined in schemas_and_paths, ignore it
+                if not abs_path in schemas_and_paths.values():
+                    schema = os.path.basename(path)[:-3]
+
+                    # has schema name been used already?
+                    if schema not in attached:
+                        cursor.execute(f"attach '{path}' as '{schema}'")
+                    else:
+                        raise FailedToConnectException(
+                            f"found {path} while scanning schema_directory, but cannot attach it as '{schema}' " +
+                            f"because that schema name is already defined in schemas_and_paths. " +
+                            f"fix your ~/.dbt/profiles.yml file")
 
             # # uncomment these lines to print out SQL: this only happens if statement is successful
             # handle.set_trace_callback(print)

test/sqlite.dbtspec

@@ -4,7 +4,7 @@ target:
   database: adapter_test
   schema: 'main'
   schemas_and_paths: "main=/tmp/dbt-sqlite-tests/adapter_test.db"
-  schema_directory: '/tmp/dbt-sqlite-tests/schemas'
+  schema_directory: '/tmp/dbt-sqlite-tests'
   extensions: "/home/jeff/git/sqlite-digest/digest.so"
   threads: 1
 sequences: