improve behavior of creating/dropping schema

Author: codeforkjeff

README.md

@@ -23,7 +23,7 @@ dbt_sqlite:
       # sqlite locks the whole db on writes so anything > 1 won't help
       threads: 1
 
-      # value of 'database' is arbitrary
+      # value is arbitrary
       database: "database"
 
       # value of 'schema' must be defined in schema_paths below. in most cases,
@@ -33,11 +33,11 @@ dbt_sqlite:
       # connect schemas to paths: at least one of these must be 'main'
       schemas_and_paths: 'main=/my_project/data/etl.db;dataset=/my_project/data/dataset_v1.db'
 
-      # directory where new schemas are created by dbt as new database files
+      # directory where all *.db files are attached as schema
       schema_directory: '/myproject/data/schemas'
 
       # optional: semi-colon separated list of file paths for SQLite extensions to load.
-      # digesto.so is needed to provide the md5 function needed for snapshots to work.
+      # digest.so is needed to provide the md5 function needed for snapshots to work.
       # see section in README on how to install it
       extensions: "/path/to/sqlite-digest/digest.so"
 
@@ -58,10 +58,9 @@ and schemas.)
   so this must be defined in your profile. Other schemas defined in your profile
   get attached when database connection is created.
 
-  - If dbt needs to create a new schema, it will be created in `schema_directory`.
-  Dropping a schema results in dropping all its relations but NOT detaching the
-  database, since this may result in a confusing conflict with the schemas you
-  defined in your profile.
+  - If dbt needs to create a new schema, it will be created in `schema_directory`
+  as `schema_name.db`. Dropping a schema results in dropping all its relations
+  and detaching the database file from the session.
 
   - Schema names are stored in view definitions, so when you access a non-'main'
   database file outside dbt, you'll need to attach it using the same name, or

dbt/adapters/sqlite/connections.py

@@ -1,6 +1,8 @@
 
 from contextlib import contextmanager
 from dataclasses import dataclass
+import glob
+import os.path
 import sqlite3
 from typing import List, Optional, Tuple, Any, Iterable, Dict
 
@@ -65,9 +67,18 @@ def open(cls, connection):
 
             cursor = handle.cursor()
 
+            attached = []
             for schema in set(schemas_and_paths.keys()) - set(['main']):
                 path = schemas_and_paths[schema]
                 cursor.execute(f"attach '{path}' as '{schema}'")
+                attached.append(schema)
+
+            for path in glob.glob(os.path.join(credentials.schema_directory, "*.db")):
+                schema = os.path.basename(path)[:-3]
+                if path in schema:
+                    raise DatabaseException(
+                        f"cannot attach schema '{schema}': defined in profile but also exists in schema_directory: {path}")
+                cursor.execute(f"attach '{path}' as '{schema}'")
 
             # # uncomment these lines to print out SQL: this only happens if statement is successful
             # handle.set_trace_callback(print)

dbt/adapters/sqlite/impl.py

@@ -264,3 +264,10 @@ def timestamp_add_sql(
         self, add_to: str, number: int = 1, interval: str = 'hour'
     ) -> str:
         return f"DATETIME({add_to}, '{number} {interval}')"
+
+    def drop_schema(self, relation: BaseRelation) -> None:
+        super().drop_schema(relation)
+
+        # never detach main
+        if relation.schema != 'main':
+            self.connections.execute(f"DETACH DATABASE {relation.schema}")

dbt/include/sqlite/macros/adapters.sql

@@ -3,15 +3,15 @@
     {# see SQLiteAdapter.list_schemas() #}
 {% endmacro %}
 
-{% macro sqlite__create_schema(database_name, schema_name, auto_begin=False) %}
-  {% set path = '/'.join(adapter.config.credentials.schema_directory, schema_name, '.db') %}
+{% macro sqlite__create_schema(relation, auto_begin=False) %}
+  {% set path = [ adapter.config.credentials.schema_directory, relation.without_identifier().include(database=False) | string + '.db' ] | join('/') %}
   {%- call statement('create_schema') -%}
-    attach database '{{ path }}' as {{ schema_name }}
+    attach database '{{ path }}' as {{ relation.without_identifier().include(database=False) }}
   {%- endcall -%}
 {% endmacro %}
 
 {% macro sqlite__drop_schema(relation) -%}
-  {# drop all tables in the schema, but leave the scgema itself alone. we can't detach 'main' #}
+  {# drop all tables in the schema; detaching happens in the adapter class #}
 
   {% set relations_in_schema = list_relations_without_caching(relation.without_identifier()) %}