compatibility with dbt 1.2.0rc1 (#24)

* reorganize Dockerfile and use docker in github CI
* in dbt-core 1.2.x, commit is now being done in SQLAdapter.drop_schema(), so there's no need for it here anymore. see this PR: dbt-labs/dbt-core#5198
* rm file on disk on DROP SCHEMA
* load text.so sqlean extension to get split_parts fn
* add math.so for functions needed by datediff

Author: codeforkjeff

.dockerignore

@@ -0,0 +1 @@
+*.log

.github/workflows/ci.yml

@@ -5,27 +5,15 @@ on: [push]
 
 jobs:
   build:
-
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python_version: ["3.8", "3.9", "3.10"]
+
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install from requirements.txt
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest pytest-dotenv dbt-tests-adapter==1.1.0
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Create data directory and install crypto.so
-        run: |
-          mkdir -p /tmp/dbt-sqlite-tests
-          cd /tmp/dbt-sqlite-tests && wget https://github.com/nalgeon/sqlean/releases/download/0.12.2/crypto.so
-      - name: Run test script
-        run: |
-          ./run_tests.sh
+      - name: Build docker image
+        run: docker build --build-arg PYTHON_VERSION=${{ matrix.python_version }}  --tag dbt-sqlite:$GITHUB_SHA .
+      - name: Run tests
+        run: docker run dbt-sqlite:$GITHUB_SHA run_tests.sh

.gitignore

@@ -14,3 +14,5 @@ env/
 *.sublime-*
 
 .idea/
+
+*.log

Dockerfile

@@ -1,18 +1,33 @@
 
-FROM ubuntu:20.04
+ARG PYTHON_VERSION=3.9
 
-RUN apt-get update && apt-get -y install git make python3 python3-pip python3-venv sqlite3 vim virtualenvwrapper wget
+FROM python:${PYTHON_VERSION}-bullseye
 
-RUN mkdir /root/dbt-sqlite
+RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv sqlite3 vim virtualenvwrapper wget
 
-WORKDIR /root/dbt-sqlite
+WORKDIR /opt/dbt-sqlite
 
-RUN mkdir -p /tmp/dbt-sqlite-tests
+RUN python3 -m pip install --upgrade pip \
+    && python3 -m pip install pytest pytest-dotenv dbt-core~=1.2.0 dbt-tests-adapter~=1.2.0
 
-RUN cd /tmp/dbt-sqlite-tests && wget https://github.com/nalgeon/sqlean/releases/download/0.12.2/crypto.so
+RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/crypto.so
+RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/math.so
+RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/text.so
 
-RUN pip install dbt-core~=1.1.0
+WORKDIR /opt/dbt-sqlite/src 
 
-RUN pip install pytest pytest-dotenv dbt-tests-adapter==1.1.0
+COPY . .
 
-ENTRYPOINT ["./run_tests.sh"]
+RUN pip install .
+
+ENV TESTDATA=/opt/dbt-sqlite/testdata
+
+RUN mkdir $TESTDATA
+
+VOLUME /opt/dbt-sqlite/testdata
+
+WORKDIR /opt/dbt-sqlite/project
+
+ENV PATH=$PATH:/opt/dbt-sqlite/src
+
+VOLUME /opt/dbt-sqlite/project

README.md

@@ -80,10 +80,14 @@ dbt_sqlite:
       # files in schemas_and_paths as long as there's no conflicts.
       schema_directory: '/my_project/data'
 
-      # optional: list of file paths of SQLite extensions to load.
-      # crypto.so is needed for snapshots to work; see README
+      # optional: list of file paths of SQLite extensions to load. see README.
+      # crypto.so is needed for snapshots to work
+      # math.so is needed for ceil function
+      # text.so is needed for split_part macro to work
       extensions:
         - "/path/to/sqlean/crypto.so"
+        - "/path/to/sqlean/math.so"
+        - "/path/to/sqlean/text.so"
 
 ```
 
@@ -126,12 +130,14 @@ not worth the effort.
 
 ## SQLite Extensions
 
-For snapshots to work, you'll need the `crypto` module from SQLean to get an `md5()`
-function. It's recommended that you install all the SQLean modules, as they provide
-many common SQL functions missing from SQLite.
+These modules from SQLean are needed for certain functionality to work:
+- `crypto`: provides `md5()` function needed for snapshots
+- `math`: provides `ceil` and `floor` needed for the datediff macro to work
+- `text`: provides `split_part` function
 
 Precompiled binaries are available for download from the [SQLean github repository page](https://github.com/nalgeon/sqlean).
-You can also compile them yourself if you want.
+You can also compile them yourself if you want. Note that some modules depend on other libraries 
+(`math` for example depends on GLIBC); if an extension fails to load, you may want to try building it yourself.
 
 Point to these module files in your profile config as shown in the example above.
 
@@ -177,10 +183,22 @@ git push --tags
 
 ## Running Tests
 
+This runs the test suite and cleans up after itself:
 ```
 ./run_tests_docker.sh
 ```
 
+To run tests interactively and be able to examine test artifacts:
+```
+docker build . -t dbt-sqlite
+
+docker run --rm -it dbt-sqlite bash
+
+# see output for the locations of artifacts
+run_tests.sh -s
+```
+
+
 ## Credits
 
 Inspired by this initial work by stephen1000: https://github.com/stephen1000/dbt_sqlite

dbt/adapters/sqlite/impl.py

@@ -1,5 +1,8 @@
 
+import datetime
 import decimal
+import os
+import os.path
 from typing import List, Optional, Set
 
 import agate
@@ -21,6 +24,33 @@ class SQLiteAdapter(SQLAdapter):
     def date_function(cls):
         return 'date()'
 
+    # sqlite reports the exact string (including case) used when declaring a column of a certain type
+
+    @classmethod
+    def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "TEXT"
+
+    @classmethod
+    def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))  # type: ignore[attr-defined]
+        return "NUMERIC" if decimals else "INT"
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "BOOLEAN"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "TIMESTAMP WITHOUT TIMEZONE"
+
+    @classmethod
+    def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "DATE"
+
+    @classmethod
+    def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str:
+        return "TIME"
+
     def get_live_relation_type(self, relation):
         """
         returns the type of relation (table, view) from the live database
@@ -244,11 +274,13 @@ def timestamp_add_sql(
     def drop_schema(self, relation: BaseRelation) -> None:
         super().drop_schema(relation)
 
-        # can't detach a database in the middle of a transaction, so commit first.
-        # I wonder if drop_schema() in SQLAdapter should do this, since create_schema() does.
-        self.commit_if_has_connection()
-
         # never detach main
         if relation.schema != 'main':
             if self.check_schema_exists(relation.database, relation.schema):
                 self.connections.execute(f"DETACH DATABASE {relation.schema}")
+
+                if relation.schema in self.config.credentials.schemas_and_paths:
+                    path = self.config.credentials.schemas_and_paths[relation.schema]
+                else:
+                    path = os.path.join(self.config.credentials.schema_directory, relation.schema + ".db")
+                os.remove(path)

dbt/include/sqlite/macros/materializations/seed/helpers.sql

@@ -0,0 +1,3 @@
+{% macro sqlite__get_binding_char() %}
+  {{ return('?') }}
+{% endmacro %}

dbt/include/sqlite/macros/materializations/seed/seed.sql

@@ -19,8 +19,7 @@
             insert into {{ this.schema }}.{{ this.identifier}} ({{ cols_sql }}) values
             {% for row in chunk -%}
                 ({%- for column in agate_table.column_names -%}
-                    {# sqlite uses ? as placeholder character #}
-                    ?
+                    {{ get_binding_char() }}
                     {%- if not loop.last%},{%- endif %}
                 {%- endfor -%})
                 {%- if not loop.last%},{%- endif %}

dbt/include/sqlite/macros/utils/any_value.sql

@@ -0,0 +1,5 @@
+{% macro sqlite__any_value(expression) -%}
+
+    min({{ expression }})
+
+{%- endmacro %}

dbt/include/sqlite/macros/utils/bool_or.sql

@@ -0,0 +1,5 @@
+{% macro sqlite__bool_or(expression) -%}
+
+    max({{ expression }})
+
+{%- endmacro %}