Split tests into FULL_TESTS and regular

Author: erezsh

.github/workflows/ci.yml

@@ -46,6 +46,7 @@ jobs:
         env:
             DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}'
             DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}'
+            DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
             DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
         run: |

.github/workflows/ci_full.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  push:
+    paths:
+      - '**.py'
+      - '.github/workflows/**'
+      - '!dev/**'
+  pull_request:
+    branches: [ master ]
+
+  workflow_dispatch:
+
+jobs:
+  unit_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version:
+          - "3.10"
+
+    name: Check Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build the stack
+        run: docker-compose up -d mysql postgres presto trino clickhouse vertica
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Install package
+        run: "poetry install"
+
+      - name: Run unit tests
+        env:
+            DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}'
+            DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}'
+            DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
+            DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
+        run: |
+          chmod +x tests/waiting_for_stack_up.sh
+          ./tests/waiting_for_stack_up.sh && FULL_TESTS=1 poetry run unittest-parallel -j 16

data_diff/databases/clickhouse.py

@@ -55,7 +55,6 @@ class Dialect(BaseDialect):
         "DateTime64": Timestamp,
     }
 
-
     def normalize_number(self, value: str, coltype: FractionalType) -> str:
         # If a decimal value has trailing zeros in a fractional part, when casting to string they are dropped.
         # For example:

data_diff/databases/oracle.py

@@ -127,9 +127,7 @@ def parse_type(
             precision = int(m.group(1))
             return t_cls(precision=precision, rounds=self.ROUNDS_ON_PREC_LOSS)
 
-        return super().parse_type(
-            table_path, col_name, type_repr, datetime_precision, numeric_precision, numeric_scale
-        )
+        return super().parse_type(table_path, col_name, type_repr, datetime_precision, numeric_precision, numeric_scale)
 
 
 class Oracle(ThreadedDatabase):

tests/common.py

@@ -36,6 +36,7 @@
 N_SAMPLES = int(os.environ.get("N_SAMPLES", DEFAULT_N_SAMPLES))
 BENCHMARK = os.environ.get("BENCHMARK", False)
 N_THREADS = int(os.environ.get("N_THREADS", 1))
+FULL_TESTS = bool(os.environ.get("FULL_TESTS", False))      # Should we run the full db<->db test suite?
 
 
 def get_git_revision_short_hash() -> str:
@@ -94,6 +95,8 @@ def _print_used_dbs():
     logging.info(f"Testing databases: {', '.join(used)}")
     if unused:
         logging.info(f"Connection not configured; skipping tests for: {', '.join(unused)}")
+    if FULL_TESTS:
+        logging.info("Full tests enabled (every db<->db). May take very long when many dbs are involved.")
 
 
 _print_used_dbs()

tests/test_database_types.py

@@ -25,6 +25,7 @@
     N_THREADS,
     BENCHMARK,
     GIT_REVISION,
+    FULL_TESTS,
     get_conn,
     random_table_suffix,
 )
@@ -418,6 +419,7 @@ def __iter__(self):
     "uuid": UUID_Faker(N_SAMPLES),
 }
 
+
 def _get_test_db_pairs(full):
     if full:
         for source_db in DATABASE_TYPES:
@@ -430,14 +432,15 @@ def _get_test_db_pairs(full):
             yield db_cls, db.Snowflake
             yield db.Snowflake, db_cls
 
+
 def get_test_db_pairs(full=True):
     active_pairs = {(db1, db2) for db1, db2 in _get_test_db_pairs(full) if db1 in CONN_STRINGS and db2 in CONN_STRINGS}
     for db1, db2 in active_pairs:
         yield db1, DATABASE_TYPES[db1], db2, DATABASE_TYPES[db2]
 
 
 type_pairs = []
-for source_db, source_type_categories, target_db, target_type_categories in get_test_db_pairs():
+for source_db, source_type_categories, target_db, target_type_categories in get_test_db_pairs(FULL_TESTS):
     for type_category, source_types in source_type_categories.items():  # int, datetime, ..
         for source_type in source_types:
             for target_type in target_type_categories[type_category]: