Merge pull request #146 from oracle/dev/v1.8.0

v1.8.0

Author: aosingh

.github/workflows/oracle-xe-adapter-tests.yml

@@ -48,7 +48,7 @@ jobs:
       - name: Install dbt-oracle with core dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest 'dbt-tests-adapter~=1.7,<1.8'
+          pip install pytest 'dbt-tests-adapter~=1.8,<1.9'
           pip install -r requirements.txt
           pip install -e .
 

Makefile

@@ -1,5 +1,5 @@
 # Configuration variables
-VERSION=1.7.5
+VERSION=1.8.0
 PROJ_DIR?=$(shell pwd)
 VENV_DIR?=${PROJ_DIR}/.bldenv
 BUILD_DIR=${PROJ_DIR}/build

dbt/adapters/oracle/__init__.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2022, Oracle and/or its affiliates.
+Copyright (c) 2024, Oracle and/or its affiliates.
 Copyright (c) 2020, Vitor Avancini
 
   Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +17,6 @@
 from dbt.adapters.oracle.connections import OracleAdapterConnectionManager
 from dbt.adapters.oracle.connections import OracleAdapterCredentials
 from dbt.adapters.oracle.impl import OracleAdapter
-
 from dbt.adapters.base import AdapterPlugin
 from dbt.include import oracle
 

dbt/adapters/oracle/__version__.py

@@ -14,4 +14,4 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-version = "1.7.14"
+version = "1.8.0"

dbt/adapters/oracle/connection_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2023, Oracle and/or its affiliates.
+Copyright (c) 2024, Oracle and/or its affiliates.
 Copyright (c) 2020, Vitor Avancini
 
   Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,10 +17,10 @@
 import enum
 import os
 
-import dbt.exceptions
-from dbt.events import AdapterLogger
+import dbt_common.exceptions
+from dbt.adapters.events.logging import AdapterLogger
 
-from dbt.ui import warning_tag, yellow, red
+from dbt_common.ui import warning_tag, yellow, red
 
 logger = AdapterLogger("oracle")
 
@@ -129,5 +129,5 @@ class OracleDriverType(str, enum.Enum):
     SQLNET_ORA_CONFIG = OracleNetConfig.from_env()
     logger.info("Running in thin mode")
 else:
-    raise dbt.exceptions.DbtRuntimeError("Invalid value set for ORA_PYTHON_DRIVER_TYPE\n"
-                                         "Use any one of 'cx', 'thin', or 'thick'")
+    raise dbt_common.exceptions.DbtRuntimeError("Invalid value set for ORA_PYTHON_DRIVER_TYPE\n"
+                                                "Use any one of 'cx', 'thin', or 'thick'")

dbt/adapters/oracle/connections.py

@@ -23,15 +23,16 @@
 import uuid
 import platform
 
-import dbt.exceptions
-from dbt.adapters.base import Credentials
+import dbt_common.exceptions
+from dbt.adapters.contracts.connection import AdapterResponse, Credentials
+from dbt.adapters.exceptions.connection import FailedToConnectError
 from dbt.adapters.sql import SQLConnectionManager
-from dbt.contracts.connection import AdapterResponse
-from dbt.events.functions import fire_event
-from dbt.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
-from dbt.events import AdapterLogger
-from dbt.events.contextvars import get_node_info
-from dbt.utils import cast_to_str
+from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
+from dbt.adapters.events.logging import AdapterLogger
+
+from dbt_common.events.functions import fire_event
+from dbt_common.events.contextvars import get_node_info
+from dbt_common.utils import cast_to_str
 
 from dbt.version import __version__ as dbt_version
 from dbt.adapters.oracle.connection_helper import oracledb, SQLNET_ORA_CONFIG
@@ -256,7 +257,7 @@ def open(cls, connection):
             connection.handle = None
             connection.state = 'fail'
 
-            raise dbt.exceptions.FailedToConnectError(str(e))
+            raise FailedToConnectError(str(e))
 
         return connection
 
@@ -302,18 +303,18 @@ def exception_handler(self, sql):
                 logger.info("Failed to release connection!")
                 pass
 
-            raise dbt.exceptions.DbtDatabaseError(str(e).strip()) from e
+            raise dbt_common.exceptions.DbtDatabaseError(str(e).strip()) from e
 
         except Exception as e:
             logger.info("Rolling back transaction.")
             self.release()
-            if isinstance(e, dbt.exceptions.DbtRuntimeError):
+            if isinstance(e, dbt_common.exceptions.DbtRuntimeError):
                 # during a sql query, an internal to dbt exception was raised.
                 # this sounds a lot like a signal handler and probably has
                 # useful information, so raise it without modification.
                 raise e
 
-            raise dbt.exceptions.DbtRuntimeError(str(e)) from e
+            raise dbt_common.exceptions.DbtRuntimeError(str(e)) from e
 
     @classmethod
     def get_credentials(cls, credentials):

dbt/adapters/oracle/impl.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2023, Oracle and/or its affiliates.
+Copyright (c) 2024, Oracle and/or its affiliates.
 Copyright (c) 2020, Vitor Avancini
 
   Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,9 +16,8 @@
 """
 import datetime
 from typing import (
-    Optional, List, Set
+    Optional, List, Set, FrozenSet, Tuple, Iterable
 )
-from itertools import chain
 from typing import (
     Any,
     Callable,
@@ -27,24 +26,26 @@
 import agate
 import requests
 
-import dbt.exceptions
+import dbt_common.exceptions
+from dbt_common.contracts.constraints import ConstraintType
+from dbt_common.utils import filter_null_values
+
+from dbt.adapters.base.connections import Connection
 from dbt.adapters.base.relation import BaseRelation, InformationSchema
-from dbt.adapters.base.impl import GET_CATALOG_MACRO_NAME, ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value
+from dbt.adapters.base.impl import ConstraintSupport, GET_CATALOG_RELATIONS_MACRO_NAME, _expect_row_value
+from dbt.adapters.contracts.relation import RelationConfig
+from dbt.adapters.events.logging import AdapterLogger
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.base.meta import available
 from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability
+
 from dbt.adapters.oracle import OracleAdapterConnectionManager
 from dbt.adapters.oracle.column import OracleColumn
 from dbt.adapters.oracle.relation import OracleRelation
-from dbt.contracts.graph.manifest import Manifest
-from dbt.contracts.graph.nodes import ConstraintType
-from dbt.events import AdapterLogger
-
-from dbt.utils import filter_null_values
-
 from dbt.adapters.oracle.keyword_catalog import KEYWORDS
 from dbt.adapters.oracle.python_submissions import OracleADBSPythonJob
-from dbt.adapters.oracle.connections import AdapterResponse
+
+from dbt_common.ui import warning_tag, yellow
 
 logger = AdapterLogger("oracle")
 
@@ -82,6 +83,15 @@
 LIST_RELATIONS_MACRO_NAME = 'list_relations_without_caching'
 GET_DATABASE_MACRO_NAME = 'get_database_name'
 
+MISSING_DATABASE_NAME_FOR_CATALOG_WARNING_MESSAGE = (
+    "database key is missing from the target profile in the file profiles.yml "
+    "\n Starting with dbt-oracle 1.8  database name is needed for catalog generation "
+    "\n Without database key in the target profile the generated catalog will be empty "
+    "\n i.e. `dbt docs generate` command will generate an empty catalog json "
+    "\n Make the following entry in dbt profile.yml file for the target profile "
+    "\n database: {0}"
+)
+
 
 class OracleAdapter(SQLAdapter):
     ConnectionManager = OracleAdapterConnectionManager
@@ -138,8 +148,8 @@ def verify_database(self, database):
         if database.startswith('"'):
             database = database.strip('"')
         expected = self.config.credentials.database
-        if expected and database.lower() != expected.lower():
-            raise dbt.exceptions.DbtRuntimeError(
+        if expected and database.lower() != 'none' and database.lower() != expected.lower():
+            raise dbt_common.exceptions.DbtRuntimeError(
                 'Cross-db references not allowed in {} ({} vs {})'
                 .format(self.type(), database, expected)
             )
@@ -206,71 +216,52 @@ def get_relation(self, database: str, schema: str, identifier: str) -> Optional[
             database = self.config.credentials.database
         return super().get_relation(database, schema, identifier)
 
-    def _get_one_catalog(
+    def _get_one_catalog_by_relations(
             self,
             information_schema: InformationSchema,
-            schemas: Set[str],
-            manifest: Manifest,
-    ) -> agate.Table:
-
-        kwargs = {"information_schema": information_schema, "schemas": schemas}
-        table = self.execute_macro(
-            GET_CATALOG_MACRO_NAME,
-            kwargs=kwargs,
-            # pass in the full manifest so we get any local project
-            # overrides
-            manifest=manifest,
-        )
-        # In case database is not defined, we can use the the configured database which we set as part of credentials
-        for node in chain(manifest.nodes.values(), manifest.sources.values()):
-            if not node.database or node.database == 'None':
-                node.database = self.config.credentials.database
-
-        results = self._catalog_filter_table(table, manifest)
-        return results
-
-    def _get_one_catalog_by_relations(
-        self,
-        information_schema: InformationSchema,
-        relations: List[BaseRelation],
-        manifest: Manifest,
-    ) -> agate.Table:
-
+            relations: List[BaseRelation],
+            used_schemas: FrozenSet[Tuple[str, str]],
+    ) -> "agate.Table":
         kwargs = {
             "information_schema": information_schema,
             "relations": relations,
         }
-        table = self.execute_macro(
-            GET_CATALOG_RELATIONS_MACRO_NAME,
-            kwargs=kwargs,
-            # pass in the full manifest, so we get any local project
-            # overrides
-            manifest=manifest,
-        )
-
-        # In case database is not defined, we can use the the configured database which we set as part of credentials
-        for node in chain(manifest.nodes.values(), manifest.sources.values()):
-            if not node.database or node.database == 'None':
-                node.database = self.config.credentials.database
-
-        results = self._catalog_filter_table(table, manifest)  # type: ignore[arg-type]
+        table = self.execute_macro(GET_CATALOG_RELATIONS_MACRO_NAME, kwargs=kwargs)
+        results = self._catalog_filter_table(table, used_schemas)  # type: ignore[arg-type]
         return results
 
     def get_filtered_catalog(
-        self, manifest: Manifest, relations: Optional[Set[BaseRelation]] = None
+            self,
+            relation_configs: Iterable[RelationConfig],
+            used_schemas: FrozenSet[Tuple[str, str]],
+            relations: Optional[Set[BaseRelation]] = None
     ):
         catalogs: agate.Table
+
+        def is_database_none(database):
+            return database is None or database == 'None'
+
+        def populate_database(database):
+            if not is_database_none(database):
+                return database
+            return self.config.credentials.database
+
+        # In case database is not defined, we can use database set in credentials object
+        if any(is_database_none(database) for database, schema in used_schemas):
+            used_schemas = frozenset([(populate_database(database).casefold(), schema)
+                                      for database, schema in used_schemas])
+
         if (
             relations is None
             or len(relations) > 100
             or not self.supports(Capability.SchemaMetadataByRelations)
         ):
             # Do it the traditional way. We get the full catalog.
-            catalogs, exceptions = self.get_catalog(manifest)
+            catalogs, exceptions = self.get_catalog(relation_configs, used_schemas)
         else:
             # Do it the new way. We try to save time by selecting information
             # only for the exact set of relations we are interested in.
-            catalogs, exceptions = self.get_catalog_by_relations(manifest, relations)
+            catalogs, exceptions = self.get_catalog_by_relations(used_schemas, relations)
 
         if relations and catalogs:
             relation_map = {
@@ -388,8 +379,8 @@ def quote_seed_column(
         elif quote_config is None:
             pass
         else:
-            raise dbt.exceptions.CompilationError(f'The seed configuration value of "quote_columns" '
-                                                  f'has an invalid type {type(quote_config)}')
+            raise dbt_common.exceptions.CompilationError(f'The seed configuration value of "quote_columns" '
+                                                         f'has an invalid type {type(quote_config)}')
 
         if quote_columns:
             return self.quote(column)
@@ -417,7 +408,7 @@ def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]])
 
     def get_oml_auth_token(self) -> str:
         if self.config.credentials.oml_auth_token_uri is None:
-            raise dbt.exceptions.DbtRuntimeError("oml_auth_token_uri should be set to run dbt-py models")
+            raise dbt_common.exceptions.DbtRuntimeError("oml_auth_token_uri should be set to run dbt-py models")
         data = {
             "grant_type": "password",
             "username": self.config.credentials.user,
@@ -428,7 +419,7 @@ def get_oml_auth_token(self) -> str:
                               json=data)
             r.raise_for_status()
         except requests.exceptions.RequestException:
-            raise dbt.exceptions.DbtRuntimeError("Error getting OML OAuth2.0 token")
+            raise dbt_common.exceptions.DbtRuntimeError("Error getting OML OAuth2.0 token")
         else:
             return r.json()["accessToken"]
 
@@ -458,3 +449,15 @@ def submit_python_job(self, parsed_model: dict, compiled_code: str):
         response, _ = self.execute(sql=py_q_drop_script)
         logger.info(response)
         return response
+
+    def acquire_connection(self, name=None) -> Connection:
+        connection = self.connections.set_connection_name(name)
+        if connection.credentials.database is None or connection.credentials.database.lower() == 'none':
+            with connection.handle.cursor() as cr:
+                cr.execute("select SYS_CONTEXT('userenv', 'DB_NAME') FROM DUAL")
+                r = cr.fetchone()
+            database = r[0]
+            logger.warning(warning_tag(yellow(MISSING_DATABASE_NAME_FOR_CATALOG_WARNING_MESSAGE.format(database))))
+            self.config.credentials.database = database
+            connection.credentials.database = database
+        return connection