From 1e258597525a6b74338c65bd937174ab996c25fa Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 19 Feb 2024 09:49:58 -0500 Subject: [PATCH 01/12] Migrate to dbt-common + dbt-adapters --- dbt/adapters/duckdb/connections.py | 21 +++++++++++-------- dbt/adapters/duckdb/credentials.py | 11 +++++----- dbt/adapters/duckdb/environments/__init__.py | 4 ++-- .../duckdb/environments/buenavista.py | 2 +- dbt/adapters/duckdb/environments/local.py | 5 +++-- dbt/adapters/duckdb/impl.py | 14 ++++++------- dbt/adapters/duckdb/plugins/__init__.py | 2 +- dbt/adapters/duckdb/utils.py | 9 ++++---- dev-requirements.txt | 8 +++---- setup.py | 3 ++- 10 files changed, 42 insertions(+), 37 deletions(-) diff --git a/dbt/adapters/duckdb/connections.py b/dbt/adapters/duckdb/connections.py index afc7ab2c..35d6ad62 100644 --- a/dbt/adapters/duckdb/connections.py +++ b/dbt/adapters/duckdb/connections.py @@ -1,6 +1,7 @@ import atexit import threading from contextlib import contextmanager +from multiprocessing.context import SpawnContext from typing import Optional from typing import Tuple @@ -8,12 +9,14 @@ import dbt.exceptions from . import environments +from dbt.adapters.contracts.connection import AdapterRequiredConfig +from dbt.adapters.contracts.connection import AdapterResponse +from dbt.adapters.contracts.connection import Connection +from dbt.adapters.contracts.connection import ConnectionState +from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.sql import SQLConnectionManager -from dbt.contracts.connection import AdapterRequiredConfig -from dbt.contracts.connection import AdapterResponse -from dbt.contracts.connection import Connection -from dbt.contracts.connection import ConnectionState -from dbt.logger import GLOBAL_LOGGER as logger + +logger = AdapterLogger("DuckDB") class DuckDBConnectionManager(SQLConnectionManager): @@ -21,9 +24,9 @@ class DuckDBConnectionManager(SQLConnectionManager): _LOCK = threading.RLock() _ENV = None - def __init__(self, profile: AdapterRequiredConfig): - super().__init__(profile) - self.disable_transactions = profile.credentials.disable_transactions # type: ignore + def __init__(self, config: AdapterRequiredConfig, mp_context: SpawnContext) -> None: + super().__init__(config, mp_context) + self.disable_transactions = config.credentials.disable_transactions # type: ignore @classmethod def env(cls) -> environments.Environment: @@ -50,7 +53,7 @@ def open(cls, connection: Connection) -> Connection: logger.debug("Got an error when attempting to connect to DuckDB: '{}'".format(e)) connection.handle = None connection.state = ConnectionState.FAIL - raise dbt.exceptions.FailedToConnectError(str(e)) + raise dbt.adapters.exceptions.FailedToConnectError(str(e)) return connection diff --git a/dbt/adapters/duckdb/credentials.py b/dbt/adapters/duckdb/credentials.py index 763ae142..99c2dc22 100644 --- a/dbt/adapters/duckdb/credentials.py +++ b/dbt/adapters/duckdb/credentials.py @@ -10,9 +10,10 @@ from typing import Tuple from urllib.parse import urlparse -import dbt.exceptions -from dbt.adapters.base import Credentials -from dbt.dataclass_schema import dbtClassMixin +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_common.exceptions import DbtRuntimeError + +from dbt.adapters.contracts.connection import Credentials @dataclass @@ -178,12 +179,12 @@ def __pre_deserialize__(cls, data: Dict[Any, Any]) -> Dict[Any, Any]: data["database"] = path_db elif path_db and data["database"] != path_db: if not data.get("remote"): - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( "Inconsistency detected between 'path' and 'database' fields in profile; " f"the 'database' property must be set to '{path_db}' to match the 'path'" ) elif not path_db: - raise dbt.exceptions.DbtRuntimeError( + raise DbtRuntimeError( "Unable to determine target database name from 'path' field in profile" ) return data diff --git a/dbt/adapters/duckdb/environments/__init__.py b/dbt/adapters/duckdb/environments/__init__.py index 6b27a02f..e9e8e59a 100644 --- a/dbt/adapters/duckdb/environments/__init__.py +++ b/dbt/adapters/duckdb/environments/__init__.py @@ -9,13 +9,13 @@ from typing import Optional import duckdb +from dbt_common.exceptions import DbtRuntimeError from ..credentials import DuckDBCredentials from ..plugins import BasePlugin from ..utils import SourceConfig from ..utils import TargetConfig -from dbt.contracts.connection import AdapterResponse -from dbt.exceptions import DbtRuntimeError +from dbt.adapters.contracts.connection import AdapterResponse def _ensure_event_loop(): diff --git a/dbt/adapters/duckdb/environments/buenavista.py b/dbt/adapters/duckdb/environments/buenavista.py index b41aaf80..6ef4fa9d 100644 --- a/dbt/adapters/duckdb/environments/buenavista.py +++ b/dbt/adapters/duckdb/environments/buenavista.py @@ -5,7 +5,7 @@ from . import Environment from .. import credentials from .. import utils -from dbt.contracts.connection import AdapterResponse +from dbt.adapters.contracts.connection import AdapterResponse class BVEnvironment(Environment): diff --git a/dbt/adapters/duckdb/environments/local.py b/dbt/adapters/duckdb/environments/local.py index bdb297d3..cb37f251 100644 --- a/dbt/adapters/duckdb/environments/local.py +++ b/dbt/adapters/duckdb/environments/local.py @@ -1,10 +1,11 @@ import threading +from dbt_common.exceptions import DbtRuntimeError + from . import Environment from .. import credentials from .. import utils -from dbt.contracts.connection import AdapterResponse -from dbt.exceptions import DbtRuntimeError +from dbt.adapters.contracts.connection import AdapterResponse class DuckDBCursorWrapper: diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 47596c9a..c35d9ef9 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -5,11 +5,18 @@ from typing import Sequence import agate +from dbt_common.contracts.constraints import ColumnLevelConstraint +from dbt_common.contracts.constraints import ConstraintType +from dbt_common.exceptions import DbtInternalError +from dbt_common.exceptions import DbtRuntimeError from dbt.adapters.base import BaseRelation from dbt.adapters.base.column import Column as BaseColumn from dbt.adapters.base.impl import ConstraintSupport from dbt.adapters.base.meta import available +from dbt.adapters.contracts.connection import AdapterResponse +from dbt.adapters.contracts.relation import Path +from dbt.adapters.contracts.relation import RelationType from dbt.adapters.duckdb.column import DuckDBColumn from dbt.adapters.duckdb.connections import DuckDBConnectionManager from dbt.adapters.duckdb.relation import DuckDBRelation @@ -17,13 +24,6 @@ from dbt.adapters.duckdb.utils import TargetLocation from dbt.adapters.sql import SQLAdapter from dbt.context.providers import RuntimeConfigObject -from dbt.contracts.connection import AdapterResponse -from dbt.contracts.graph.nodes import ColumnLevelConstraint -from dbt.contracts.graph.nodes import ConstraintType -from dbt.contracts.relation import Path -from dbt.contracts.relation import RelationType -from dbt.exceptions import DbtInternalError -from dbt.exceptions import DbtRuntimeError TEMP_SCHEMA_NAME = "temp_schema_name" DEFAULT_TEMP_SCHEMA_NAME = "dbt_temp" diff --git a/dbt/adapters/duckdb/plugins/__init__.py b/dbt/adapters/duckdb/plugins/__init__.py index 2c658145..0dc463f1 100644 --- a/dbt/adapters/duckdb/plugins/__init__.py +++ b/dbt/adapters/duckdb/plugins/__init__.py @@ -4,12 +4,12 @@ from typing import Dict from typing import Optional +from dbt_common.dataclass_schema import dbtClassMixin from duckdb import DuckDBPyConnection from ..credentials import DuckDBCredentials from ..utils import SourceConfig from ..utils import TargetConfig -from dbt.dataclass_schema import dbtClassMixin class PluginConfig(dbtClassMixin): diff --git a/dbt/adapters/duckdb/utils.py b/dbt/adapters/duckdb/utils.py index b8e444f5..c6a31c87 100644 --- a/dbt/adapters/duckdb/utils.py +++ b/dbt/adapters/duckdb/utils.py @@ -7,8 +7,9 @@ from dbt.adapters.base.column import Column from dbt.adapters.base.relation import BaseRelation -from dbt.context.providers import RuntimeConfigObject -from dbt.contracts.graph.nodes import SourceDefinition +# TODO +# from dbt.context.providers import RuntimeConfigObject +# from dbt_common.contracts.graph.nodes import SourceDefinition @dataclass @@ -47,7 +48,7 @@ def as_dict(self) -> Dict[str, Any]: return base @classmethod - def create_from_source(cls, source: SourceDefinition) -> "SourceConfig": + def create_from_source(cls, source: Any) -> "SourceConfig": meta = source.source_meta.copy() meta.update(source.meta) # Use the config properties as well if they are present @@ -75,7 +76,7 @@ def as_dict(self) -> Dict[str, Any]: class TargetConfig: relation: BaseRelation column_list: Sequence[Column] - config: RuntimeConfigObject + config: Any # TODO location: Optional[TargetLocation] = None def as_dict(self) -> Dict[str, Any]: diff --git a/dev-requirements.txt b/dev-requirements.txt index dffc5a41..67b9ff73 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,6 @@ -# install latest changes in dbt-core -# git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -# git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter - -dbt-tests-adapter==1.7.7 +# install latest changes in dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter boto3 mypy-boto3-glue diff --git a/setup.py b/setup.py index c3f10a0d..9817eb62 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,8 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.7.0", + "dbt-common<1.0", + "dbt-adapters<=0.2.0", "duckdb>=0.7.0", ], extras_require={"glue": ["boto3", "mypy-boto3-glue"], "md": ["duckdb>=0.7.0,<=0.9.2"]}, From 86eface7b416d0bbe546f4c6412172ab8fa1d3a8 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 19 Feb 2024 10:06:22 -0500 Subject: [PATCH 02/12] Try different install reqs --- dev-requirements.txt | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 67b9ff73..2ee60438 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -# install latest changes in dbt-tests-adapter +# install latest changes in dbt-core + dbt-tests-adapter git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter diff --git a/setup.py b/setup.py index 9817eb62..a1e5e2b4 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,8 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common<1.0", - "dbt-adapters<=0.2.0", + "dbt-common>=0.1.0,<0.2.0", + "dbt-adapters>=0.1.0a1,<0.2.0", "duckdb>=0.7.0", ], extras_require={"glue": ["boto3", "mypy-boto3-glue"], "md": ["duckdb>=0.7.0,<=0.9.2"]}, From 2d3696893c6e80094fafa449b142af6e606fcca7 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 19 Feb 2024 11:04:05 -0500 Subject: [PATCH 03/12] Fix unit tests --- tests/unit/test_duckdb_adapter.py | 3 ++- tests/unit/test_external_utils.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_duckdb_adapter.py b/tests/unit/test_duckdb_adapter.py index 690949fd..4214aa74 100644 --- a/tests/unit/test_duckdb_adapter.py +++ b/tests/unit/test_duckdb_adapter.py @@ -39,8 +39,9 @@ def setUp(self): @property def adapter(self): + self.mock_mp_context = mock.MagicMock() if self._adapter is None: - self._adapter = DuckDBAdapter(self.config) + self._adapter = DuckDBAdapter(self.config, self.mock_mp_context) return self._adapter @mock.patch("dbt.adapters.duckdb.environments.duckdb") diff --git a/tests/unit/test_external_utils.py b/tests/unit/test_external_utils.py index 6a3b5a2c..1921e7ec 100644 --- a/tests/unit/test_external_utils.py +++ b/tests/unit/test_external_utils.py @@ -1,5 +1,6 @@ import unittest from argparse import Namespace +from unittest import mock from dbt.flags import set_from_args from dbt.adapters.duckdb import DuckDBAdapter @@ -32,8 +33,9 @@ def setUp(self): @property def adapter(self): + self.mock_mp_context = mock.MagicMock() if self._adapter is None: - self._adapter = DuckDBAdapter(self.config) + self._adapter = DuckDBAdapter(self.config, self.mock_mp_context) return self._adapter def test_external_write_options(self): From 3ad787e1a2406c6db50968edd8c8d06595a02c4e Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 19 Feb 2024 11:04:14 -0500 Subject: [PATCH 04/12] Bonus: functional tests for dbt unit testing --- tests/functional/adapter/test_unit_testing.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/functional/adapter/test_unit_testing.py diff --git a/tests/functional/adapter/test_unit_testing.py b/tests/functional/adapter/test_unit_testing.py new file mode 100644 index 00000000..bcc3cda6 --- /dev/null +++ b/tests/functional/adapter/test_unit_testing.py @@ -0,0 +1,35 @@ +import pytest + +from dbt.tests.adapter.unit_testing.test_types import BaseUnitTestingTypes +from dbt.tests.adapter.unit_testing.test_case_insensitivity import BaseUnitTestCaseInsensivity +from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput + + +class TestUnitTestingTypesDuckDB(BaseUnitTestingTypes): + @pytest.fixture + def data_types(self): + # sql_value, yaml_value + return [ + ["1", "1"], + ["2.0", "2.0"], + ["'12345'", "12345"], + ["'string'", "string"], + ["true", "true"], + ["DATE '2020-01-02'", "2020-01-02"], + ["TIMESTAMP '2013-11-03 00:00:00-0'", "2013-11-03 00:00:00-0"], + ["'2013-11-03 00:00:00-0'::TIMESTAMPTZ", "2013-11-03 00:00:00-0"], + [ + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + "{'Alberta':'Edmonton','Manitoba':'Winnipeg'}", + ], + ["ARRAY['a','b','c']", "['a','b','c']"], + ["ARRAY[1,2,3]", "[1, 2, 3]"], + ] + + +class TestUnitTestCaseInsensitivityDuckDB(BaseUnitTestCaseInsensivity): + pass + + +class TestUnitTestInvalidInputDuckDB(BaseUnitTestInvalidInput): + pass From d4d5939ffbe6e1c21ba10177b6c14cf0298dc205 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 29 Feb 2024 17:34:25 -0500 Subject: [PATCH 05/12] bump dbt-common and dbt-adapters to 1.0.0b1 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a1e5e2b4..1f257bc3 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,8 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=0.1.0,<0.2.0", - "dbt-adapters>=0.1.0a1,<0.2.0", + "dbt-common>=1.0.0b1,<2.0", + "dbt-adapters>=1.0.0b1,<2.0", "duckdb>=0.7.0", ], extras_require={"glue": ["boto3", "mypy-boto3-glue"], "md": ["duckdb>=0.7.0,<=0.9.2"]}, From 5712711939d39daace0ec505c7d7753ed6370193 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 29 Feb 2024 18:09:43 -0500 Subject: [PATCH 06/12] implement DuckDbRelation.create_from, fix TestExternalSources::test_external_sources --- dbt/adapters/duckdb/impl.py | 5 +++-- dbt/adapters/duckdb/relation.py | 21 ++++++++++++++++++--- dbt/adapters/duckdb/utils.py | 4 ++-- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index c35d9ef9..a778fe28 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -23,7 +23,8 @@ from dbt.adapters.duckdb.utils import TargetConfig from dbt.adapters.duckdb.utils import TargetLocation from dbt.adapters.sql import SQLAdapter -from dbt.context.providers import RuntimeConfigObject +# TODO +# from dbt.context.providers import RuntimeConfigObject TEMP_SCHEMA_NAME = "temp_schema_name" DEFAULT_TEMP_SCHEMA_NAME = "dbt_temp" @@ -99,7 +100,7 @@ def store_relation( column_list: Sequence[BaseColumn], path: str, format: str, - config: RuntimeConfigObject, + config: Any, ) -> None: target_config = TargetConfig( relation=relation, diff --git a/dbt/adapters/duckdb/relation.py b/dbt/adapters/duckdb/relation.py index 1eed82da..492ad6a2 100644 --- a/dbt/adapters/duckdb/relation.py +++ b/dbt/adapters/duckdb/relation.py @@ -8,15 +8,30 @@ from .utils import SourceConfig from dbt.adapters.base.relation import BaseRelation from dbt.adapters.base.relation import Self -from dbt.contracts.graph.nodes import SourceDefinition +from dbt.adapters.contracts.relation import ( + HasQuoting, + RelationConfig, +) @dataclass(frozen=True, eq=False, repr=False) class DuckDBRelation(BaseRelation): external: Optional[str] = None @classmethod - def create_from_source(cls: Type[Self], source: SourceDefinition, **kwargs: Any) -> Self: + def create_from( + cls: Type[Self], + quoting: HasQuoting, + relation_config: RelationConfig, + **kwargs: Any, + ) -> Self: + if relation_config.resource_type == 'source': + return cls.create_from_source(quoting, relation_config, **kwargs) + else: + return super().create_from(quoting, relation_config, **kwargs) + + @classmethod + def create_from_source(cls: Type[Self], quoting: HasQuoting, source: RelationConfig, **kwargs: Any) -> Self: """ This method creates a new DuckDBRelation instance from a source definition. It first checks if a 'plugin' is defined in the meta argument for the source or its parent configuration. @@ -59,7 +74,7 @@ def create_from_source(cls: Type[Self], source: SourceDefinition, **kwargs: Any) ext_location = f"'{ext_location}'" kwargs["external"] = ext_location - return super().create_from_source(source, **kwargs) # type: ignore + return super().create_from(quoting, source, **kwargs) # type: ignore def render(self) -> str: if self.external: diff --git a/dbt/adapters/duckdb/utils.py b/dbt/adapters/duckdb/utils.py index c6a31c87..2d7deaf7 100644 --- a/dbt/adapters/duckdb/utils.py +++ b/dbt/adapters/duckdb/utils.py @@ -7,9 +7,9 @@ from dbt.adapters.base.column import Column from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.contracts.relation import RelationConfig # TODO # from dbt.context.providers import RuntimeConfigObject -# from dbt_common.contracts.graph.nodes import SourceDefinition @dataclass @@ -48,7 +48,7 @@ def as_dict(self) -> Dict[str, Any]: return base @classmethod - def create_from_source(cls, source: Any) -> "SourceConfig": + def create_from_source(cls, source: RelationConfig) -> "SourceConfig": meta = source.source_meta.copy() meta.update(source.meta) # Use the config properties as well if they are present From 9c8113a4ccffc5bc1b8dfdeaef932f278f4e5f57 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Fri, 15 Mar 2024 16:39:11 -0400 Subject: [PATCH 07/12] use RelationConfig attributes in create_from_source --- dbt/adapters/duckdb/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/duckdb/utils.py b/dbt/adapters/duckdb/utils.py index 2d7deaf7..fa0e9c7d 100644 --- a/dbt/adapters/duckdb/utils.py +++ b/dbt/adapters/duckdb/utils.py @@ -49,10 +49,9 @@ def as_dict(self) -> Dict[str, Any]: @classmethod def create_from_source(cls, source: RelationConfig) -> "SourceConfig": - meta = source.source_meta.copy() - meta.update(source.meta) + meta = source.meta.copy() # Use the config properties as well if they are present - meta.update(source.config._extra) + meta.update(source.config.extra) return SourceConfig( name=source.name, identifier=source.identifier, From b8fdca014c54db58a9ee2027fde15e6567a4f625 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Mon, 18 Mar 2024 20:27:18 -0400 Subject: [PATCH 08/12] formatting + initial mypy fixes --- dbt/adapters/duckdb/relation.py | 17 ++++++++--------- dbt/adapters/duckdb/utils.py | 3 ++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dbt/adapters/duckdb/relation.py b/dbt/adapters/duckdb/relation.py index 492ad6a2..01542316 100644 --- a/dbt/adapters/duckdb/relation.py +++ b/dbt/adapters/duckdb/relation.py @@ -7,12 +7,9 @@ from .connections import DuckDBConnectionManager from .utils import SourceConfig from dbt.adapters.base.relation import BaseRelation -from dbt.adapters.base.relation import Self +from dbt.adapters.contracts.relation import HasQuoting +from dbt.adapters.contracts.relation import RelationConfig -from dbt.adapters.contracts.relation import ( - HasQuoting, - RelationConfig, -) @dataclass(frozen=True, eq=False, repr=False) class DuckDBRelation(BaseRelation): @@ -20,18 +17,20 @@ class DuckDBRelation(BaseRelation): @classmethod def create_from( - cls: Type[Self], + cls: Type["DuckDBRelation"], quoting: HasQuoting, relation_config: RelationConfig, **kwargs: Any, - ) -> Self: - if relation_config.resource_type == 'source': + ) -> "DuckDBRelation": + if relation_config.resource_type == "source": return cls.create_from_source(quoting, relation_config, **kwargs) else: return super().create_from(quoting, relation_config, **kwargs) @classmethod - def create_from_source(cls: Type[Self], quoting: HasQuoting, source: RelationConfig, **kwargs: Any) -> Self: + def create_from_source( + cls: Type["DuckDBRelation"], quoting: HasQuoting, source: RelationConfig, **kwargs: Any + ) -> "DuckDBRelation": """ This method creates a new DuckDBRelation instance from a source definition. It first checks if a 'plugin' is defined in the meta argument for the source or its parent configuration. diff --git a/dbt/adapters/duckdb/utils.py b/dbt/adapters/duckdb/utils.py index fa0e9c7d..19d3486f 100644 --- a/dbt/adapters/duckdb/utils.py +++ b/dbt/adapters/duckdb/utils.py @@ -51,7 +51,8 @@ def as_dict(self) -> Dict[str, Any]: def create_from_source(cls, source: RelationConfig) -> "SourceConfig": meta = source.meta.copy() # Use the config properties as well if they are present - meta.update(source.config.extra) + config_properties = source.config.extra if source.config else {} + meta.update(config_properties) return SourceConfig( name=source.name, identifier=source.identifier, From 9a0ba0296f0b59cd7a4e52df9a859376c8b0e6dc Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 10 Apr 2024 10:18:43 +0200 Subject: [PATCH 09/12] Revert dev-requirements --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 48edba68..1217eb75 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core + dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +# git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +# git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter boto3 mypy-boto3-glue From 6dcc39c83aee677406c889c49cdfdc8b884aba1f Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Wed, 10 Apr 2024 10:19:51 +0200 Subject: [PATCH 10/12] Readd dbt-tests-adapter --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 1217eb75..e3cb823e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,8 @@ # git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core # git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +dbt-tests-adapter>=1.8.0b1 + boto3 mypy-boto3-glue pandas From 5f594c40292c3dd7a7266b9931e8a5cad9a5b736 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Tue, 16 Apr 2024 11:27:01 +0200 Subject: [PATCH 11/12] Readd dbt-core to setup.py for install back-compat --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1f257bc3..d8208fcc 100644 --- a/setup.py +++ b/setup.py @@ -37,9 +37,11 @@ def _dbt_duckdb_version(): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-common>=1.0.0b1,<2.0", - "dbt-adapters>=1.0.0b1,<2.0", + "dbt-common>=1,<2", + "dbt-adapters>=1,<2", "duckdb>=0.7.0", + # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency + "dbt-core>=1.8.0b1", ], extras_require={"glue": ["boto3", "mypy-boto3-glue"], "md": ["duckdb>=0.7.0,<=0.9.2"]}, classifiers=[ From a85cef9f5d986a049fa69ac89105a10647a9abe1 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 19 Apr 2024 15:37:34 +0200 Subject: [PATCH 12/12] Skip BV for TestUnitTestingTypesDuckDB --- tests/functional/adapter/test_unit_testing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/test_unit_testing.py b/tests/functional/adapter/test_unit_testing.py index bcc3cda6..a0751f1e 100644 --- a/tests/functional/adapter/test_unit_testing.py +++ b/tests/functional/adapter/test_unit_testing.py @@ -5,6 +5,7 @@ from dbt.tests.adapter.unit_testing.test_invalid_input import BaseUnitTestInvalidInput +@pytest.mark.skip_profile("buenavista") class TestUnitTestingTypesDuckDB(BaseUnitTestingTypes): @pytest.fixture def data_types(self):