From 860507d3b9e22f3a4e625dd9579796c3b38e6e25 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:45:15 +0000 Subject: [PATCH 1/7] Remove hard dbt-core dependency; support dbt v2 (dbt/dbt-oss) without extras Co-Authored-By: Itamar Hartstein --- elementary/clients/dbt/dbt_installation.py | 25 +++-- elementary/clients/dbt/factory.py | 19 +++- elementary/exceptions/exceptions.py | 16 +++ pyproject.toml | 25 ++++- tests/unit/clients/dbt_runner/test_factory.py | 100 +++++++++++++----- 5 files changed, 146 insertions(+), 39 deletions(-) diff --git a/elementary/clients/dbt/dbt_installation.py b/elementary/clients/dbt/dbt_installation.py index 084f7bc1b..18421dc42 100644 --- a/elementary/clients/dbt/dbt_installation.py +++ b/elementary/clients/dbt/dbt_installation.py @@ -21,13 +21,18 @@ def get_dbt_core_version() -> Optional[version.Version]: return _get_package_version("dbt-core") -def get_dbt_package_version() -> Optional[version.Version]: - """Version of the installed `dbt` package, or None if not installed. +def get_dbt2_package_version() -> Optional[version.Version]: + """Version of the installed dbt v2 package, or None if not installed. - From 2.0, the `dbt` package on PyPI ships the dbt (Fusion) binary as a - platform wheel with no importable Python module. + dbt v2 (the Fusion engine) is distributed on PyPI as the `dbt` package (full + feature set) and the `dbt-oss` package (Apache 2 subset); both ship the dbt + binary. dbt-core stays on 1.x. """ - return _get_package_version("dbt") + for package_name in ("dbt", "dbt-oss"): + package_version = _get_package_version(package_name) + if package_version is not None and package_version.major >= 2: + return package_version + return None def is_dbt2_binary_available() -> bool: @@ -35,8 +40,7 @@ def is_dbt2_binary_available() -> bool: if env_path and os.path.exists(os.path.expanduser(env_path)): return True - dbt_package_version = get_dbt_package_version() - if dbt_package_version is not None and dbt_package_version.major >= 2: + if get_dbt2_package_version() is not None: return True return os.path.exists(os.path.expanduser(DEFAULT_DBT_FUSION_PATH)) @@ -49,10 +53,9 @@ def get_dbt2_binary_path() -> str: # When only dbt-core 1.x is installed, the `dbt` executable on PATH is its # entrypoint, so it can't be trusted to be the dbt 2.0 binary. dbt_core_version = get_dbt_core_version() - dbt_package_version = get_dbt_package_version() - dbt2_installed_via_pip = ( - dbt_package_version is not None and dbt_package_version.major >= 2 - ) or (dbt_core_version is not None and dbt_core_version.major >= 2) + dbt2_installed_via_pip = get_dbt2_package_version() is not None or ( + dbt_core_version is not None and dbt_core_version.major >= 2 + ) if dbt2_installed_via_pip or dbt_core_version is None: which_path = shutil.which("dbt") if which_path: diff --git a/elementary/clients/dbt/factory.py b/elementary/clients/dbt/factory.py index 8751a9ec3..09535e52a 100644 --- a/elementary/clients/dbt/factory.py +++ b/elementary/clients/dbt/factory.py @@ -1,4 +1,5 @@ import os +import shutil from enum import Enum from typing import Any, Dict, Optional, Type @@ -11,6 +12,10 @@ is_dbt2_binary_available, ) from elementary.clients.dbt.subprocess_dbt_runner import SubprocessDbtRunner +from elementary.exceptions.exceptions import NoDbtInstallationError +from elementary.utils.log import get_logger + +logger = get_logger(__name__) class RunnerMethod(Enum): @@ -60,6 +65,13 @@ def get_dbt_runner_method() -> RunnerMethod: if dbt_core_version is not None: if dbt_core_version.major >= 2: return RunnerMethod.DBT2 + if is_dbt2_binary_available(): + logger.info( + f"Both dbt-core {dbt_core_version} and a dbt v2 installation were " + "detected - using dbt-core. To use dbt v2 instead, uninstall " + "dbt-core (and the elementary-data adapter extra) or set " + "DBT_RUNNER_METHOD=dbt2." + ) if dbt_core_version >= version.Version("1.5.0"): return RunnerMethod.API return RunnerMethod.SUBPROCESS @@ -67,7 +79,12 @@ def get_dbt_runner_method() -> RunnerMethod: if is_dbt2_binary_available(): return RunnerMethod.DBT2 - return RunnerMethod.SUBPROCESS + # dbt may be installed without pip package metadata (e.g. a system-wide or + # pipx-managed dbt 1.x) - fall back to running it as a subprocess. + if shutil.which("dbt"): + return RunnerMethod.SUBPROCESS + + raise NoDbtInstallationError() def get_dbt_runner_class(runner_method: RunnerMethod) -> Type[CommandLineDbtRunner]: diff --git a/elementary/exceptions/exceptions.py b/elementary/exceptions/exceptions.py index d6ad58bf6..a53659f88 100644 --- a/elementary/exceptions/exceptions.py +++ b/elementary/exceptions/exceptions.py @@ -31,6 +31,22 @@ def anonymous_tracking_context(self): } +class NoDbtInstallationError(ConfigError): + """Exception raised when no dbt installation is found in the environment""" + + def __init__(self) -> None: + super().__init__( + "No dbt installation was found. Please either install dbt v2 " + "(`pip install dbt` or `pip install dbt-oss`), or install " + "elementary-data with your warehouse's dbt 1.x adapter " + "(e.g. `pip install 'elementary-data[dbt1-snowflake]'`)." + ) + + @property + def anonymous_tracking_context(self): + return {"exception_message": str(self)} + + class DbtCommandError(Error): """Exception raised while executing a dbt command""" diff --git a/pyproject.toml b/pyproject.toml index 587113452..e8c743fad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,9 +25,6 @@ packages = [{include = "elementary"}] [tool.poetry.dependencies] python = ">=3.10,<3.14" click = ">=7.0,<9.0" -# From 2.0, dbt-core is the Fusion engine (requires Python >=3.11); adapters are -# built into the engine, so the adapter extras below are only relevant for 1.x. -dbt-core = ">=1.8,<3.0.0" requests = ">=2.28.1,<3.0.0" urllib3 = ">=2.7.0" # transitive dependency via requests, which caps the major itself idna = ">=3.15" # transitive dependency via requests, floored to address CVE-2026-45409 (GHSA-65pc-fj4g-8rjx) ReDoS in idna.encode() @@ -53,6 +50,10 @@ tabulate = ">= 0.9.0" tenacity = ">=8.0,<10.0" pytz = ">= 2025.1" +# There is intentionally no direct dbt dependency: dbt v2 ships as the `dbt` / +# `dbt-oss` packages (dbt-core stays on 1.x) and requires no extra alongside +# elementary-data, while dbt-core 1.x (>=1.8) comes through the adapter extras +# below - every dbt- package depends on it. dbt-snowflake = {version = ">=1.8,<2.0.0", optional = true} dbt-bigquery = {version = ">=1.8,<2.0.0", optional = true} dbt-redshift = {version = ">=1.8,<2.0.0", optional = true} @@ -69,6 +70,23 @@ dbt-dremio = {version = ">=1.8,<2.0.0", optional = true} dbt-fabric = {version = ">=1.8,<2.0.0", optional = true} dbt-sqlserver = {version = ">=1.8,<2.0.0", optional = true} [tool.poetry.extras] +# The extras install the dbt-core 1.x stack (adapter + dbt-core) for a warehouse. +# They are not needed with dbt v2 (`pip install dbt` / `dbt-oss`), which has the +# adapters built in. The dbt1-* names are the documented ones; the plain +# warehouse names are kept as aliases for backward compatibility. +dbt1-snowflake = ["dbt-snowflake"] +dbt1-bigquery = ["dbt-bigquery"] +dbt1-redshift = ["dbt-redshift"] +dbt1-postgres = ["dbt-postgres"] +dbt1-databricks = ["dbt-databricks"] +dbt1-spark = ["dbt-spark"] +dbt1-athena = ["dbt-athena-community", "pyathena"] +dbt1-clickhouse = ["dbt-clickhouse"] +dbt1-trino = ["dbt-trino"] +dbt1-duckdb = ["dbt-duckdb"] +dbt1-dremio = ["dbt-dremio"] +dbt1-fabric = ["dbt-fabric"] +dbt1-sqlserver = ["dbt-sqlserver"] snowflake = ["dbt-snowflake"] bigquery = ["dbt-bigquery"] redshift = ["dbt-redshift"] @@ -86,6 +104,7 @@ sqlserver = ["dbt-sqlserver"] # outdated transitive dependencies (dbt-core==1.8.5, azure-cli pre-release) that block # security patches for deepdiff, protobuf, and pyopenssl. Users who need these adapters # can still install them directly (e.g. pip install dbt-fabricspark dbt-vertica). +dbt1-all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] [build-system] diff --git a/tests/unit/clients/dbt_runner/test_factory.py b/tests/unit/clients/dbt_runner/test_factory.py index d78bc386c..79d6da618 100644 --- a/tests/unit/clients/dbt_runner/test_factory.py +++ b/tests/unit/clients/dbt_runner/test_factory.py @@ -8,6 +8,7 @@ from elementary.clients.dbt.dbt2_runner import Dbt2Runner from elementary.clients.dbt.dbt_installation import ( get_dbt2_binary_path, + get_dbt2_package_version, is_dbt2_binary_available, ) from elementary.clients.dbt.factory import ( @@ -16,12 +17,14 @@ get_dbt_runner_method, ) from elementary.clients.dbt.subprocess_dbt_runner import SubprocessDbtRunner +from elementary.exceptions.exceptions import NoDbtInstallationError def _mock_installation( monkeypatch, dbt_core_version: Optional[str] = None, dbt2_binary_available: bool = False, + dbt_on_path: bool = False, ): monkeypatch.setattr( factory, @@ -31,27 +34,52 @@ def _mock_installation( monkeypatch.setattr( factory, "is_dbt2_binary_available", lambda: dbt2_binary_available ) + monkeypatch.setattr( + factory.shutil, "which", lambda name: "/usr/bin/dbt" if dbt_on_path else None + ) @pytest.mark.parametrize( - "dbt_core_version,dbt2_binary_available,expected_method", + "dbt_core_version,dbt2_binary_available,dbt_on_path,expected_method", [ - ("1.8.0", False, RunnerMethod.API), - ("1.10.5", True, RunnerMethod.API), - ("1.4.0", False, RunnerMethod.SUBPROCESS), - ("2.0.0b2", False, RunnerMethod.DBT2), - (None, True, RunnerMethod.DBT2), - (None, False, RunnerMethod.SUBPROCESS), + ("1.8.0", False, True, RunnerMethod.API), + ("1.10.5", True, True, RunnerMethod.API), + ("1.4.0", False, True, RunnerMethod.SUBPROCESS), + ("2.0.0b2", False, True, RunnerMethod.DBT2), + (None, True, True, RunnerMethod.DBT2), + # dbt installed without pip metadata (e.g. system-wide dbt 1.x) + (None, False, True, RunnerMethod.SUBPROCESS), ], ) def test_get_dbt_runner_method_auto_detection( - monkeypatch, dbt_core_version, dbt2_binary_available, expected_method + monkeypatch, dbt_core_version, dbt2_binary_available, dbt_on_path, expected_method ): monkeypatch.delenv("DBT_RUNNER_METHOD", raising=False) - _mock_installation(monkeypatch, dbt_core_version, dbt2_binary_available) + _mock_installation( + monkeypatch, dbt_core_version, dbt2_binary_available, dbt_on_path + ) assert get_dbt_runner_method() == expected_method +def test_get_dbt_runner_method_raises_when_no_dbt_installation(monkeypatch): + monkeypatch.delenv("DBT_RUNNER_METHOD", raising=False) + _mock_installation(monkeypatch, dbt_core_version=None, dbt2_binary_available=False) + with pytest.raises(NoDbtInstallationError): + get_dbt_runner_method() + + +def test_get_dbt_runner_method_hints_when_dbt_core_and_dbt2_coexist( + monkeypatch, caplog +): + monkeypatch.delenv("DBT_RUNNER_METHOD", raising=False) + _mock_installation( + monkeypatch, dbt_core_version="1.10.0", dbt2_binary_available=True + ) + with caplog.at_level("INFO"): + assert get_dbt_runner_method() == RunnerMethod.API + assert any("using dbt-core" in record.message for record in caplog.records) + + @pytest.mark.parametrize( "env_value,expected_method", [ @@ -73,41 +101,41 @@ def test_get_dbt_runner_class(): @mock.patch("elementary.clients.dbt.dbt_installation.shutil.which") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") def test_dbt2_runner_uses_path_binary_when_no_dbt_core( - mock_get_dbt_core_version, mock_get_dbt_package_version, mock_which, monkeypatch + mock_get_dbt_core_version, mock_get_dbt2_package_version, mock_which, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) mock_get_dbt_core_version.return_value = None - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None mock_which.return_value = "/some/venv/bin/dbt" assert get_dbt2_binary_path() == "/some/venv/bin/dbt" @mock.patch("elementary.clients.dbt.dbt_installation.shutil.which") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") def test_dbt2_runner_uses_path_binary_when_dbt2_pip_installed_alongside_dbt_core_1x( - mock_get_dbt_core_version, mock_get_dbt_package_version, mock_which, monkeypatch + mock_get_dbt_core_version, mock_get_dbt2_package_version, mock_which, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) mock_get_dbt_core_version.return_value = version.Version("1.10.0") - mock_get_dbt_package_version.return_value = version.Version("2.0.0rc212") + mock_get_dbt2_package_version.return_value = version.Version("2.0.6") mock_which.return_value = "/some/venv/bin/dbt" assert get_dbt2_binary_path() == "/some/venv/bin/dbt" -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") def test_dbt2_runner_ignores_path_binary_when_dbt_core_1x( - mock_get_dbt_core_version, mock_get_dbt_package_version, monkeypatch + mock_get_dbt_core_version, mock_get_dbt2_package_version, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) mock_get_dbt_core_version.return_value = version.Version("1.10.0") - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None assert get_dbt2_binary_path().endswith("/.local/bin/dbt") @@ -119,24 +147,48 @@ def test_dbt2_runner_honors_dbt_fusion_path_env_var(monkeypatch): @mock.patch("elementary.clients.dbt.dbt_installation.os.path.exists") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") def test_dbt2_binary_available_when_dbt_fusion_path_env_var_points_to_binary( - mock_get_dbt_package_version, mock_exists, monkeypatch + mock_get_dbt2_package_version, mock_exists, monkeypatch ): monkeypatch.setenv("DBT_FUSION_PATH", "/custom/path/dbt") - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None mock_exists.side_effect = lambda path: path == "/custom/path/dbt" assert is_dbt2_binary_available() @mock.patch("elementary.clients.dbt.dbt_installation.os.path.exists") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") def test_dbt2_binary_not_available_when_nothing_installed( - mock_get_dbt_package_version, mock_exists, monkeypatch + mock_get_dbt2_package_version, mock_exists, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None mock_exists.return_value = False assert not is_dbt2_binary_available() + + +@pytest.mark.parametrize( + "installed_packages,expected_version", + [ + ({"dbt": "2.0.6"}, "2.0.6"), + ({"dbt-oss": "2.0.5"}, "2.0.5"), + # A 1.x `dbt` package is not a dbt v2 installation + ({"dbt": "1.8.0"}, None), + ({}, None), + ], +) +@mock.patch("elementary.clients.dbt.dbt_installation._get_package_version") +def test_get_dbt2_package_version( + mock_get_package_version, installed_packages, expected_version +): + mock_get_package_version.side_effect = lambda name: ( + version.Version(installed_packages[name]) + if name in installed_packages + else None + ) + + expected = version.Version(expected_version) if expected_version else None + assert get_dbt2_package_version() == expected From fbf003ead8c52c71a3c2dcd4899f5d9020ebd7f7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:49:52 +0000 Subject: [PATCH 2/7] Fix CI: mypy without dbt-core installed, pull minio images from quay.io Co-Authored-By: Itamar Hartstein --- mypy.ini | 4 ++++ tests/e2e_dbt_project/docker-compose.yml | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mypy.ini b/mypy.ini index 073e24ede..a2d7aad32 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,6 +3,10 @@ packages = elementary exclude = ^elementary/monitor/dbt_project/ check_untyped_defs = true +# dbt-core is an optional dependency (only installed via the dbt1-* extras) +[mypy-dbt,dbt.*] +ignore_missing_imports = True + [mypy-alive_progress] ignore_missing_imports = True diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml index 7c7c1dc93..bcb40cd7e 100644 --- a/tests/e2e_dbt_project/docker-compose.yml +++ b/tests/e2e_dbt_project/docker-compose.yml @@ -120,7 +120,7 @@ services: trino-minio: hostname: minio - image: "minio/minio:RELEASE.2022-05-26T05-48-41Z" + image: "quay.io/minio/minio:RELEASE.2022-05-26T05-48-41Z" ports: - "9002:9000" - "9003:9001" @@ -135,7 +135,7 @@ services: retries: 10 trino-mc-job: - image: "minio/mc:RELEASE.2022-05-09T04-08-26Z" + image: "quay.io/minio/mc:RELEASE.2022-05-09T04-08-26Z" entrypoint: | /bin/bash -c " until /usr/bin/mc config --quiet host add myminio http://minio:9000 $${TRINO_MINIO_ACCESS_KEY:-minio} $${TRINO_MINIO_SECRET_KEY:-minio123}; do @@ -163,7 +163,7 @@ services: retries: 10 dremio-minio: - image: minio/minio:RELEASE.2024-01-18T22-51-28Z + image: quay.io/minio/minio:RELEASE.2024-01-18T22-51-28Z container_name: dremio-storage environment: - MINIO_ROOT_USER=${DREMIO_MINIO_ROOT_USER:-admin} @@ -186,7 +186,7 @@ services: retries: 10 dremio-minio-setup: - image: minio/mc:RELEASE.2024-01-18T07-03-39Z + image: quay.io/minio/mc:RELEASE.2024-01-18T07-03-39Z container_name: dremio-minio-setup depends_on: dremio-minio: From 23d7fbba3bb1a00e267922220776360d9c0a3f67 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:02:59 +0000 Subject: [PATCH 3/7] Prefer dbt v2 over dbt-core, rename runner methods to DBT1_*, fix dremio/clickhouse CI Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 4 ++ elementary/clients/dbt/factory.py | 41 +++++++++++-------- .../external_seeders/dremio.py | 4 +- tests/unit/clients/dbt_runner/test_factory.py | 24 +++++++---- 4 files changed, 44 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 4c149d059..fba9cf2a1 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -303,6 +303,10 @@ jobs: # mssql-python, and its service-principal connection string # ("Authority Id=...") is rejected by the driver. fabric) DBT_ADAPTER_VERSION_SPEC="<1.11" ;; + # dbt-clickhouse 1.10.3 fails to create the target database + # ("Failed to create ... database due to ClickHouse exception"); + # 1.10.2 works with the same server version. + clickhouse) DBT_ADAPTER_VERSION_SPEC="<1.10.3" ;; esac if [ -n "$DBT_VERSION" ]; then diff --git a/elementary/clients/dbt/factory.py b/elementary/clients/dbt/factory.py index 09535e52a..2d78b845b 100644 --- a/elementary/clients/dbt/factory.py +++ b/elementary/clients/dbt/factory.py @@ -19,11 +19,15 @@ class RunnerMethod(Enum): - # SUBPROCESS and API drive dbt-core 1.x (API requires dbt-core >= 1.5.0). + # DBT1_SUBPROCESS and DBT1_API drive dbt-core 1.x (DBT1_API requires + # dbt-core >= 1.5.0). + DBT1_SUBPROCESS = "subprocess" + DBT1_API = "api" + DBT2 = "dbt2" + # Legacy aliases for the dbt 1.x runner methods. SUBPROCESS = "subprocess" API = "api" - DBT2 = "dbt2" - # Legacy alias for DBT2 (dbt 2.0 is the Fusion engine). + # Legacy alias for DBT2 (dbt v2 is the Fusion engine). FUSION = "fusion" @@ -62,39 +66,40 @@ def get_dbt_runner_method() -> RunnerMethod: return RunnerMethod(runner_method) dbt_core_version = get_dbt_core_version() - if dbt_core_version is not None: - if dbt_core_version.major >= 2: - return RunnerMethod.DBT2 - if is_dbt2_binary_available(): + if is_dbt2_binary_available(): + if dbt_core_version is not None and dbt_core_version.major < 2: logger.info( f"Both dbt-core {dbt_core_version} and a dbt v2 installation were " - "detected - using dbt-core. To use dbt v2 instead, uninstall " - "dbt-core (and the elementary-data adapter extra) or set " - "DBT_RUNNER_METHOD=dbt2." + "detected - using dbt v2. To use dbt-core instead, set " + "DBT_RUNNER_METHOD=api (or uninstall dbt v2)." ) - if dbt_core_version >= version.Version("1.5.0"): - return RunnerMethod.API - return RunnerMethod.SUBPROCESS - - if is_dbt2_binary_available(): return RunnerMethod.DBT2 + if dbt_core_version is not None: + # dbt-core published some 2.x pre-releases before dbt v2 moved to the + # dbt/dbt-oss packages - drive those with the dbt v2 runner too. + if dbt_core_version.major >= 2: + return RunnerMethod.DBT2 + if dbt_core_version >= version.Version("1.5.0"): + return RunnerMethod.DBT1_API + return RunnerMethod.DBT1_SUBPROCESS + # dbt may be installed without pip package metadata (e.g. a system-wide or # pipx-managed dbt 1.x) - fall back to running it as a subprocess. if shutil.which("dbt"): - return RunnerMethod.SUBPROCESS + return RunnerMethod.DBT1_SUBPROCESS raise NoDbtInstallationError() def get_dbt_runner_class(runner_method: RunnerMethod) -> Type[CommandLineDbtRunner]: - if runner_method == RunnerMethod.API: + if runner_method == RunnerMethod.DBT1_API: # Import it internally since it will fail if dbt-core is not installed # or its version is below 1.5.0 from elementary.clients.dbt.api_dbt_runner import APIDbtRunner return APIDbtRunner - elif runner_method == RunnerMethod.SUBPROCESS: + elif runner_method == RunnerMethod.DBT1_SUBPROCESS: return SubprocessDbtRunner elif runner_method in (RunnerMethod.DBT2, RunnerMethod.FUSION): return Dbt2Runner diff --git a/tests/e2e_dbt_project/external_seeders/dremio.py b/tests/e2e_dbt_project/external_seeders/dremio.py index 949e8bdaa..3e8479f2f 100644 --- a/tests/e2e_dbt_project/external_seeders/dremio.py +++ b/tests/e2e_dbt_project/external_seeders/dremio.py @@ -175,7 +175,7 @@ def _create_nessie_namespace(self, token: str, namespace: str) -> None: def _upload_csvs_to_minio(self) -> None: """Upload seed CSVs to the Dremio MinIO bucket. - Mounts the local ``data_dir`` into a temporary ``minio/mc`` container + Mounts the local ``data_dir`` into a temporary ``mc`` container and copies files directly into the MinIO bucket. """ import shlex @@ -204,7 +204,7 @@ def _upload_csvs_to_minio(self) -> None: f"{self.data_dir}:/seed-data:ro", "--entrypoint", "/bin/sh", - "minio/mc", + "quay.io/minio/mc:RELEASE.2024-01-18T07-03-39Z", "-c", mc_cmds, ] diff --git a/tests/unit/clients/dbt_runner/test_factory.py b/tests/unit/clients/dbt_runner/test_factory.py index 79d6da618..d4ba17b3e 100644 --- a/tests/unit/clients/dbt_runner/test_factory.py +++ b/tests/unit/clients/dbt_runner/test_factory.py @@ -42,13 +42,14 @@ def _mock_installation( @pytest.mark.parametrize( "dbt_core_version,dbt2_binary_available,dbt_on_path,expected_method", [ - ("1.8.0", False, True, RunnerMethod.API), - ("1.10.5", True, True, RunnerMethod.API), - ("1.4.0", False, True, RunnerMethod.SUBPROCESS), + ("1.8.0", False, True, RunnerMethod.DBT1_API), + # dbt v2 takes precedence when installed alongside dbt-core 1.x + ("1.10.5", True, True, RunnerMethod.DBT2), + ("1.4.0", False, True, RunnerMethod.DBT1_SUBPROCESS), ("2.0.0b2", False, True, RunnerMethod.DBT2), (None, True, True, RunnerMethod.DBT2), # dbt installed without pip metadata (e.g. system-wide dbt 1.x) - (None, False, True, RunnerMethod.SUBPROCESS), + (None, False, True, RunnerMethod.DBT1_SUBPROCESS), ], ) def test_get_dbt_runner_method_auto_detection( @@ -76,15 +77,15 @@ def test_get_dbt_runner_method_hints_when_dbt_core_and_dbt2_coexist( monkeypatch, dbt_core_version="1.10.0", dbt2_binary_available=True ) with caplog.at_level("INFO"): - assert get_dbt_runner_method() == RunnerMethod.API - assert any("using dbt-core" in record.message for record in caplog.records) + assert get_dbt_runner_method() == RunnerMethod.DBT2 + assert any("using dbt v2" in record.message for record in caplog.records) @pytest.mark.parametrize( "env_value,expected_method", [ - ("subprocess", RunnerMethod.SUBPROCESS), - ("api", RunnerMethod.API), + ("subprocess", RunnerMethod.DBT1_SUBPROCESS), + ("api", RunnerMethod.DBT1_API), ("dbt2", RunnerMethod.DBT2), ("fusion", RunnerMethod.FUSION), ], @@ -95,11 +96,16 @@ def test_get_dbt_runner_method_env_override(monkeypatch, env_value, expected_met def test_get_dbt_runner_class(): - assert get_dbt_runner_class(RunnerMethod.SUBPROCESS) is SubprocessDbtRunner + assert get_dbt_runner_class(RunnerMethod.DBT1_SUBPROCESS) is SubprocessDbtRunner assert get_dbt_runner_class(RunnerMethod.DBT2) is Dbt2Runner assert get_dbt_runner_class(RunnerMethod.FUSION) is Dbt2Runner +def test_runner_method_legacy_aliases(): + assert RunnerMethod.API is RunnerMethod.DBT1_API + assert RunnerMethod.SUBPROCESS is RunnerMethod.DBT1_SUBPROCESS + + @mock.patch("elementary.clients.dbt.dbt_installation.shutil.which") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") From e0d143986f796129112e5e41540a19f36699e87c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:25:24 +0000 Subject: [PATCH 4/7] Bump CI clickhouse-server to 24.8 and drop the dbt-clickhouse pin dbt-clickhouse 1.10.3 sends lightweight_deletes_sync as a default connection setting, which only exists on ClickHouse >= 24.5 (24.3 rejects it with UNKNOWN_SETTING on the first command). Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 4 ---- tests/e2e_dbt_project/docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index fba9cf2a1..4c149d059 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -303,10 +303,6 @@ jobs: # mssql-python, and its service-principal connection string # ("Authority Id=...") is rejected by the driver. fabric) DBT_ADAPTER_VERSION_SPEC="<1.11" ;; - # dbt-clickhouse 1.10.3 fails to create the target database - # ("Failed to create ... database due to ClickHouse exception"); - # 1.10.2 works with the same server version. - clickhouse) DBT_ADAPTER_VERSION_SPEC="<1.10.3" ;; esac if [ -n "$DBT_VERSION" ]; then diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml index bcb40cd7e..271c5fd92 100644 --- a/tests/e2e_dbt_project/docker-compose.yml +++ b/tests/e2e_dbt_project/docker-compose.yml @@ -18,7 +18,7 @@ services: retries: 10 clickhouse: - image: clickhouse/clickhouse-server:24.3 + image: clickhouse/clickhouse-server:24.8 container_name: clickhouse ports: - "8123:8123" From bb0407427209902f46fa3d2197eb649803d2d68c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:39:25 +0000 Subject: [PATCH 5/7] Constrain dbt-core <2.0 explicitly in the dbt1 extras The adapters only lower-bound dbt-core, so the extras alone would not prevent resolving a future dbt-core 2.x release. Co-Authored-By: Itamar Hartstein --- pyproject.toml | 64 ++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e8c743fad..3ab2ff201 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,10 +50,12 @@ tabulate = ">= 0.9.0" tenacity = ">=8.0,<10.0" pytz = ">= 2025.1" -# There is intentionally no direct dbt dependency: dbt v2 ships as the `dbt` / +# There is intentionally no mandatory dbt dependency: dbt v2 ships as the `dbt` / # `dbt-oss` packages (dbt-core stays on 1.x) and requires no extra alongside -# elementary-data, while dbt-core 1.x (>=1.8) comes through the adapter extras -# below - every dbt- package depends on it. +# elementary-data, while dbt-core 1.x comes through the adapter extras below. +# dbt-core is explicitly constrained in the extras so they can never resolve a +# dbt-core 2.x release (the adapters themselves only lower-bound dbt-core). +dbt-core = {version = ">=1.8,<2.0.0", optional = true} dbt-snowflake = {version = ">=1.8,<2.0.0", optional = true} dbt-bigquery = {version = ">=1.8,<2.0.0", optional = true} dbt-redshift = {version = ">=1.8,<2.0.0", optional = true} @@ -74,38 +76,38 @@ dbt-sqlserver = {version = ">=1.8,<2.0.0", optional = true} # They are not needed with dbt v2 (`pip install dbt` / `dbt-oss`), which has the # adapters built in. The dbt1-* names are the documented ones; the plain # warehouse names are kept as aliases for backward compatibility. -dbt1-snowflake = ["dbt-snowflake"] -dbt1-bigquery = ["dbt-bigquery"] -dbt1-redshift = ["dbt-redshift"] -dbt1-postgres = ["dbt-postgres"] -dbt1-databricks = ["dbt-databricks"] -dbt1-spark = ["dbt-spark"] -dbt1-athena = ["dbt-athena-community", "pyathena"] -dbt1-clickhouse = ["dbt-clickhouse"] -dbt1-trino = ["dbt-trino"] -dbt1-duckdb = ["dbt-duckdb"] -dbt1-dremio = ["dbt-dremio"] -dbt1-fabric = ["dbt-fabric"] -dbt1-sqlserver = ["dbt-sqlserver"] -snowflake = ["dbt-snowflake"] -bigquery = ["dbt-bigquery"] -redshift = ["dbt-redshift"] -postgres = ["dbt-postgres"] -databricks = ["dbt-databricks"] -spark = ["dbt-spark"] -athena = ["dbt-athena-community", "pyathena"] -clickhouse = ["dbt-clickhouse"] -trino = ["dbt-trino"] -duckdb = ["dbt-duckdb"] -dremio = ["dbt-dremio"] -fabric = ["dbt-fabric"] -sqlserver = ["dbt-sqlserver"] +dbt1-snowflake = ["dbt-core", "dbt-snowflake"] +dbt1-bigquery = ["dbt-core", "dbt-bigquery"] +dbt1-redshift = ["dbt-core", "dbt-redshift"] +dbt1-postgres = ["dbt-core", "dbt-postgres"] +dbt1-databricks = ["dbt-core", "dbt-databricks"] +dbt1-spark = ["dbt-core", "dbt-spark"] +dbt1-athena = ["dbt-core", "dbt-athena-community", "pyathena"] +dbt1-clickhouse = ["dbt-core", "dbt-clickhouse"] +dbt1-trino = ["dbt-core", "dbt-trino"] +dbt1-duckdb = ["dbt-core", "dbt-duckdb"] +dbt1-dremio = ["dbt-core", "dbt-dremio"] +dbt1-fabric = ["dbt-core", "dbt-fabric"] +dbt1-sqlserver = ["dbt-core", "dbt-sqlserver"] +snowflake = ["dbt-core", "dbt-snowflake"] +bigquery = ["dbt-core", "dbt-bigquery"] +redshift = ["dbt-core", "dbt-redshift"] +postgres = ["dbt-core", "dbt-postgres"] +databricks = ["dbt-core", "dbt-databricks"] +spark = ["dbt-core", "dbt-spark"] +athena = ["dbt-core", "dbt-athena-community", "pyathena"] +clickhouse = ["dbt-core", "dbt-clickhouse"] +trino = ["dbt-core", "dbt-trino"] +duckdb = ["dbt-core", "dbt-duckdb"] +dremio = ["dbt-core", "dbt-dremio"] +fabric = ["dbt-core", "dbt-fabric"] +sqlserver = ["dbt-core", "dbt-sqlserver"] # dbt-fabricspark and dbt-vertica are removed from the lockfile resolution because they pin # outdated transitive dependencies (dbt-core==1.8.5, azure-cli pre-release) that block # security patches for deepdiff, protobuf, and pyopenssl. Users who need these adapters # can still install them directly (e.g. pip install dbt-fabricspark dbt-vertica). -dbt1-all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] -all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] +dbt1-all = ["dbt-core", "dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] +all = ["dbt-core", "dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] [build-system] requires = ["poetry-core>=1.0.0"] From 813eda892b23f8933298d94753ed8c49e60a03e9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:43:29 +0000 Subject: [PATCH 6/7] Retrigger CI (spark job hit AWS token expiry after slow thrift startup) Co-Authored-By: Itamar Hartstein From e962dd4c8b66f7a4619308da5a8a91165af74a8b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:45:38 +0000 Subject: [PATCH 7/7] Refresh AWS credentials after slow Spark startup to avoid token expiry Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 4c149d059..d38b0b1cd 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -238,6 +238,15 @@ jobs: run: | docker compose up -d --build --wait spark-thrift + # The Spark image build can take most of the default 1-hour AWS session, + # expiring the token before the report upload steps at the end of the job. + - name: Refresh AWS credentials + if: inputs.warehouse-type == 'spark' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} + aws-region: eu-west-1 + - name: Start SQL Server if: inputs.warehouse-type == 'sqlserver' working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}