diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 95f086db7..20e03c2d8 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -238,6 +238,15 @@ jobs: run: | docker compose up -d --build --wait spark-thrift + # The Spark image build can take most of the default 1-hour AWS session, + # expiring the token before the report upload steps at the end of the job. + - name: Refresh AWS credentials + if: inputs.warehouse-type == 'spark' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }} + aws-region: eu-west-1 + - name: Start SQL Server if: inputs.warehouse-type == 'sqlserver' working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} diff --git a/elementary/clients/dbt/dbt_installation.py b/elementary/clients/dbt/dbt_installation.py index 084f7bc1b..18421dc42 100644 --- a/elementary/clients/dbt/dbt_installation.py +++ b/elementary/clients/dbt/dbt_installation.py @@ -21,13 +21,18 @@ def get_dbt_core_version() -> Optional[version.Version]: return _get_package_version("dbt-core") -def get_dbt_package_version() -> Optional[version.Version]: - """Version of the installed `dbt` package, or None if not installed. +def get_dbt2_package_version() -> Optional[version.Version]: + """Version of the installed dbt v2 package, or None if not installed. - From 2.0, the `dbt` package on PyPI ships the dbt (Fusion) binary as a - platform wheel with no importable Python module. + dbt v2 (the Fusion engine) is distributed on PyPI as the `dbt` package (full + feature set) and the `dbt-oss` package (Apache 2 subset); both ship the dbt + binary. dbt-core stays on 1.x. """ - return _get_package_version("dbt") + for package_name in ("dbt", "dbt-oss"): + package_version = _get_package_version(package_name) + if package_version is not None and package_version.major >= 2: + return package_version + return None def is_dbt2_binary_available() -> bool: @@ -35,8 +40,7 @@ def is_dbt2_binary_available() -> bool: if env_path and os.path.exists(os.path.expanduser(env_path)): return True - dbt_package_version = get_dbt_package_version() - if dbt_package_version is not None and dbt_package_version.major >= 2: + if get_dbt2_package_version() is not None: return True return os.path.exists(os.path.expanduser(DEFAULT_DBT_FUSION_PATH)) @@ -49,10 +53,9 @@ def get_dbt2_binary_path() -> str: # When only dbt-core 1.x is installed, the `dbt` executable on PATH is its # entrypoint, so it can't be trusted to be the dbt 2.0 binary. dbt_core_version = get_dbt_core_version() - dbt_package_version = get_dbt_package_version() - dbt2_installed_via_pip = ( - dbt_package_version is not None and dbt_package_version.major >= 2 - ) or (dbt_core_version is not None and dbt_core_version.major >= 2) + dbt2_installed_via_pip = get_dbt2_package_version() is not None or ( + dbt_core_version is not None and dbt_core_version.major >= 2 + ) if dbt2_installed_via_pip or dbt_core_version is None: which_path = shutil.which("dbt") if which_path: diff --git a/elementary/clients/dbt/factory.py b/elementary/clients/dbt/factory.py index 4b1159804..aaf57b924 100644 --- a/elementary/clients/dbt/factory.py +++ b/elementary/clients/dbt/factory.py @@ -1,4 +1,5 @@ import os +import shutil from enum import Enum from typing import Any, Dict, Optional, Type @@ -12,14 +13,22 @@ ) from elementary.clients.dbt.subprocess_dbt_runner import SubprocessDbtRunner from elementary.config.config import Config +from elementary.exceptions.exceptions import NoDbtInstallationError +from elementary.utils.log import get_logger + +logger = get_logger(__name__) class RunnerMethod(Enum): - # SUBPROCESS and API drive dbt-core 1.x (API requires dbt-core >= 1.5.0). + # DBT1_SUBPROCESS and DBT1_API drive dbt-core 1.x (DBT1_API requires + # dbt-core >= 1.5.0). + DBT1_SUBPROCESS = "subprocess" + DBT1_API = "api" + DBT2 = "dbt2" + # Legacy aliases for the dbt 1.x runner methods. SUBPROCESS = "subprocess" API = "api" - DBT2 = "dbt2" - # Legacy alias for DBT2 (dbt 2.0 is the Fusion engine). + # Legacy alias for DBT2 (dbt v2 is the Fusion engine). FUSION = "fusion" @@ -74,27 +83,40 @@ def get_dbt_runner_method() -> RunnerMethod: return RunnerMethod(runner_method) dbt_core_version = get_dbt_core_version() + if is_dbt2_binary_available(): + if dbt_core_version is not None and dbt_core_version.major < 2: + logger.info( + f"Both dbt-core {dbt_core_version} and a dbt v2 installation were " + "detected - using dbt v2. To use dbt-core instead, set " + "DBT_RUNNER_METHOD=api (or uninstall dbt v2)." + ) + return RunnerMethod.DBT2 + if dbt_core_version is not None: + # dbt-core published some 2.x pre-releases before dbt v2 moved to the + # dbt/dbt-oss packages - drive those with the dbt v2 runner too. if dbt_core_version.major >= 2: return RunnerMethod.DBT2 if dbt_core_version >= version.Version("1.5.0"): - return RunnerMethod.API - return RunnerMethod.SUBPROCESS + return RunnerMethod.DBT1_API + return RunnerMethod.DBT1_SUBPROCESS - if is_dbt2_binary_available(): - return RunnerMethod.DBT2 + # dbt may be installed without pip package metadata (e.g. a system-wide or + # pipx-managed dbt 1.x) - fall back to running it as a subprocess. + if shutil.which("dbt"): + return RunnerMethod.DBT1_SUBPROCESS - return RunnerMethod.SUBPROCESS + raise NoDbtInstallationError() def get_dbt_runner_class(runner_method: RunnerMethod) -> Type[CommandLineDbtRunner]: - if runner_method == RunnerMethod.API: + if runner_method == RunnerMethod.DBT1_API: # Import it internally since it will fail if dbt-core is not installed # or its version is below 1.5.0 from elementary.clients.dbt.api_dbt_runner import APIDbtRunner return APIDbtRunner - elif runner_method == RunnerMethod.SUBPROCESS: + elif runner_method == RunnerMethod.DBT1_SUBPROCESS: return SubprocessDbtRunner elif runner_method in (RunnerMethod.DBT2, RunnerMethod.FUSION): return Dbt2Runner diff --git a/elementary/exceptions/exceptions.py b/elementary/exceptions/exceptions.py index d6ad58bf6..a53659f88 100644 --- a/elementary/exceptions/exceptions.py +++ b/elementary/exceptions/exceptions.py @@ -31,6 +31,22 @@ def anonymous_tracking_context(self): } +class NoDbtInstallationError(ConfigError): + """Exception raised when no dbt installation is found in the environment""" + + def __init__(self) -> None: + super().__init__( + "No dbt installation was found. Please either install dbt v2 " + "(`pip install dbt` or `pip install dbt-oss`), or install " + "elementary-data with your warehouse's dbt 1.x adapter " + "(e.g. `pip install 'elementary-data[dbt1-snowflake]'`)." + ) + + @property + def anonymous_tracking_context(self): + return {"exception_message": str(self)} + + class DbtCommandError(Error): """Exception raised while executing a dbt command""" diff --git a/mypy.ini b/mypy.ini index 073e24ede..a2d7aad32 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,6 +3,10 @@ packages = elementary exclude = ^elementary/monitor/dbt_project/ check_untyped_defs = true +# dbt-core is an optional dependency (only installed via the dbt1-* extras) +[mypy-dbt,dbt.*] +ignore_missing_imports = True + [mypy-alive_progress] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 9fd4115ac..3459af782 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,9 +25,6 @@ packages = [{include = "elementary"}] [tool.poetry.dependencies] python = ">=3.10,<3.14" click = ">=7.0,<9.0" -# From 2.0, dbt-core is the Fusion engine (requires Python >=3.11); adapters are -# built into the engine, so the adapter extras below are only relevant for 1.x. -dbt-core = ">=1.8,<3.0.0" requests = ">=2.28.1,<3.0.0" urllib3 = ">=2.8.0" # transitive dependency via requests, which caps the major itself idna = ">=3.15" # transitive dependency via requests, floored to address CVE-2026-45409 (GHSA-65pc-fj4g-8rjx) ReDoS in idna.encode() @@ -53,6 +50,12 @@ tabulate = ">= 0.9.0" tenacity = ">=8.0,<10.0" pytz = ">= 2025.1" +# There is intentionally no mandatory dbt dependency: dbt v2 ships as the `dbt` / +# `dbt-oss` packages (dbt-core stays on 1.x) and requires no extra alongside +# elementary-data, while dbt-core 1.x comes through the adapter extras below. +# dbt-core is explicitly constrained in the extras so they can never resolve a +# dbt-core 2.x release (the adapters themselves only lower-bound dbt-core). +dbt-core = {version = ">=1.8,<2.0.0", optional = true} dbt-snowflake = {version = ">=1.8,<2.0.0", optional = true} dbt-bigquery = {version = ">=1.8,<2.0.0", optional = true} dbt-redshift = {version = ">=1.8,<2.0.0", optional = true} @@ -70,24 +73,42 @@ dbt-fabric = {version = ">=1.8,<2.0.0", optional = true} # dbt-sqlserver 1.12 moved pyodbc (still its default backend) to the [pyodbc] extra. dbt-sqlserver = {version = ">=1.8,<2.0.0", optional = true, extras = ["pyodbc"]} [tool.poetry.extras] -snowflake = ["dbt-snowflake"] -bigquery = ["dbt-bigquery"] -redshift = ["dbt-redshift"] -postgres = ["dbt-postgres"] -databricks = ["dbt-databricks"] -spark = ["dbt-spark"] -athena = ["dbt-athena-community", "pyathena"] -clickhouse = ["dbt-clickhouse"] -trino = ["dbt-trino"] -duckdb = ["dbt-duckdb"] -dremio = ["dbt-dremio"] -fabric = ["dbt-fabric"] -sqlserver = ["dbt-sqlserver"] +# The extras install the dbt-core 1.x stack (adapter + dbt-core) for a warehouse. +# They are not needed with dbt v2 (`pip install dbt` / `dbt-oss`), which has the +# adapters built in. The dbt1-* names are the documented ones; the plain +# warehouse names are kept as aliases for backward compatibility. +dbt1-snowflake = ["dbt-core", "dbt-snowflake"] +dbt1-bigquery = ["dbt-core", "dbt-bigquery"] +dbt1-redshift = ["dbt-core", "dbt-redshift"] +dbt1-postgres = ["dbt-core", "dbt-postgres"] +dbt1-databricks = ["dbt-core", "dbt-databricks"] +dbt1-spark = ["dbt-core", "dbt-spark"] +dbt1-athena = ["dbt-core", "dbt-athena-community", "pyathena"] +dbt1-clickhouse = ["dbt-core", "dbt-clickhouse"] +dbt1-trino = ["dbt-core", "dbt-trino"] +dbt1-duckdb = ["dbt-core", "dbt-duckdb"] +dbt1-dremio = ["dbt-core", "dbt-dremio"] +dbt1-fabric = ["dbt-core", "dbt-fabric"] +dbt1-sqlserver = ["dbt-core", "dbt-sqlserver"] +snowflake = ["dbt-core", "dbt-snowflake"] +bigquery = ["dbt-core", "dbt-bigquery"] +redshift = ["dbt-core", "dbt-redshift"] +postgres = ["dbt-core", "dbt-postgres"] +databricks = ["dbt-core", "dbt-databricks"] +spark = ["dbt-core", "dbt-spark"] +athena = ["dbt-core", "dbt-athena-community", "pyathena"] +clickhouse = ["dbt-core", "dbt-clickhouse"] +trino = ["dbt-core", "dbt-trino"] +duckdb = ["dbt-core", "dbt-duckdb"] +dremio = ["dbt-core", "dbt-dremio"] +fabric = ["dbt-core", "dbt-fabric"] +sqlserver = ["dbt-core", "dbt-sqlserver"] # dbt-fabricspark and dbt-vertica are removed from the lockfile resolution because they pin # outdated transitive dependencies (dbt-core==1.8.5, azure-cli pre-release) that block # security patches for deepdiff, protobuf, and pyopenssl. Users who need these adapters # can still install them directly (e.g. pip install dbt-fabricspark dbt-vertica). -all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] +dbt1-all = ["dbt-core", "dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] +all = ["dbt-core", "dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-athena-community", "pyathena", "dbt-trino", "dbt-clickhouse", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/unit/clients/dbt_runner/test_factory.py b/tests/unit/clients/dbt_runner/test_factory.py index d78bc386c..d4ba17b3e 100644 --- a/tests/unit/clients/dbt_runner/test_factory.py +++ b/tests/unit/clients/dbt_runner/test_factory.py @@ -8,6 +8,7 @@ from elementary.clients.dbt.dbt2_runner import Dbt2Runner from elementary.clients.dbt.dbt_installation import ( get_dbt2_binary_path, + get_dbt2_package_version, is_dbt2_binary_available, ) from elementary.clients.dbt.factory import ( @@ -16,12 +17,14 @@ get_dbt_runner_method, ) from elementary.clients.dbt.subprocess_dbt_runner import SubprocessDbtRunner +from elementary.exceptions.exceptions import NoDbtInstallationError def _mock_installation( monkeypatch, dbt_core_version: Optional[str] = None, dbt2_binary_available: bool = False, + dbt_on_path: bool = False, ): monkeypatch.setattr( factory, @@ -31,32 +34,58 @@ def _mock_installation( monkeypatch.setattr( factory, "is_dbt2_binary_available", lambda: dbt2_binary_available ) + monkeypatch.setattr( + factory.shutil, "which", lambda name: "/usr/bin/dbt" if dbt_on_path else None + ) @pytest.mark.parametrize( - "dbt_core_version,dbt2_binary_available,expected_method", + "dbt_core_version,dbt2_binary_available,dbt_on_path,expected_method", [ - ("1.8.0", False, RunnerMethod.API), - ("1.10.5", True, RunnerMethod.API), - ("1.4.0", False, RunnerMethod.SUBPROCESS), - ("2.0.0b2", False, RunnerMethod.DBT2), - (None, True, RunnerMethod.DBT2), - (None, False, RunnerMethod.SUBPROCESS), + ("1.8.0", False, True, RunnerMethod.DBT1_API), + # dbt v2 takes precedence when installed alongside dbt-core 1.x + ("1.10.5", True, True, RunnerMethod.DBT2), + ("1.4.0", False, True, RunnerMethod.DBT1_SUBPROCESS), + ("2.0.0b2", False, True, RunnerMethod.DBT2), + (None, True, True, RunnerMethod.DBT2), + # dbt installed without pip metadata (e.g. system-wide dbt 1.x) + (None, False, True, RunnerMethod.DBT1_SUBPROCESS), ], ) def test_get_dbt_runner_method_auto_detection( - monkeypatch, dbt_core_version, dbt2_binary_available, expected_method + monkeypatch, dbt_core_version, dbt2_binary_available, dbt_on_path, expected_method ): monkeypatch.delenv("DBT_RUNNER_METHOD", raising=False) - _mock_installation(monkeypatch, dbt_core_version, dbt2_binary_available) + _mock_installation( + monkeypatch, dbt_core_version, dbt2_binary_available, dbt_on_path + ) assert get_dbt_runner_method() == expected_method +def test_get_dbt_runner_method_raises_when_no_dbt_installation(monkeypatch): + monkeypatch.delenv("DBT_RUNNER_METHOD", raising=False) + _mock_installation(monkeypatch, dbt_core_version=None, dbt2_binary_available=False) + with pytest.raises(NoDbtInstallationError): + get_dbt_runner_method() + + +def test_get_dbt_runner_method_hints_when_dbt_core_and_dbt2_coexist( + monkeypatch, caplog +): + monkeypatch.delenv("DBT_RUNNER_METHOD", raising=False) + _mock_installation( + monkeypatch, dbt_core_version="1.10.0", dbt2_binary_available=True + ) + with caplog.at_level("INFO"): + assert get_dbt_runner_method() == RunnerMethod.DBT2 + assert any("using dbt v2" in record.message for record in caplog.records) + + @pytest.mark.parametrize( "env_value,expected_method", [ - ("subprocess", RunnerMethod.SUBPROCESS), - ("api", RunnerMethod.API), + ("subprocess", RunnerMethod.DBT1_SUBPROCESS), + ("api", RunnerMethod.DBT1_API), ("dbt2", RunnerMethod.DBT2), ("fusion", RunnerMethod.FUSION), ], @@ -67,47 +96,52 @@ def test_get_dbt_runner_method_env_override(monkeypatch, env_value, expected_met def test_get_dbt_runner_class(): - assert get_dbt_runner_class(RunnerMethod.SUBPROCESS) is SubprocessDbtRunner + assert get_dbt_runner_class(RunnerMethod.DBT1_SUBPROCESS) is SubprocessDbtRunner assert get_dbt_runner_class(RunnerMethod.DBT2) is Dbt2Runner assert get_dbt_runner_class(RunnerMethod.FUSION) is Dbt2Runner +def test_runner_method_legacy_aliases(): + assert RunnerMethod.API is RunnerMethod.DBT1_API + assert RunnerMethod.SUBPROCESS is RunnerMethod.DBT1_SUBPROCESS + + @mock.patch("elementary.clients.dbt.dbt_installation.shutil.which") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") def test_dbt2_runner_uses_path_binary_when_no_dbt_core( - mock_get_dbt_core_version, mock_get_dbt_package_version, mock_which, monkeypatch + mock_get_dbt_core_version, mock_get_dbt2_package_version, mock_which, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) mock_get_dbt_core_version.return_value = None - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None mock_which.return_value = "/some/venv/bin/dbt" assert get_dbt2_binary_path() == "/some/venv/bin/dbt" @mock.patch("elementary.clients.dbt.dbt_installation.shutil.which") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") def test_dbt2_runner_uses_path_binary_when_dbt2_pip_installed_alongside_dbt_core_1x( - mock_get_dbt_core_version, mock_get_dbt_package_version, mock_which, monkeypatch + mock_get_dbt_core_version, mock_get_dbt2_package_version, mock_which, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) mock_get_dbt_core_version.return_value = version.Version("1.10.0") - mock_get_dbt_package_version.return_value = version.Version("2.0.0rc212") + mock_get_dbt2_package_version.return_value = version.Version("2.0.6") mock_which.return_value = "/some/venv/bin/dbt" assert get_dbt2_binary_path() == "/some/venv/bin/dbt" -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") @mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_core_version") def test_dbt2_runner_ignores_path_binary_when_dbt_core_1x( - mock_get_dbt_core_version, mock_get_dbt_package_version, monkeypatch + mock_get_dbt_core_version, mock_get_dbt2_package_version, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) mock_get_dbt_core_version.return_value = version.Version("1.10.0") - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None assert get_dbt2_binary_path().endswith("/.local/bin/dbt") @@ -119,24 +153,48 @@ def test_dbt2_runner_honors_dbt_fusion_path_env_var(monkeypatch): @mock.patch("elementary.clients.dbt.dbt_installation.os.path.exists") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") def test_dbt2_binary_available_when_dbt_fusion_path_env_var_points_to_binary( - mock_get_dbt_package_version, mock_exists, monkeypatch + mock_get_dbt2_package_version, mock_exists, monkeypatch ): monkeypatch.setenv("DBT_FUSION_PATH", "/custom/path/dbt") - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None mock_exists.side_effect = lambda path: path == "/custom/path/dbt" assert is_dbt2_binary_available() @mock.patch("elementary.clients.dbt.dbt_installation.os.path.exists") -@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt_package_version") +@mock.patch("elementary.clients.dbt.dbt_installation.get_dbt2_package_version") def test_dbt2_binary_not_available_when_nothing_installed( - mock_get_dbt_package_version, mock_exists, monkeypatch + mock_get_dbt2_package_version, mock_exists, monkeypatch ): monkeypatch.delenv("DBT_FUSION_PATH", raising=False) - mock_get_dbt_package_version.return_value = None + mock_get_dbt2_package_version.return_value = None mock_exists.return_value = False assert not is_dbt2_binary_available() + + +@pytest.mark.parametrize( + "installed_packages,expected_version", + [ + ({"dbt": "2.0.6"}, "2.0.6"), + ({"dbt-oss": "2.0.5"}, "2.0.5"), + # A 1.x `dbt` package is not a dbt v2 installation + ({"dbt": "1.8.0"}, None), + ({}, None), + ], +) +@mock.patch("elementary.clients.dbt.dbt_installation._get_package_version") +def test_get_dbt2_package_version( + mock_get_package_version, installed_packages, expected_version +): + mock_get_package_version.side_effect = lambda name: ( + version.Version(installed_packages[name]) + if name in installed_packages + else None + ) + + expected = version.Version(expected_version) if expected_version else None + assert get_dbt2_package_version() == expected