diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6af344bf0..71c784a03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: args: ["--profile", "black"] - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 diff --git a/integration_tests/dbt_project/dbt_project.yml b/integration_tests/dbt_project/dbt_project.yml index bb2663282..1de92bbce 100644 --- a/integration_tests/dbt_project/dbt_project.yml +++ b/integration_tests/dbt_project/dbt_project.yml @@ -31,4 +31,4 @@ models: +file_format: "{{ 'delta' if target.type in ['spark', 'fabricspark'] else none }}" flags: - require_batched_execution_for_custom_microbatch_strategy: True \ No newline at end of file + require_batched_execution_for_custom_microbatch_strategy: True diff --git a/integration_tests/tests/test_dbt_artifacts/test_microbatch_compiled_code.py b/integration_tests/tests/test_dbt_artifacts/test_microbatch_compiled_code.py index dea35420a..60e49fb36 100644 --- a/integration_tests/tests/test_dbt_artifacts/test_microbatch_compiled_code.py +++ b/integration_tests/tests/test_dbt_artifacts/test_microbatch_compiled_code.py @@ -1,7 +1,6 @@ from contextlib import contextmanager import pytest - from dbt_project import DbtProject @@ -44,20 +43,30 @@ def _microbatch_model_sql(source_model_name: str) -> str: amount, order_date from {{ ref('__MICROBATCH_SOURCE_MODEL__') }} -""".replace("__MICROBATCH_SOURCE_MODEL__", source_model_name) +""".replace( + "__MICROBATCH_SOURCE_MODEL__", source_model_name + ) @contextmanager def _with_microbatch_test_models(dbt_project: DbtProject, model_suffix: str): source_model_name = f"mb_src_{model_suffix}" target_model_name = f"mb_tgt_{model_suffix}" - source_model_path = dbt_project.tmp_models_dir_path.joinpath(f"{source_model_name}.sql") - target_model_path = dbt_project.tmp_models_dir_path.joinpath(f"{target_model_name}.sql") + source_model_path = dbt_project.tmp_models_dir_path.joinpath( + f"{source_model_name}.sql" + ) + target_model_path = dbt_project.tmp_models_dir_path.joinpath( + f"{target_model_name}.sql" + ) source_model_path.write_text(_microbatch_source_model_sql()) target_model_path.write_text(_microbatch_model_sql(source_model_name)) - relative_source_model_path = source_model_path.relative_to(dbt_project.project_dir_path) - relative_target_model_path = target_model_path.relative_to(dbt_project.project_dir_path) + relative_source_model_path = source_model_path.relative_to( + dbt_project.project_dir_path + ) + relative_target_model_path = target_model_path.relative_to( + dbt_project.project_dir_path + ) try: yield relative_source_model_path, relative_target_model_path, target_model_name finally: @@ -75,9 +84,7 @@ def _run_microbatch_model_and_get_latest_success_result( model_path, target_model_name, ): - dbt_project.dbt_runner.run( - select=f"{source_model_path} {model_path}" - ) + dbt_project.dbt_runner.run(select=f"{source_model_path} {model_path}") unique_id = f"model.elementary_tests.{target_model_name}" run_results = dbt_project.read_table( @@ -91,14 +98,14 @@ def _run_microbatch_model_and_get_latest_success_result( @contextmanager def _with_microbatch_macro_file(dbt_project: DbtProject, macro_name: str): - macro_path = ( - dbt_project.project_dir_path / "macros" / "microbatch.sql" - ) + macro_path = dbt_project.project_dir_path / "macros" / "microbatch.sql" macro_sql = """ {% macro __MACRO_NAME__(arg_dict) %} {{ return(elementary.get_incremental_microbatch_sql(arg_dict)) }} {% endmacro %} -""".replace("__MACRO_NAME__", macro_name) +""".replace( + "__MACRO_NAME__", macro_name + ) if macro_path.exists(): raise FileExistsError(f"Expected no macro file at {macro_path}") @@ -110,7 +117,9 @@ def _with_microbatch_macro_file(dbt_project: DbtProject, macro_name: str): macro_path.unlink() -@pytest.mark.skip_targets(["spark", "vertica", "bigquery", "athena", "clickhouse", "dremio"]) +@pytest.mark.skip_targets( + ["spark", "vertica", "bigquery", "athena", "clickhouse", "dremio"] +) @pytest.mark.skip_for_dbt_fusion @pytest.mark.parametrize( "macro_name,expected_compiled_code,model_suffix", @@ -134,10 +143,10 @@ def test_microbatch_run_results_compiled_code_behavior( ) assert run_results, "Expected a successful run result row for microbatch model" if expected_compiled_code: - assert run_results[0]["compiled_code"], ( - "Expected compiled_code to be populated when override macro is present" - ) + assert run_results[0][ + "compiled_code" + ], "Expected compiled_code to be populated when override macro is present" else: - assert not run_results[0]["compiled_code"], ( - "Expected compiled_code to stay empty when override macro is absent" - ) + assert not run_results[0][ + "compiled_code" + ], "Expected compiled_code to stay empty when override macro is absent" diff --git a/macros/edr/dbt_artifacts/microbatch/capture_microbatch_compiled_code.sql b/macros/edr/dbt_artifacts/microbatch/capture_microbatch_compiled_code.sql index 856dd91cf..588c32cd5 100644 --- a/macros/edr/dbt_artifacts/microbatch/capture_microbatch_compiled_code.sql +++ b/macros/edr/dbt_artifacts/microbatch/capture_microbatch_compiled_code.sql @@ -31,14 +31,12 @@ model.get("unique_id") if model is mapping else model.unique_id ) | default(none, true) %} {% set model_compiled_code = ( - model.get("compiled_code") if model is mapping else model.compiled_code + model.get("compiled_code") + if model is mapping + else model.compiled_code ) | default(none, true) %} - {% if model_unique_id is none %} - {{ return(none) }} - {% endif %} - {% if not model_compiled_code %} - {{ return(none) }} - {% endif %} + {% if model_unique_id is none %} {{ return(none) }} {% endif %} + {% if not model_compiled_code %} {{ return(none) }} {% endif %} {% set compiled_code_by_unique_id = elementary.get_cache( "microbatch_compiled_code_by_unique_id" diff --git a/macros/utils/graph/get_compiled_code.sql b/macros/utils/graph/get_compiled_code.sql index 6ce92316e..80f78a578 100644 --- a/macros/utils/graph/get_compiled_code.sql +++ b/macros/utils/graph/get_compiled_code.sql @@ -5,7 +5,9 @@ "microbatch_compiled_code_by_unique_id", {} ).get(node.get("unique_id")) %} {% endif %} - {% set compiled_code = adapter.dispatch("format_compiled_code", "elementary")(compiled_code) %} + {% set compiled_code = adapter.dispatch("format_compiled_code", "elementary")( + compiled_code + ) %} {% set max_column_size = elementary.get_column_size() %} {% if as_column_value and max_column_size and compiled_code and compiled_code | length > max_column_size %}