From 58957d6efd79932960c26bd85b5b922d28ea908c Mon Sep 17 00:00:00 2001 From: Can Bekleyici Date: Mon, 13 Jan 2025 15:46:12 +0100 Subject: [PATCH 1/2] fix: replicated database materializations --- dbt/adapters/clickhouse/impl.py | 5 +-- dbt/adapters/clickhouse/relation.py | 11 ++++--- dbt/include/clickhouse/macros/adapters.sql | 5 ++- .../test_replicated_database.py | 31 +++++++++++++++++++ 4 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 tests/integration/adapter/replicated_database/test_replicated_database.py diff --git a/dbt/adapters/clickhouse/impl.py b/dbt/adapters/clickhouse/impl.py index 91238ddf..2bfec7d2 100644 --- a/dbt/adapters/clickhouse/impl.py +++ b/dbt/adapters/clickhouse/impl.py @@ -176,7 +176,7 @@ def can_exchange(self, schema: str, rel_type: str) -> bool: def should_on_cluster(self, materialized: str = '', engine: str = '') -> bool: conn = self.connections.get_if_exists() if conn and conn.credentials.cluster: - return ClickHouseRelation.get_on_cluster(conn.credentials.cluster, materialized, engine) + return ClickHouseRelation.get_on_cluster(conn.credentials.cluster, materialized, engine, conn.credentials.database_engine) return ClickHouseRelation.get_on_cluster('', materialized, engine) @available.parse_none @@ -324,6 +324,7 @@ def list_relations_without_caching( and rel_type == ClickHouseRelationType.Table and db_engine in ('Atomic', 'Replicated') ) + can_on_cluster = (on_cluster >= 1) and db_engine != 'Replicated' relation = self.Relation.create( database='', @@ -331,7 +332,7 @@ def list_relations_without_caching( identifier=name, type=rel_type, can_exchange=can_exchange, - can_on_cluster=(on_cluster >= 1), + can_on_cluster=can_on_cluster, ) relations.append(relation) diff --git a/dbt/adapters/clickhouse/relation.py b/dbt/adapters/clickhouse/relation.py index 87134c5a..51f11baa 100644 --- a/dbt/adapters/clickhouse/relation.py +++ b/dbt/adapters/clickhouse/relation.py @@ -78,17 +78,17 @@ def should_on_cluster(self) -> bool: @classmethod def get_on_cluster( - cls: Type[Self], cluster: str = '', materialized: str = '', engine: str = '' + cls: Type[Self], cluster: str = '', materialized: str = '', engine: str = '', database_engine: str = '' ) -> bool: + if 'replicated' in database_engine.lower(): + return False if cluster.strip(): return ( materialized in ('view', 'dictionary') or 'distributed' in materialized or 'Replicated' in engine ) - - else: - return False + return False @classmethod def create_from( @@ -126,7 +126,8 @@ def create_from( else: materialized = relation_config.config.get('materialized') or '' engine = relation_config.config.get('engine') or '' - can_on_cluster = cls.get_on_cluster(cluster, materialized, engine) + database_engine = quoting.credentials.database_engine or '' + can_on_cluster = cls.get_on_cluster(cluster, materialized, engine, database_engine) return cls.create( database='', diff --git a/dbt/include/clickhouse/macros/adapters.sql b/dbt/include/clickhouse/macros/adapters.sql index 6ae897d7..71084938 100644 --- a/dbt/include/clickhouse/macros/adapters.sql +++ b/dbt/include/clickhouse/macros/adapters.sql @@ -59,14 +59,13 @@ {% macro clickhouse__drop_relation(relation, obj_type='table') -%} {% call statement('drop_relation', auto_begin=False) -%} - {# drop relation on cluster by default if cluster is set #} - drop {{ obj_type }} if exists {{ relation }} {{ on_cluster_clause(relation.without_identifier(), True)}} + drop {{ obj_type }} if exists {{ relation }} {{ on_cluster_clause(relation, True)}} {%- endcall %} {% endmacro %} {% macro clickhouse__rename_relation(from_relation, to_relation, obj_type='table') -%} {% call statement('drop_relation') %} - drop {{ obj_type }} if exists {{ to_relation }} {{ on_cluster_clause(to_relation.without_identifier())}} + drop {{ obj_type }} if exists {{ to_relation }} {{ on_cluster_clause(to_relation)}} {% endcall %} {% call statement('rename_relation') %} rename {{ obj_type }} {{ from_relation }} to {{ to_relation }} {{ on_cluster_clause(from_relation)}} diff --git a/tests/integration/adapter/replicated_database/test_replicated_database.py b/tests/integration/adapter/replicated_database/test_replicated_database.py new file mode 100644 index 00000000..39d9dab2 --- /dev/null +++ b/tests/integration/adapter/replicated_database/test_replicated_database.py @@ -0,0 +1,31 @@ +import pytest + +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.files import model_incremental, schema_base_yml + + +class TestReplicatedDatabaseSimpleMaterialization(BaseSimpleMaterializations): + """Contains tests for table, view and swappable view materialization.""" + @pytest.fixture(scope="class") + def test_config(self, test_config): + test_config["db_engine"] = "Replicated('/clickhouse/databases/{uuid}', '{shard}', '{replica}')" + return test_config + + +class TestReplicatedDatabaseIncremental(BaseIncremental): + @pytest.fixture(scope="class") + def test_config(self, test_config): + test_config["db_engine"] = "Replicated('/clickhouse/databases/{uuid}', '{shard}', '{replica}')" + return test_config + + @pytest.fixture(scope="class") + def models(self): + config_materialized_incremental = """ + {{ config(order_by='(some_date, id, name)', inserts_only=True, materialized='incremental', unique_key='id') }} + """ + incremental_sql = config_materialized_incremental + model_incremental + return { + "incremental.sql": incremental_sql, + "schema.yml": schema_base_yml, + } From 587bd248d2892ab3cb7092b866891efe5a8d92cb Mon Sep 17 00:00:00 2001 From: Can Bekleyici Date: Mon, 13 Jan 2025 15:54:51 +0100 Subject: [PATCH 2/2] docs: add bug fix change --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07a42ff6..f9ce571b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,10 @@ The index config should be added to the model config. for instance: }] ) }} ``` - + +### Bug Fixes +* Materializations are now compatible with `Replicated` database engine, as they will no longer use `ON CLUSTER` statements. + ### Release [1.8.7], 2025-01-05 ### New Features