Merge branch 'release/0.7.6-b1'

.circleci/config.yml

@@ -56,7 +56,7 @@ jobs:
   integration:
     docker:
       - image: cimg/python:3.8.5
-    parallelism: 15
+    parallelism: 20
     steps:
       - build_test_env
       - run:

macros/materialisations/incremental_bridge_helpers.sql

@@ -0,0 +1,26 @@
+{% macro is_bridge_incremental() %}
+    {#-- do not run introspective queries in parsing #}
+    {% if not execute %}
+        {{ return(False) }}
+    {% else %}
+        {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
+
+            {{ return(relation is not none
+                      and relation.type == 'table'
+                      and model.config.materialized == 'bridge_incremental'
+                      and not flags.FULL_REFRESH) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %}
+    {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
+    {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
+
+    TRUNCATE TABLE {{ target_relation }};
+
+    INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
+    (
+       SELECT {{ dest_cols_csv }}
+       FROM {{ tmp_relation }}
+    );
+{%- endmacro %}

macros/materialisations/incremental_bridge_materialization.sql

@@ -0,0 +1,53 @@
+{% materialization bridge_incremental, default -%}
+
+  {% set full_refresh_mode = flags.FULL_REFRESH %}
+
+  {% set target_relation = this %}
+  {% set existing_relation = load_relation(this) %}
+  {% set tmp_relation = make_temp_relation(this) %}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  {% set to_drop = [] %}
+  {% if existing_relation is none %}
+      {% set build_sql = create_table_as(False, target_relation, sql) %}
+  {% elif existing_relation.is_view or full_refresh_mode %}
+      {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
+      {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
+      {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
+      {% do adapter.drop_relation(backup_relation) %}
+
+      {% do adapter.rename_relation(target_relation, backup_relation) %}
+      {% set build_sql = create_table_as(False, target_relation, sql) %}
+      {% do to_drop.append(backup_relation) %}
+  {% else %}
+
+      {% set tmp_relation = make_temp_relation(target_relation) %}
+      {% do run_query(create_table_as(True, tmp_relation, sql)) %}
+      {% do adapter.expand_target_column_types(
+             from_relation=tmp_relation,
+             to_relation=target_relation) %}
+      {% set build_sql = dbtvault.incremental_bridge_replace(tmp_relation, target_relation) %}
+{% endif %}
+
+  {% call statement("main") %}
+      {{ build_sql }}
+  {% endcall %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {% do adapter.commit() %}
+
+  {% for rel in to_drop %}
+      {% do adapter.drop_relation(rel) %}
+  {% endfor %}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{%- endmaterialization %}

macros/materialisations/incremental_pit_helper.sql

@@ -0,0 +1,26 @@
+{% macro is_pit_incremental() %}
+    {#-- do not run introspective queries in parsing #}
+    {% if not execute %}
+        {{ return(False) }}
+    {% else %}
+        {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
+
+            {{ return(relation is not none
+                      and relation.type == 'table'
+                      and model.config.materialized == 'pit_incremental'
+                      and not flags.FULL_REFRESH) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %}
+    {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
+    {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
+
+    TRUNCATE TABLE {{ target_relation }};
+
+    INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
+    (
+       SELECT {{ dest_cols_csv }}
+       FROM {{ tmp_relation }}
+    );
+{%- endmacro %}

macros/materialisations/incremental_pit_materialization.sql

@@ -0,0 +1,53 @@
+{% materialization pit_incremental, default -%}
+
+  {% set full_refresh_mode = flags.FULL_REFRESH %}
+
+  {% set target_relation = this %}
+  {% set existing_relation = load_relation(this) %}
+  {% set tmp_relation = make_temp_relation(this) %}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  {% set to_drop = [] %}
+  {% if existing_relation is none %}
+      {% set build_sql = create_table_as(False, target_relation, sql) %}
+  {% elif existing_relation.is_view or full_refresh_mode %}
+      {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
+      {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
+      {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
+      {% do adapter.drop_relation(backup_relation) %}
+
+      {% do adapter.rename_relation(target_relation, backup_relation) %}
+      {% set build_sql = create_table_as(False, target_relation, sql) %}
+      {% do to_drop.append(backup_relation) %}
+  {% else %}
+
+      {% set tmp_relation = make_temp_relation(target_relation) %}
+      {% do run_query(create_table_as(True, tmp_relation, sql)) %}
+      {% do adapter.expand_target_column_types(
+             from_relation=tmp_relation,
+             to_relation=target_relation) %}
+      {% set build_sql = dbtvault.incremental_pit_replace(tmp_relation, target_relation) %}
+{% endif %}
+
+  {% call statement("main") %}
+      {{ build_sql }}
+  {% endcall %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {% do adapter.commit() %}
+
+  {% for rel in to_drop %}
+      {% do adapter.drop_relation(rel) %}
+  {% endfor %}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+  {{ return({'relations': [target_relation]}) }}
+
+{%- endmaterialization %}

macros/materialisations/shared_helpers.sql

@@ -11,7 +11,7 @@
 
 
 {%- macro is_any_incremental() -%}
-    {%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or is_incremental() -%}
+    {%- if dbtvault.is_vault_insert_by_period() or dbtvault.is_vault_insert_by_rank() or dbtvault.is_pit_incremental() or dbtvault.is_bridge_incremental() or is_incremental() -%}
         {%- do return(true) -%}
     {%- else -%}
         {%- do return(false) -%}