Skip to content

Commit 855dbef

Browse files
fivetran-joemarkiewiczfivetran-joemarkiewicz
committed
release review updates and databricks sql test run addition
1 parent 3d32ce3 commit 855dbef

File tree

13 files changed

+109
-33
lines changed

13 files changed

+109
-33
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,7 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
2121
export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
2222
export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
2323
export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
24-
export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
24+
export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
25+
export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
26+
export CI_DATABRICKS_SQL_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
27+
export CI_DATABRICKS_SQL_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_SQL_DBT_TOKEN" --project="dbt-package-testing-363917")

‎.buildkite/pipeline.yml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,19 @@ steps:
7070
- "CI_DATABRICKS_DBT_HTTP_PATH"
7171
- "CI_DATABRICKS_DBT_TOKEN"
7272
commands: |
73-
bash .buildkite/scripts/run_models.sh databricks
73+
bash .buildkite/scripts/run_models.sh databricks
74+
75+
- label: ":databricks: :database: Run Tests - Databricks SQL Warehouse"
76+
key: "run_dbt_databricks_sql"
77+
plugins:
78+
- docker#v3.13.0:
79+
image: "python:3.8"
80+
shell: [ "/bin/bash", "-e", "-c" ]
81+
environment:
82+
- "BASH_ENV=/tmp/.bashrc"
83+
- "CI_DATABRICKS_DBT_HOST"
84+
- "CI_DATABRICKS_SQL_DBT_HTTP_PATH"
85+
- "CI_DATABRICKS_SQL_DBT_TOKEN"
86+
- "CI_DATABRICKS_DBT_CATALOG"
87+
commands: |
88+
bash .buildkite/scripts/run_models.sh databricks-sql

‎.buildkite/scripts/run_models.sh‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,15 @@ db=$1
1616
echo `pwd`
1717
cd integration_tests
1818
dbt deps
19+
if [ "$db" = "databricks-sql" ]; then
20+
dbt seed --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
21+
dbt compile --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
22+
dbt run --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db" --full-refresh
23+
dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
24+
dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db"
25+
dbt run --vars '{hubspot_schema: hubspot_sqlw_tests, hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db"
26+
dbt test --vars '{hubspot_schema: hubspot_sqlw_tests}' --target "$db"
27+
else
1928
dbt seed --target "$db" --full-refresh
2029
dbt run --target "$db" --full-refresh
2130
dbt test --target "$db"
@@ -26,5 +35,5 @@ dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: false}
2635
dbt run --vars '{hubspot_deal_enabled: true, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true}' --target "$db"
2736
dbt run --vars '{hubspot__pass_through_all_columns: true, hubspot_service_enabled: true, hubspot_using_all_email_events: false}' --target "$db"
2837
dbt test --target "$db"
29-
38+
fi
3039
dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

‎docs/catalog.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎docs/index.html‎

Lines changed: 37 additions & 10 deletions
Large diffs are not rendered by default.

‎docs/manifest.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎docs/run_results.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎integration_tests/ci/sample.profiles.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,18 @@ integration_tests:
4545
schema: hubspot_source_integration_tests_21
4646
threads: 8
4747
databricks:
48-
catalog: null
48+
catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
4949
host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
5050
http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
5151
schema: hubspot_source_integration_tests_21
5252
threads: 8
5353
token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
5454
type: databricks
55+
databricks-sql:
56+
catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
57+
host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
58+
http_path: "{{ env_var('CI_DATABRICKS_SQL_DBT_HTTP_PATH') }}"
59+
schema: hubspot_sqlw_tests
60+
threads: 8
61+
token: "{{ env_var('CI_DATABRICKS_SQL_DBT_TOKEN') }}"
62+
type: databricks

‎integration_tests/dbt_project.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@ name: 'hubspot_source_integration_tests'
22
version: '0.15.0'
33
profile: 'integration_tests'
44
config-version: 2
5+
56
models:
6-
hubspot_source:
7-
+schema:
7+
+schema: "{{ 'hubspot_sqlw_tests' if target.name == 'databricks-sql' else 'hubspot' }}"
8+
# +schema: "hubspot_{{ var('directed_schema','dev') }}" ## To be used for validation testing
9+
810
vars:
911
hubspot_schema: hubspot_source_integration_tests_21
1012
hubspot_source:

‎integration_tests/seeds/ticket_property_history_data.csv‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
_fivetran_synced,ticket_id,name,source,source_id,timestamp_instant,value
2-
2020-07-09 11:06:21.056,1,seed name, source name, sg5923,2020-07-09 11:06:21.056,value
3-
2020-07-09 11:06:21.056,1,seed data is cool, source name1, sg5923,2020-07-09 11:06:21.056,value
4-
2020-07-09 11:06:21.056,1,i can make these names whatever i want, source name2, sg5923,2020-07-09 11:06:21.056,value
5-
2020-07-09 11:06:21.056,1,seed name1, source name3, sg5923,2020-07-09 11:06:21.056,value
6-
2020-07-09 11:06:21.056,1,seed name2, source name4, sg5923,2020-07-09 11:06:21.056,value
7-
2020-07-09 11:06:21.056,1,seed name3, source name5, sg5923,2020-07-09 11:06:21.056,value
8-
2020-07-09 11:06:21.056,1,stop being lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value
9-
2020-07-09 11:06:21.056,1,seed name4, source name6?, sg5923,2020-07-09 11:06:21.056,value
10-
2020-07-09 11:06:21.056,1,seed name5, source name7, sg5923,2020-07-09 11:06:21.056,value
11-
2020-07-09 11:06:21.056,1,seed name6, source name8, sg5923,2020-07-09 11:06:21.056,value
12-
2020-07-09 11:06:21.056,1,i got lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value
1+
_fivetran_synced,ticket_id,name,source,source_id,timestamp_instant,value,_fivetran_start,_fivetran_end
2+
2020-07-09 11:06:21.056,1,seed name, source name, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
3+
2020-07-09 11:06:21.056,1,seed data is cool, source name1, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
4+
2020-07-09 11:06:21.056,1,i can make these names whatever i want, source name2, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
5+
2020-07-09 11:06:21.056,1,seed name1, source name3, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
6+
2020-07-09 11:06:21.056,1,seed name2, source name4, sg5923,2020-07-09 11:06:21.056,value,2023-01-04 12:39:41,9999-12-31 23:59:59
7+
2020-07-09 11:06:21.056,1,seed name3, source name5, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
8+
2020-07-09 11:06:21.056,1,stop being lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
9+
2020-07-09 11:06:21.056,1,seed name4, source name6?, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
10+
2020-07-09 11:06:21.056,1,seed name5, source name7, sg5923,2020-07-09 11:06:21.056,value,2023-09-04 12:39:41,9999-12-31 23:59:59
11+
2020-07-09 11:06:21.056,1,seed name6, source name8, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59
12+
2020-07-09 11:06:21.056,1,i got lazy with names, source name99, sg5923,2020-07-09 11:06:21.056,value,2023-10-04 12:39:41,9999-12-31 23:59:59

0 commit comments

Comments
 (0)