Skip to content

Commit e0e3873

Browse files
charlie-costanzocharlie-costanzo
authored
Payments: Add string handling, refactor modeling of Elavon data (#2957)
* refactored staging table to focus on cleaning * broke out intermediate tables to focus on billing and deposits exclusively, and deduplicate * removed cleaning steps and refactored to focus on union of billing and deposits * replace previous transactions table * broke out intermediate tables to focus on billing and deposits exclusively, and deduplicate * made parse_elavon_date macro, implemented in stg_elavon_transactions * combined import CTEs for conciseness * implement qualify for intermediate table deduplication * changed elavon fact table name back to fct_elavon__transactions for the time being * removed _deduped from elavon intermediate tables since this is now handled in staging * fixed table names downstream for renamed tables * missed some renaming downstream * removed always null columns from int_elavon__billing_transactions * moved elavon deduplication to staging table * updated dbt docs
1 parent 3a27a09 commit e0e3873

File tree

7 files changed

+341
-229
lines changed

7 files changed

+341
-229
lines changed

‎warehouse/macros/parse_elavon_date.sql

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{% macro parse_elavon_date(column_name) %}
2+
3+
CASE
4+
WHEN LENGTH({{ column_name }}) < 8
5+
THEN PARSE_DATE('%m%d%Y', CONCAT(0, {{ column_name }}))
6+
ELSE PARSE_DATE('%m%d%Y', {{ column_name }})
7+
END
8+
9+
{% endmacro %}

‎warehouse/models/mart/payments/elavon_littlepay__daily_history_transactions_deposits_billing.sql

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
{{ config(materialized='table') }}
22

3-
WITH fct_elavon__transactions AS (
3+
WITH
4+
5+
fct_elavon__transactions AS (
46
SELECT
57

68
*,

‎warehouse/models/mart/payments/fct_elavon__transactions.sql

Lines changed: 62 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,98 +1,78 @@
11
{{ config(materialized='table') }}
22

3-
WITH stg_elavon__transactions AS (
4-
SELECT * FROM {{ ref('stg_elavon__transactions') }}
5-
),
3+
WITH
64

7-
remove_special_characters AS (
8-
SELECT
5+
int_elavon__billing_transactions AS (
6+
SELECT * FROM {{ ref('int_elavon__billing_transactions') }}
7+
),
98

10-
* EXCEPT (fund_amt, batch_amt, amount, surchg_amount, convnce_amt, payment_date, transaction_date, settlement_date),
9+
int_elavon__deposit_transactions AS (
10+
SELECT * FROM {{ ref('int_elavon__deposit_transactions') }}
11+
),
1112

12-
CAST(REGEXP_REPLACE(fund_amt, r'\$|,', '') as NUMERIC) AS fund_amt,
13-
CAST(REGEXP_REPLACE(batch_amt, r'\$|,', '') as NUMERIC) AS batch_amt,
14-
CAST(REGEXP_REPLACE(amount, r'\$|,', '') as NUMERIC) AS amount,
15-
CAST(REGEXP_REPLACE(surchg_amount, r'\$|,', '') as NUMERIC) AS surchg_amount,
16-
CAST(REGEXP_REPLACE(convnce_amt, r'\$|,', '') as NUMERIC) AS convnce_amt,
13+
union_deposits_and_billing AS (
1714

18-
REGEXP_EXTRACT(payment_date, r'[^@\.]+') AS payment_date,
19-
REGEXP_EXTRACT(transaction_date, r'[^@\.]+') AS transaction_date,
20-
REGEXP_EXTRACT(settlement_date, r'[^@\.]+') AS settlement_date
15+
SELECT
16+
*
17+
FROM int_elavon__billing_transactions
18+
UNION ALL
19+
SELECT
20+
*
21+
FROM int_elavon__deposit_transactions
2122

22-
FROM stg_elavon__transactions
2323
),
2424

2525
fct_elavon__transactions AS (
26-
SELECT
27-
28-
* EXCEPT (payment_date, transaction_date, settlement_date),
2926

30-
CASE WHEN
31-
LENGTH(payment_date) < 8
32-
THEN PARSE_DATE('%m%d%Y', CONCAT(0, payment_date))
33-
ELSE PARSE_DATE('%m%d%Y', payment_date)
34-
END AS payment_date,
27+
SELECT
3528

36-
CASE WHEN
37-
LENGTH(transaction_date) < 8
38-
THEN PARSE_DATE('%m%d%Y', CONCAT(0, transaction_date))
39-
ELSE PARSE_DATE('%m%d%Y', transaction_date)
40-
END AS transaction_date,
29+
payment_reference,
30+
payment_date,
31+
account_number,
32+
routing_number,
33+
fund_amt,
34+
batch_reference,
35+
batch_type,
36+
customer_batch_reference,
37+
customer_name,
38+
merchant_number,
39+
external_mid,
40+
store_number,
41+
chain,
42+
batch_amt,
43+
amount,
44+
surchg_amount,
45+
convnce_amt,
46+
card_type,
47+
charge_type,
48+
charge_type_description,
49+
card_plan,
50+
card_no,
51+
chk_num,
52+
transaction_date,
53+
settlement_date,
54+
authorization_code,
55+
chargeback_control_no,
56+
roc_text,
57+
trn_aci,
58+
card_scheme_ref,
59+
trn_ref_num,
60+
settlement_method,
61+
currency_code,
62+
cb_acq_ref_id,
63+
chgbk_rsn_code,
64+
chgbk_rsn_desc,
65+
mer_ref,
66+
purch_id,
67+
cust_cod,
68+
trn_arn,
69+
term_id,
70+
ent_num,
71+
dt,
72+
execution_ts
4173

42-
CASE WHEN
43-
LENGTH(settlement_date) < 8
44-
THEN PARSE_DATE('%m%d%Y', CONCAT(0, settlement_date))
45-
ELSE PARSE_DATE('%m%d%Y', settlement_date)
46-
END AS settlement_date
74+
FROM union_deposits_and_billing
4775

48-
FROM remove_special_characters
4976
)
5077

51-
SELECT
52-
53-
payment_reference,
54-
payment_date,
55-
account_number,
56-
routing_number,
57-
fund_amt,
58-
batch_reference,
59-
batch_type,
60-
customer_batch_reference,
61-
customer_name,
62-
merchant_number,
63-
external_mid,
64-
store_number,
65-
chain,
66-
batch_amt,
67-
amount,
68-
surchg_amount,
69-
convnce_amt,
70-
card_type,
71-
charge_type,
72-
charge_type_description,
73-
card_plan,
74-
card_no,
75-
chk_num,
76-
transaction_date,
77-
settlement_date,
78-
authorization_code,
79-
chargeback_control_no,
80-
roc_text,
81-
trn_aci,
82-
card_scheme_ref,
83-
trn_ref_num,
84-
settlement_method,
85-
currency_code,
86-
cb_acq_ref_id,
87-
chgbk_rsn_code,
88-
chgbk_rsn_desc,
89-
mer_ref,
90-
purch_id,
91-
cust_cod,
92-
trn_arn,
93-
term_id,
94-
ent_num,
95-
dt,
96-
execution_ts
97-
98-
FROM fct_elavon__transactions
78+
SELECT * FROM fct_elavon__transactions

‎warehouse/models/staging/payments/elavon/_elavon.yml

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,81 @@ sources:
1111

1212
models:
1313
- name: stg_elavon__transactions
14-
description: Transactions processed by Elavon
14+
description: Transactions processed by Elavon, cleaned and deduped.
15+
columns:
16+
- name: payment_reference
17+
- name: payment_date
18+
- name: account_number
19+
- name: routing_number
20+
- name: fund_amt
21+
- name: batch_reference
22+
- name: batch_type
23+
- name: customer_batch_reference
24+
- name: customer_name
25+
- name: merchant_number
26+
- name: external_mid
27+
- name: store_number
28+
- name: chain
29+
- name: batch_amt
30+
- name: amount
31+
- name: surchg_amount
32+
- name: convnce_amt
33+
- name: card_type
34+
- name: charge_type
35+
- name: charge_type_description
36+
- name: card_plan
37+
- name: card_no
38+
- name: chk_num
39+
- name: transaction_date
40+
- name: settlement_date
41+
- name: authorization_code
42+
- name: chargeback_control_no
43+
- name: roc_text
44+
- name: trn_aci
45+
- name: card_scheme_ref
46+
- name: trn_ref_num
47+
- name: settlement_method
48+
- name: currency_code
49+
- name: cb_acq_ref_id
50+
- name: chgbk_rsn_code
51+
- name: chgbk_rsn_desc
52+
- name: mer_ref
53+
- name: purch_id
54+
- name: cust_cod
55+
- name: trn_arn
56+
- name: term_id
57+
- name: ent_num
58+
- name: dt
59+
- name: execution_ts
60+
61+
- name: int_elavon__billing_transactions
62+
description: Billing-specific transactions processed by Elavon
63+
columns:
64+
- name: payment_reference
65+
- name: payment_date
66+
- name: account_number
67+
- name: routing_number
68+
- name: fund_amt
69+
- name: batch_reference
70+
- name: batch_type
71+
- name: customer_name
72+
- name: merchant_number
73+
- name: external_mid
74+
- name: chain
75+
- name: batch_amt
76+
- name: amount
77+
- name: card_type
78+
- name: charge_type
79+
- name: charge_type_description
80+
- name: card_plan
81+
- name: settlement_method
82+
- name: currency_code
83+
- name: ent_num
84+
- name: dt
85+
- name: execution_ts
86+
87+
- name: int_elavon__deposit_transactions
88+
description: Deposit-specific transactions processed by Elavon
1589
columns:
1690
- name: payment_reference
1791
- name: payment_date

‎warehouse/models/staging/payments/elavon/int_elavon__billing_transactions.sql

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{{ config(materialized='table') }}
2+
3+
WITH
4+
5+
billing_transactions AS (
6+
7+
SELECT * FROM {{ ref('stg_elavon__transactions') }}
8+
WHERE batch_type = 'B'
9+
10+
),
11+
12+
int_elavon__billing_transactions AS (
13+
14+
SELECT
15+
16+
payment_reference,
17+
payment_date,
18+
account_number,
19+
routing_number,
20+
fund_amt,
21+
batch_reference,
22+
batch_type,
23+
customer_name,
24+
merchant_number,
25+
external_mid,
26+
chain,
27+
batch_amt,
28+
amount,
29+
card_type,
30+
charge_type,
31+
charge_type_description,
32+
card_plan,
33+
settlement_method,
34+
currency_code,
35+
ent_num,
36+
dt,
37+
execution_ts
38+
39+
FROM billing_transactions
40+
41+
)
42+
43+
SELECT * FROM int_elavon__billing_transactions

‎warehouse/models/staging/payments/elavon/int_elavon__deposit_transactions.sql

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{{ config(materialized='table') }}
2+
3+
WITH
4+
5+
deposit_transactions AS (
6+
7+
SELECT * FROM {{ ref('stg_elavon__transactions') }}
8+
WHERE batch_type = 'D'
9+
10+
),
11+
12+
int_elavon__deposit_transactions AS (
13+
14+
SELECT
15+
payment_reference,
16+
payment_date,
17+
account_number,
18+
routing_number,
19+
fund_amt,
20+
batch_reference,
21+
batch_type,
22+
customer_batch_reference,
23+
customer_name,
24+
merchant_number,
25+
external_mid,
26+
store_number,
27+
chain,
28+
batch_amt,
29+
amount,
30+
surchg_amount,
31+
convnce_amt,
32+
card_type,
33+
charge_type,
34+
charge_type_description,
35+
card_plan,
36+
card_no,
37+
chk_num,
38+
transaction_date,
39+
settlement_date,
40+
authorization_code,
41+
chargeback_control_no,
42+
roc_text,
43+
trn_aci,
44+
card_scheme_ref,
45+
trn_ref_num,
46+
settlement_method,
47+
currency_code,
48+
cb_acq_ref_id,
49+
chgbk_rsn_code,
50+
chgbk_rsn_desc,
51+
mer_ref,
52+
purch_id,
53+
cust_cod,
54+
trn_arn,
55+
term_id,
56+
ent_num,
57+
dt,
58+
execution_ts
59+
60+
FROM deposit_transactions
61+
62+
)
63+
64+
SELECT * FROM int_elavon__deposit_transactions

0 commit comments

Comments
 (0)