Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Creating a new incremental table for with-trip vehicle position message age by day #3492

Merged
merged 2 commits into from
Oct 8, 2024
Merged

Creating a new incremental table for with-trip vehicle position message age by day #3492

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bd24a7d
Creating a new incremental table for with-trip vehicle position messa…
Oct 7, 2024
54b5af8
Merge branch 'main' into trip_vehicle_position
fsalemi Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions ...se/models/mart/gtfs_quality/fct_daily_with_trip_vehicle_positions_message_age_summary.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{{
config(
materialized = 'incremental',
incremental_strategy = 'insert_overwrite',
partition_by = {
'field': 'dt',
'data_type': 'date',
'granularity': 'day'
}
)
}}

WITH vehicle_positions_ages AS (
SELECT DISTINCT
dt,
gtfs_dataset_key,
gtfs_dataset_name,
trip_id IS NOT NULL AS with_trip,
_header_message_age,
_vehicle_message_age,
_vehicle_message_age_vs_header
FROM {{ ref('fct_vehicle_positions_messages') }}
WHERE {{ incremental_where(default_start_var = 'PROD_GTFS_RT_START') }}
),

-- Deduplicating rows to the overall message level for meaningful summary statistics
vehicle_age_percentiles AS (
SELECT
*,
-- Calculating percentiles for vehicle message age
PERCENTILE_CONT(_vehicle_message_age, 0.5) OVER(PARTITION BY dt, gtfs_dataset_key, with_trip) AS median_vehicle_message_age,
PERCENTILE_CONT(_vehicle_message_age, 0.25) OVER(PARTITION BY dt, gtfs_dataset_key, with_trip) AS p25_vehicle_message_age,
PERCENTILE_CONT(_vehicle_message_age, 0.75) OVER(PARTITION BY dt, gtfs_dataset_key, with_trip) AS p75_vehicle_message_age,
PERCENTILE_CONT(_vehicle_message_age, 0.90) OVER(PARTITION BY dt, gtfs_dataset_key, with_trip) AS p90_vehicle_message_age,
PERCENTILE_CONT(_vehicle_message_age, 0.99) OVER(PARTITION BY dt, gtfs_dataset_key, with_trip) AS p99_vehicle_message_age
FROM vehicle_positions_ages
),

summarize_vehicle_ages AS (
SELECT
dt,
gtfs_dataset_key,
gtfs_dataset_name,
with_trip,
median_vehicle_message_age,
p25_vehicle_message_age,
p75_vehicle_message_age,
p90_vehicle_message_age,
p99_vehicle_message_age,
MAX(_vehicle_message_age) AS max_vehicle_message_age,
MIN(_vehicle_message_age) AS min_vehicle_message_age,
AVG(_vehicle_message_age) AS avg_vehicle_message_age
FROM vehicle_age_percentiles
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9
),

fct_daily_with_trip_vehicle_positions_message_age_summary AS (
SELECT
{{ dbt_utils.generate_surrogate_key(['dt', 'gtfs_dataset_key', 'with_trip']) }} AS key,
dt,
gtfs_dataset_key,
gtfs_dataset_name,
with_trip,
median_vehicle_message_age,
p25_vehicle_message_age,
p75_vehicle_message_age,
p90_vehicle_message_age,
p99_vehicle_message_age,
max_vehicle_message_age,
min_vehicle_message_age,
avg_vehicle_message_age
FROM summarize_vehicle_ages
ORDER BY 1, 3, 4
)

SELECT *
FROM fct_daily_with_trip_vehicle_positions_message_age_summary
Loading