Skip to content

Commit 77475d0

Browse files
dmytrotskodmytrotsko
committed
Added non-covidcast signals import. Fixed signals filters.
1 parent cd6d64d commit 77475d0

12 files changed

+493
-82
lines changed

‎src/signal_sets/resources.py

Lines changed: 43 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1+
from typing import Any
2+
13
from import_export import resources
24
from import_export.fields import Field, widgets
35

4-
from signal_sets.models import SignalSet
5-
from signals.models import GeographicScope, Pathogen, SeverityPyramidRung, Geography
66
from datasources.models import DataSource
7+
from signal_sets.models import SignalSet
8+
from signals.models import GeographicScope, Geography, Pathogen, SeverityPyramidRung
79

810

911
def process_pathogens(row) -> None:
@@ -58,18 +60,29 @@ def process_datasources(row) -> None:
5860
"""
5961
if row["Data Source"]:
6062
data_source = row["Data Source"]
61-
data_source_obj, _ = DataSource.objects.get_or_create(
62-
name=data_source
63-
)
63+
data_source_obj, _ = DataSource.objects.get_or_create(name=data_source)
6464
row["Data Source"] = data_source_obj
6565

6666

67+
def fix_boolean_fields(row) -> Any:
68+
"""
69+
Fixes boolean fields.
70+
"""
71+
fields = [
72+
"Include in signal app",
73+
]
74+
for k in fields:
75+
if row[k] == "TRUE":
76+
row[k] = True
77+
if row[k] == "FALSE" or row[k] == "":
78+
row[k] = False
79+
return row
80+
81+
6782
class SignalSetResource(resources.ModelResource):
6883

6984
name = Field(attribute="name", column_name="Signal Set name* ")
70-
description = Field(
71-
attribute="description", column_name="Signal Set description*"
72-
)
85+
description = Field(attribute="description", column_name="Signal Set description*")
7386
maintainer_name = Field(
7487
attribute="maintainer_name", column_name="Maintainer/\nKey Contact *"
7588
)
@@ -177,24 +190,32 @@ class Meta:
177190
store_instance = True
178191

179192
def before_import_row(self, row, **kwargs):
193+
fix_boolean_fields(row)
180194
process_pathogens(row)
181195
process_severity_pyramid_rungs(row)
182196
process_geographic_scope(row)
183197
process_avaliable_geographies(row)
184198
process_datasources(row)
185199

186-
def after_import_row(self, row, row_result, **kwargs):
187-
signal_set_obj = SignalSet.objects.get(id=row_result.object_id)
188-
for pathogen in row["Disease(s)/Pathogen(s)/Syndrome(s)"].split(","):
189-
pathogen = Pathogen.objects.get(name=pathogen)
190-
signal_set_obj.pathogens.add(pathogen)
191-
for severity_pyramid_rung in row["Severity Pyramid Rung(s)"].split(","):
192-
severity_pyramid_rung = SeverityPyramidRung.objects.filter(
193-
name=severity_pyramid_rung
194-
).first()
195-
signal_set_obj.severity_pyramid_rungs.add(severity_pyramid_rung)
200+
def skip_row(self, instance, original, row, import_validation_errors=None):
201+
if not row["Include in signal app"]:
202+
return True
196203

197-
for available_geography in row["Geographic Granularity - Delphi"].split(","):
198-
available_geography = Geography.objects.get(name=available_geography)
199-
signal_set_obj.available_geographies.add(available_geography)
200-
signal_set_obj.save()
204+
def after_import_row(self, row, row_result, **kwargs):
205+
try:
206+
signal_set_obj = SignalSet.objects.get(id=row_result.object_id)
207+
for pathogen in row["Disease(s)/Pathogen(s)/Syndrome(s)"].split(","):
208+
pathogen = Pathogen.objects.get(name=pathogen)
209+
signal_set_obj.pathogens.add(pathogen)
210+
for severity_pyramid_rung in row["Severity Pyramid Rung(s)"].split(","):
211+
severity_pyramid_rung = SeverityPyramidRung.objects.filter(
212+
name=severity_pyramid_rung
213+
).first()
214+
signal_set_obj.severity_pyramid_rungs.add(severity_pyramid_rung)
215+
216+
for available_geography in row["Geographic Granularity - Delphi"].split(","):
217+
available_geography = Geography.objects.get(name=available_geography)
218+
signal_set_obj.available_geographies.add(available_geography)
219+
signal_set_obj.save()
220+
except SignalSet.DoesNotExist as e:
221+
print(f"SignalSet.DoesNotExist: {e}")

‎src/signals/filters.py

Lines changed: 76 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import logging
22

33
import django_filters
4+
from django.db.models import Q
5+
import ast
46
from django_filters.widgets import BooleanWidget, QueryArrayWidget
57

68
from signals.models import (
@@ -21,27 +23,21 @@ class SignalFilter(django_filters.FilterSet):
2123
widget=BooleanWidget(),
2224
)
2325

24-
pathogens = django_filters.ModelMultipleChoiceFilter(
25-
field_name="pathogens",
26-
queryset=Pathogen.objects.all(),
27-
widget=QueryArrayWidget,
26+
pathogens = django_filters.CharFilter(
27+
method="filter_pathogens", widget=QueryArrayWidget
2828
)
2929

30-
geographic_scope = django_filters.ModelMultipleChoiceFilter(
31-
field_name="geographic_scope",
32-
queryset=GeographicScope.objects.all(),
33-
widget=QueryArrayWidget,
30+
geographic_scope = django_filters.CharFilter(
31+
method="filter_geographic_scope", widget=QueryArrayWidget
3432
)
3533

36-
available_geography = django_filters.ModelMultipleChoiceFilter(
37-
field_name="available_geography",
38-
queryset=Geography.objects.all().order_by("display_order_number"),
34+
available_geography = django_filters.CharFilter(
35+
method="filter_available_geography",
3936
widget=QueryArrayWidget,
4037
)
4138

42-
severity_pyramid_rung = django_filters.ModelMultipleChoiceFilter(
43-
field_name="severity_pyramid_rung",
44-
queryset=SeverityPyramidRung.objects.all(),
39+
severity_pyramid_rung = django_filters.CharFilter(
40+
method="filter_severity_pyramid_rung",
4541
widget=QueryArrayWidget,
4642
)
4743

@@ -88,3 +84,69 @@ class Meta:
8884
"to_date",
8985
"signal_availability_days",
9086
]
87+
88+
def filter_pathogens(self, queryset, name, value):
89+
if not value:
90+
return queryset
91+
pathogens = list(
92+
Pathogen.objects.filter(id__in=ast.literal_eval(value)).values_list(
93+
"name", flat=True
94+
)
95+
)
96+
queries: list[Q] = [Q((f"{name}__icontains", p)) for p in pathogens]
97+
query: Q = queries.pop()
98+
99+
for item in queries:
100+
query |= item
101+
102+
return queryset.filter(query)
103+
104+
def filter_geographic_scope(self, queryset, name, value):
105+
if not value:
106+
return queryset
107+
geographic_scopes = list(
108+
GeographicScope.objects.filter(id__in=ast.literal_eval(value)).values_list(
109+
"name", flat=True
110+
)
111+
)
112+
queries: list[Q] = [Q((f"{name}__icontains", g)) for g in geographic_scopes]
113+
query: Q = queries.pop()
114+
115+
for item in queries:
116+
query |= item
117+
118+
return queryset.filter(query)
119+
120+
def filter_available_geography(self, queryset, name, value):
121+
if not value:
122+
return queryset
123+
available_geography = list(
124+
Geography.objects.filter(id__in=ast.literal_eval(value)).values_list(
125+
"name", flat=True
126+
)
127+
)
128+
queries: list[Q] = [Q((f"{name}__icontains", ag)) for ag in available_geography]
129+
query: Q = queries.pop()
130+
131+
for item in queries:
132+
query |= item
133+
134+
return queryset.filter(query)
135+
136+
def filter_severity_pyramid_rung(self, queryset, name, value):
137+
if not value:
138+
return queryset
139+
severity_pyramid_rungs = list(
140+
SeverityPyramidRung.objects.filter(
141+
id__in=ast.literal_eval(value)
142+
).values_list("name", flat=True)
143+
)
144+
queries: list[Q] = [
145+
Q((f"{name}__icontains", s)) for s in severity_pyramid_rungs
146+
]
147+
query: Q = queries.pop()
148+
149+
for item in queries:
150+
query |= item
151+
152+
return queryset.filter(query)

‎src/signals/migrations/0002_auto_20241002_2027.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ class Migration(migrations.Migration):
1515
CREATE OR REPLACE VIEW signals_signal_view AS (
1616
SELECT
1717
ss.id,
18-
ss.display_name AS "name",
18+
ss.name as "name",
19+
ss.display_name AS "display_name",
1920
ss.active AS "active",
2021
ds.display_name AS "datasource",
2122
ss.description AS "description",
@@ -24,7 +25,7 @@ class Migration(migrations.Migration):
2425
ss.temporal_scope_end AS "temporal_scope_end",
2526
ss.time_type AS "time_type",
2627
GROUP_CONCAT(DISTINCT CONCAT(signal_geo.name, '', IF(ss2.aggregated_by_delphi, '(by Delphi)', ''))) AS "available_geography",
27-
GROUP_CONCAT(DISTINCT sp.name) AS "pathogens",
28+
GROUP_CONCAT(DISTINCT sp.name) AS "pathogens",
2829
ss.reporting_cadence AS "reporting_cadence",
2930
ss.typical_reporting_lag AS "typical_reporting_lag",
3031
ss.typical_revision_cadence AS "typical_revision_cadence",

‎src/signals/migrations/0004_alter_signal_active_alter_signal_available_geography_and_more.py

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
# Generated by Django 5.0.7 on 2024-11-11 16:59
2+
3+
import django.db.models.deletion
4+
from django.db import migrations, models
5+
6+
7+
class Migration(migrations.Migration):
8+
9+
dependencies = [
10+
('base', '0001_initial'),
11+
('datasources', '0002_alter_datasource_display_name'),
12+
('signals', '0003_signal_signal_set'),
13+
]
14+
15+
operations = [
16+
migrations.AlterField(
17+
model_name='signal',
18+
name='active',
19+
field=models.BooleanField(blank=True, default=False, help_text='Ongoing', null=True, verbose_name='active'),
20+
),
21+
migrations.AlterField(
22+
model_name='signal',
23+
name='available_geography',
24+
field=models.ManyToManyField(blank=True, help_text='Available geographies for the signal.', null=True, related_name='signals', through='signals.SignalGeography', to='signals.geography'),
25+
),
26+
migrations.AlterField(
27+
model_name='signal',
28+
name='demographic_scope',
29+
field=models.CharField(blank=True, help_text='Demographic scope of the signal.', max_length=255, null=True, verbose_name='demographic scope'),
30+
),
31+
migrations.AlterField(
32+
model_name='signal',
33+
name='description',
34+
field=models.TextField(blank=True, help_text='Description of the signal.', null=True, verbose_name='description'),
35+
),
36+
migrations.AlterField(
37+
model_name='signal',
38+
name='display_name',
39+
field=models.CharField(blank=True, help_text='Display name of the signal.', max_length=255, null=True, verbose_name='display name'),
40+
),
41+
migrations.AlterField(
42+
model_name='signal',
43+
name='geographic_scope',
44+
field=models.ForeignKey(blank=True, help_text='Geographic scope of the signal.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='signals', to='signals.geographicscope'),
45+
),
46+
migrations.AlterField(
47+
model_name='signal',
48+
name='has_sample_size',
49+
field=models.BooleanField(blank=True, default=False, help_text='Has sample size', null=True, verbose_name='has sample size'),
50+
),
51+
migrations.AlterField(
52+
model_name='signal',
53+
name='has_stderr',
54+
field=models.BooleanField(blank=True, default=False, help_text='Has stderr', null=True, verbose_name='has stderr'),
55+
),
56+
migrations.AlterField(
57+
model_name='signal',
58+
name='high_values_are',
59+
field=models.CharField(blank=True, help_text='High values are', max_length=128, null=True),
60+
),
61+
migrations.AlterField(
62+
model_name='signal',
63+
name='is_cumulative',
64+
field=models.BooleanField(blank=True, default=False, help_text='Is cumulative', null=True, verbose_name='is cumulative'),
65+
),
66+
migrations.AlterField(
67+
model_name='signal',
68+
name='is_smoothed',
69+
field=models.BooleanField(blank=True, default=False, help_text='Is smoothed', null=True, verbose_name='is smoothed'),
70+
),
71+
migrations.AlterField(
72+
model_name='signal',
73+
name='is_weighted',
74+
field=models.BooleanField(blank=True, default=False, help_text='Is weighted', null=True, verbose_name='is weighted'),
75+
),
76+
migrations.AlterField(
77+
model_name='signal',
78+
name='organization_access_list',
79+
field=models.CharField(blank=True, help_text='Organisations Access List. Who may access this signal?', max_length=128, null=True),
80+
),
81+
migrations.AlterField(
82+
model_name='signal',
83+
name='organization_sharing_list',
84+
field=models.CharField(blank=True, help_text='Organisations Sharing List. Who may share this signal?', max_length=128, null=True),
85+
),
86+
migrations.AlterField(
87+
model_name='signal',
88+
name='pathogen',
89+
field=models.ManyToManyField(blank=True, help_text='Pathogen/Disease area', null=True, related_name='signals', to='signals.pathogen'),
90+
),
91+
migrations.AlterField(
92+
model_name='signal',
93+
name='related_links',
94+
field=models.ManyToManyField(blank=True, help_text='Related signal links.', null=True, related_name='signals', to='base.link'),
95+
),
96+
migrations.AlterField(
97+
model_name='signal',
98+
name='reporting_cadence',
99+
field=models.CharField(blank=True, help_text='Reporting cadence of the signal.', max_length=255, null=True, verbose_name='reporting cadence'),
100+
),
101+
migrations.AlterField(
102+
model_name='signal',
103+
name='source',
104+
field=models.ForeignKey(blank=True, help_text='Source Subdivision', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='signals', to='datasources.sourcesubdivision'),
105+
),
106+
migrations.AlterField(
107+
model_name='signal',
108+
name='temporal_scope_end',
109+
field=models.CharField(blank=True, help_text='Temporal scope end of the signal.', max_length=255, null=True, verbose_name='temporal scope end'),
110+
),
111+
migrations.AlterField(
112+
model_name='signal',
113+
name='temporal_scope_end_note',
114+
field=models.TextField(blank=True, help_text='Temporal scope end note of the signal.', null=True, verbose_name='temporal scope end note'),
115+
),
116+
migrations.AlterField(
117+
model_name='signal',
118+
name='temporal_scope_start',
119+
field=models.CharField(blank=True, help_text='Temporal scope start of the signal.', max_length=255, null=True, verbose_name='temporal scope start'),
120+
),
121+
migrations.AlterField(
122+
model_name='signal',
123+
name='temporal_scope_start_note',
124+
field=models.TextField(blank=True, help_text='Temporal scope start note of the signal.', null=True, verbose_name='temporal scope start note'),
125+
),
126+
migrations.AlterField(
127+
model_name='signal',
128+
name='time_label',
129+
field=models.CharField(blank=True, help_text='Time label of the signal.', max_length=255, null=True, verbose_name='time label'),
130+
),
131+
migrations.AlterField(
132+
model_name='signal',
133+
name='time_type',
134+
field=models.CharField(blank=True, help_text='Time type of the signal.', max_length=255, null=True, verbose_name='time type'),
135+
),
136+
migrations.AlterField(
137+
model_name='signal',
138+
name='typical_reporting_lag',
139+
field=models.CharField(blank=True, help_text='Typical reporting lag of the signal.', max_length=255, null=True, verbose_name='typical reporting lag'),
140+
),
141+
migrations.AlterField(
142+
model_name='signal',
143+
name='typical_revision_cadence',
144+
field=models.TextField(blank=True, help_text='Typical revision cadence of the signal.', null=True, verbose_name='typical revision cadence'),
145+
),
146+
]

0 commit comments

Comments
 (0)