Added non-covidcast signals import. Fixed signals filters.

src/signal_sets/resources.py

@@ -1,9 +1,11 @@
+from typing import Any
+
 from import_export import resources
 from import_export.fields import Field, widgets
 
-from signal_sets.models import SignalSet
-from signals.models import GeographicScope, Pathogen, SeverityPyramidRung, Geography
 from datasources.models import DataSource
+from signal_sets.models import SignalSet
+from signals.models import GeographicScope, Geography, Pathogen, SeverityPyramidRung
 
 
 def process_pathogens(row) -> None:
@@ -58,18 +60,29 @@ def process_datasources(row) -> None:
     """
     if row["Data Source"]:
         data_source = row["Data Source"]
-        data_source_obj, _ = DataSource.objects.get_or_create(
-            name=data_source
-        )
+        data_source_obj, _ = DataSource.objects.get_or_create(name=data_source)
         row["Data Source"] = data_source_obj
 
 
+def fix_boolean_fields(row) -> Any:
+    """
+    Fixes boolean fields.
+    """
+    fields = [
+        "Include in signal app",
+    ]
+    for k in fields:
+        if row[k] == "TRUE":
+            row[k] = True
+        if row[k] == "FALSE" or row[k] == "":
+            row[k] = False
+    return row
+
+
 class SignalSetResource(resources.ModelResource):
 
     name = Field(attribute="name", column_name="Signal Set name* ")
-    description = Field(
-        attribute="description", column_name="Signal Set description*"
-    )
+    description = Field(attribute="description", column_name="Signal Set description*")
     maintainer_name = Field(
         attribute="maintainer_name", column_name="Maintainer/\nKey Contact *"
     )
@@ -177,24 +190,32 @@ class Meta:
         store_instance = True
 
     def before_import_row(self, row, **kwargs):
+        fix_boolean_fields(row)
         process_pathogens(row)
         process_severity_pyramid_rungs(row)
         process_geographic_scope(row)
         process_avaliable_geographies(row)
         process_datasources(row)
 
-    def after_import_row(self, row, row_result, **kwargs):
-        signal_set_obj = SignalSet.objects.get(id=row_result.object_id)
-        for pathogen in row["Disease(s)/Pathogen(s)/Syndrome(s)"].split(","):
-            pathogen = Pathogen.objects.get(name=pathogen)
-            signal_set_obj.pathogens.add(pathogen)
-        for severity_pyramid_rung in row["Severity Pyramid Rung(s)"].split(","):
-            severity_pyramid_rung = SeverityPyramidRung.objects.filter(
-                name=severity_pyramid_rung
-            ).first()
-            signal_set_obj.severity_pyramid_rungs.add(severity_pyramid_rung)
+    def skip_row(self, instance, original, row, import_validation_errors=None):
+        if not row["Include in signal app"]:
+            return True
 
-        for available_geography in row["Geographic Granularity - Delphi"].split(","):
-            available_geography = Geography.objects.get(name=available_geography)
-            signal_set_obj.available_geographies.add(available_geography)
-        signal_set_obj.save()
+    def after_import_row(self, row, row_result, **kwargs):
+        try:
+            signal_set_obj = SignalSet.objects.get(id=row_result.object_id)
+            for pathogen in row["Disease(s)/Pathogen(s)/Syndrome(s)"].split(","):
+                pathogen = Pathogen.objects.get(name=pathogen)
+                signal_set_obj.pathogens.add(pathogen)
+            for severity_pyramid_rung in row["Severity Pyramid Rung(s)"].split(","):
+                severity_pyramid_rung = SeverityPyramidRung.objects.filter(
+                    name=severity_pyramid_rung
+                ).first()
+                signal_set_obj.severity_pyramid_rungs.add(severity_pyramid_rung)
+
+            for available_geography in row["Geographic Granularity - Delphi"].split(","):
+                available_geography = Geography.objects.get(name=available_geography)
+                signal_set_obj.available_geographies.add(available_geography)
+            signal_set_obj.save()
+        except SignalSet.DoesNotExist as e:
+            print(f"SignalSet.DoesNotExist: {e}")

src/signals/filters.py

@@ -1,6 +1,8 @@
 import logging
 
 import django_filters
+from django.db.models import Q
+import ast
 from django_filters.widgets import BooleanWidget, QueryArrayWidget
 
 from signals.models import (
@@ -21,27 +23,21 @@ class SignalFilter(django_filters.FilterSet):
         widget=BooleanWidget(),
     )
 
-    pathogens = django_filters.ModelMultipleChoiceFilter(
-        field_name="pathogens",
-        queryset=Pathogen.objects.all(),
-        widget=QueryArrayWidget,
+    pathogens = django_filters.CharFilter(
+        method="filter_pathogens", widget=QueryArrayWidget
     )
 
-    geographic_scope = django_filters.ModelMultipleChoiceFilter(
-        field_name="geographic_scope",
-        queryset=GeographicScope.objects.all(),
-        widget=QueryArrayWidget,
+    geographic_scope = django_filters.CharFilter(
+        method="filter_geographic_scope", widget=QueryArrayWidget
     )
 
-    available_geography = django_filters.ModelMultipleChoiceFilter(
-        field_name="available_geography",
-        queryset=Geography.objects.all().order_by("display_order_number"),
+    available_geography = django_filters.CharFilter(
+        method="filter_available_geography",
         widget=QueryArrayWidget,
     )
 
-    severity_pyramid_rung = django_filters.ModelMultipleChoiceFilter(
-        field_name="severity_pyramid_rung",
-        queryset=SeverityPyramidRung.objects.all(),
+    severity_pyramid_rung = django_filters.CharFilter(
+        method="filter_severity_pyramid_rung",
         widget=QueryArrayWidget,
     )
 
@@ -88,3 +84,69 @@ class Meta:
             "to_date",
             "signal_availability_days",
         ]
+
+    def filter_pathogens(self, queryset, name, value):
+        if not value:
+            return queryset
+        pathogens = list(
+            Pathogen.objects.filter(id__in=ast.literal_eval(value)).values_list(
+                "name", flat=True
+            )
+        )
+        queries: list[Q] = [Q((f"{name}__icontains", p)) for p in pathogens]
+        query: Q = queries.pop()
+
+        for item in queries:
+            query |= item
+
+        return queryset.filter(query)
+
+    def filter_geographic_scope(self, queryset, name, value):
+        if not value:
+            return queryset
+        geographic_scopes = list(
+            GeographicScope.objects.filter(id__in=ast.literal_eval(value)).values_list(
+                "name", flat=True
+            )
+        )
+        queries: list[Q] = [Q((f"{name}__icontains", g)) for g in geographic_scopes]
+        query: Q = queries.pop()
+
+        for item in queries:
+            query |= item
+
+        return queryset.filter(query)
+
+    def filter_available_geography(self, queryset, name, value):
+        if not value:
+            return queryset
+        available_geography = list(
+            Geography.objects.filter(id__in=ast.literal_eval(value)).values_list(
+                "name", flat=True
+            )
+        )
+        queries: list[Q] = [Q((f"{name}__icontains", ag)) for ag in available_geography]
+        query: Q = queries.pop()
+
+        for item in queries:
+            query |= item
+
+        return queryset.filter(query)
+
+    def filter_severity_pyramid_rung(self, queryset, name, value):
+        if not value:
+            return queryset
+        severity_pyramid_rungs = list(
+            SeverityPyramidRung.objects.filter(
+                id__in=ast.literal_eval(value)
+            ).values_list("name", flat=True)
+        )
+        queries: list[Q] = [
+            Q((f"{name}__icontains", s)) for s in severity_pyramid_rungs
+        ]
+        query: Q = queries.pop()
+
+        for item in queries:
+            query |= item
+
+        return queryset.filter(query)

src/signals/migrations/0002_auto_20241002_2027.py

@@ -15,7 +15,8 @@ class Migration(migrations.Migration):
             CREATE OR REPLACE VIEW signals_signal_view AS (
                 SELECT
                 ss.id,
-                ss.display_name AS "name",
+                ss.name as "name",
+                ss.display_name AS "display_name",
                 ss.active AS "active",
                 ds.display_name AS "datasource",
                 ss.description AS "description",
@@ -24,7 +25,7 @@ class Migration(migrations.Migration):
                 ss.temporal_scope_end AS "temporal_scope_end",
                 ss.time_type AS "time_type",
                 GROUP_CONCAT(DISTINCT CONCAT(signal_geo.name, '', IF(ss2.aggregated_by_delphi, '(by Delphi)', ''))) AS "available_geography", 
-                GROUP_CONCAT(DISTINCT sp.name) AS "pathogens", 
+                GROUP_CONCAT(DISTINCT sp.name) AS "pathogens",
                 ss.reporting_cadence AS "reporting_cadence",
                 ss.typical_reporting_lag AS "typical_reporting_lag",
                 ss.typical_revision_cadence AS "typical_revision_cadence",

src/signals/migrations/0004_alter_signal_active_alter_signal_available_geography_and_more.py

@@ -0,0 +1,146 @@
+# Generated by Django 5.0.7 on 2024-11-11 16:59
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0001_initial'),
+        ('datasources', '0002_alter_datasource_display_name'),
+        ('signals', '0003_signal_signal_set'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='signal',
+            name='active',
+            field=models.BooleanField(blank=True, default=False, help_text='Ongoing', null=True, verbose_name='active'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='available_geography',
+            field=models.ManyToManyField(blank=True, help_text='Available geographies for the signal.', null=True, related_name='signals', through='signals.SignalGeography', to='signals.geography'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='demographic_scope',
+            field=models.CharField(blank=True, help_text='Demographic scope of the signal.', max_length=255, null=True, verbose_name='demographic scope'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='description',
+            field=models.TextField(blank=True, help_text='Description of the signal.', null=True, verbose_name='description'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='display_name',
+            field=models.CharField(blank=True, help_text='Display name of the signal.', max_length=255, null=True, verbose_name='display name'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='geographic_scope',
+            field=models.ForeignKey(blank=True, help_text='Geographic scope of the signal.', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='signals', to='signals.geographicscope'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='has_sample_size',
+            field=models.BooleanField(blank=True, default=False, help_text='Has sample size', null=True, verbose_name='has sample size'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='has_stderr',
+            field=models.BooleanField(blank=True, default=False, help_text='Has stderr', null=True, verbose_name='has stderr'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='high_values_are',
+            field=models.CharField(blank=True, help_text='High values are', max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='is_cumulative',
+            field=models.BooleanField(blank=True, default=False, help_text='Is cumulative', null=True, verbose_name='is cumulative'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='is_smoothed',
+            field=models.BooleanField(blank=True, default=False, help_text='Is smoothed', null=True, verbose_name='is smoothed'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='is_weighted',
+            field=models.BooleanField(blank=True, default=False, help_text='Is weighted', null=True, verbose_name='is weighted'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='organization_access_list',
+            field=models.CharField(blank=True, help_text='Organisations Access List. Who may access this signal?', max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='organization_sharing_list',
+            field=models.CharField(blank=True, help_text='Organisations Sharing List. Who may share this signal?', max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='pathogen',
+            field=models.ManyToManyField(blank=True, help_text='Pathogen/Disease area', null=True, related_name='signals', to='signals.pathogen'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='related_links',
+            field=models.ManyToManyField(blank=True, help_text='Related signal links.', null=True, related_name='signals', to='base.link'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='reporting_cadence',
+            field=models.CharField(blank=True, help_text='Reporting cadence of the signal.', max_length=255, null=True, verbose_name='reporting cadence'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='source',
+            field=models.ForeignKey(blank=True, help_text='Source Subdivision', null=True, on_delete=django.db.models.deletion.PROTECT, related_name='signals', to='datasources.sourcesubdivision'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='temporal_scope_end',
+            field=models.CharField(blank=True, help_text='Temporal scope end of the signal.', max_length=255, null=True, verbose_name='temporal scope end'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='temporal_scope_end_note',
+            field=models.TextField(blank=True, help_text='Temporal scope end note of the signal.', null=True, verbose_name='temporal scope end note'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='temporal_scope_start',
+            field=models.CharField(blank=True, help_text='Temporal scope start of the signal.', max_length=255, null=True, verbose_name='temporal scope start'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='temporal_scope_start_note',
+            field=models.TextField(blank=True, help_text='Temporal scope start note of the signal.', null=True, verbose_name='temporal scope start note'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='time_label',
+            field=models.CharField(blank=True, help_text='Time label of the signal.', max_length=255, null=True, verbose_name='time label'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='time_type',
+            field=models.CharField(blank=True, help_text='Time type of the signal.', max_length=255, null=True, verbose_name='time type'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='typical_reporting_lag',
+            field=models.CharField(blank=True, help_text='Typical reporting lag of the signal.', max_length=255, null=True, verbose_name='typical reporting lag'),
+        ),
+        migrations.AlterField(
+            model_name='signal',
+            name='typical_revision_cadence',
+            field=models.TextField(blank=True, help_text='Typical revision cadence of the signal.', null=True, verbose_name='typical revision cadence'),
+        ),
+    ]