Merge pull request #19 from cmu-delphi/staging

Staging

.ci.env

@@ -0,0 +1,22 @@
+# This file is used to allow CI to start the compose services. It will typcially
+# not need to be modified.
+
+MYSQL_DATABASE=mysql_database
+MYSQL_USER=mysql_user
+MYSQL_PASSWORD=mysql_password
+MYSQL_PORT=3306
+MYSQL_ROOT_PASSWORD=test123!
+MYSQL_HOST=db
+
+ALLOWED_HOSTS='127.0.0.1,localhost'
+CORS_ORIGIN_WHITELIST='http://127.0.0.1:3000,http://localhost:3000'
+CSRF_TRUSTED_ORIGINS='http://127.0.0.1:8000,http://localhost:8000'
+
+SECRET_KEY='secret_key'
+DEBUG='True'
+
+# Add the following to your local .env file. They will be used in the CI process
+# and you can largely forget about them, but including them in your .env file
+# will act like a safe default and help suppress warnings.
+REGISTRY=""
+TAG=""

.github/workflows/build-and-deploy.yaml

@@ -0,0 +1,60 @@
+name: 'Build and deploy application containers'
+on:
+  push:
+jobs:
+  build-tag-push-deploy:
+    runs-on: ubuntu-latest
+    # CI/CD will run on these branches
+    if: >
+      github.ref == 'refs/heads/main' ||
+      github.ref == 'refs/heads/staging' ||
+      github.ref == 'refs/heads/development'
+
+    strategy:
+      matrix:
+        # Specify the docker-compose services to build images from. These should match the service
+        # names in the docker-compose.yml file.
+        service: [sdwebapp, sdnginx]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: cmu-delphi-deploy-machine
+          password: ${{ secrets.CMU_DELPHI_DEPLOY_MACHINE_PAT }}
+      - name: Create container image tags
+        id: image-tag
+        run: |
+          baseRef="${GITHUB_REF#*/}"
+          baseRef="${baseRef#*/}"
+          case "${baseRef}" in
+          main)
+            image_tag="latest"
+            ;;
+          *)
+            image_tag="${baseRef//\//_}" # replace `/` with `_` in branch name
+            ;;
+          esac
+          echo "IMAGE_TAG=${image_tag}" >> $GITHUB_OUTPUT
+      - name: Copy env file
+        run: |
+          cp ./.ci.env ./.env
+      - name: Set up docker-compose
+        uses: ndeloof/[email protected]
+      - name: docker-compose build --push
+        run: |
+          docker-compose build --push ${{ matrix.service }}
+        env:
+          TAG: ":${{ steps.image-tag.outputs.IMAGE_TAG }}"
+          REGISTRY: "ghcr.io/${{ github.repository_owner }}/"
+      - name: docker-compose down
+        run: |
+          docker-compose down
+      - name: Trigger smee.io webhook to pull new container images
+        run: |
+          curl -H "Authorization: Bearer ${{ secrets.DELPHI_DEPLOY_WEBHOOK_TOKEN }}" \
+               -X POST ${{ secrets.DELPHI_DEPLOY_WEBHOOK_URL }} \
+               -H "Content-Type: application/x-www-form-urlencoded" \
+               -d "repository=ghcr.io/${{ github.repository }}-${{ matrix.service }}&tag=${{ steps.image-tag.outputs.IMAGE_TAG }}"

docker-compose.yaml

@@ -1,7 +1,7 @@
 services:
   db:
       image: mysql:latest
-      container_name: signal_documentation-db
+      container_name: signal-discovery-db
       restart: always
       env_file:
           - ./.env
@@ -16,21 +16,21 @@ services:
           - "3306:3306"
 
   # Production service - "service", "image", and "container_name" should all contain the same
-  # reference, based on the name of the service.
+  # reference based on the name of the service.
   sdwebapp:
-      image: ${REGISTRY}signal_documentation-sdwebapp${TAG}
+      image: ${REGISTRY}signal-discovery-sdwebapp${TAG}
       build:
           context: .
 
       env_file:
           - ./.env
-      container_name: signal_documentation-sdwebapp
+      container_name: signal-discovery-sdwebapp
       restart: on-failure
-      command:
-          sh -c "python3 /usr/src/signal_documentation/src/manage.py migrate --noinput &&
-          python3 /usr/src/signal_documentation/src/manage.py collectstatic --noinput &&
-          python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/* &&
-          python3 /usr/src/signal_documentation/src/manage.py runserver 0.0.0.0:8000"
+      command: sh -c  "python3 /usr/src/signal_documentation/src/manage.py migrate --noinput &&
+                       python3 /usr/src/signal_documentation/src/manage.py collectstatic --noinput &&
+                       python3 /usr/src/signal_documentation/src/manage.py loaddata ./fixtures/* &&
+                       python3 /usr/src/signal_documentation/src/manage.py initadmin &&
+                       python3 /usr/src/signal_documentation/src/manage.py runserver 0.0.0.0:8000"
       volumes:
           - .:/usr/src/signal_documentation
       ports:
@@ -41,11 +41,11 @@ services:
   # Production service - "service", "image", and "container_name" should all contain the same
   # reference, based on the name of the service.
   sdnginx:
-      image: ${REGISTRY}signal_documentation-sdnginx${TAG}
+      image: ${REGISTRY}signal-discovery-sdnginx${TAG}
       build: ./nginx
       env_file:
           - ./.env
-      container_name: signal_documentation-sdnginx
+      container_name: signal-discovery-sdnginx
       restart: on-failure
       volumes:
           - ./src/staticfiles:/staticfiles

gunicorn/gunicorn.py

@@ -2,3 +2,5 @@
 
 bind = "0.0.0.0:8000"
 workers = multiprocessing.cpu_count() * 2 + 1
+keepalive = 120
+timeout = 300

nginx/default.conf.template

@@ -18,5 +18,11 @@ server {
         proxy_set_header Host $http_host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout 300;
+        proxy_connect_timeout 300;
+        proxy_send_timeout 300;
+        send_timeout 300;
+        keepalive_timeout 300;
+        client_max_body_size 64m;
     }
 }

src/base/management/commands/initadmin.py

@@ -0,0 +1,19 @@
+import os
+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+
+
+ADMIN_USERNAME = os.environ.get("ADMIN_USERNAME", "admin")  # Default username
+ADMIN_EMAIL = os.environ.get("ADMIN_EMAIL", "[email protected]")  # Default email
+ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "admin123")  # Default password
+
+
+class Command(BaseCommand):
+    help = "Automatically creates a superuser"
+
+    def handle(self, *args, **kwargs):
+        if not User.objects.filter(username=ADMIN_USERNAME).exists():
+            User.objects.create_superuser(ADMIN_USERNAME, ADMIN_EMAIL, ADMIN_PASSWORD)
+            self.stdout.write(self.style.SUCCESS("Superuser created successfully"))
+        else:
+            self.stdout.write(self.style.WARNING("Superuser already exists"))

src/datasources/migrations/0002_alter_datasource_display_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.7 on 2024-10-15 12:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('datasources', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasource',
+            name='display_name',
+            field=models.CharField(blank=True, max_length=128, verbose_name='Display Name'),
+        ),
+    ]

src/datasources/models.py

@@ -16,6 +16,7 @@ class DataSource(TimeStampedModel):
     display_name: models.CharField = models.CharField(
         verbose_name=_('Display Name'),
         max_length=128,
+        blank=True
     )
     description: models.TextField = models.TextField(
         verbose_name=_('Description'),

src/signal_documentation/settings.py

@@ -104,6 +104,7 @@
     'base',
     'datasources',
     'signals',
+    'signal_sets',
 ]
 
 INSTALLED_APPS: list[str] = DEFAULT_APPS + EXTERNAL_APPS + LOCAL_APPS

src/signal_documentation/urls.py

@@ -20,8 +20,12 @@
 from django.contrib import admin
 from django.urls import include, path
 
-from base.views import (BadRequestErrorView, ForbiddenErrorView,
-                        InternalServerErrorView, NotFoundErrorView)
+from base.views import (
+    BadRequestErrorView,
+    ForbiddenErrorView,
+    InternalServerErrorView,
+    NotFoundErrorView,
+)
 
 handler400 = BadRequestErrorView.as_view()
 handler403 = ForbiddenErrorView.as_view()
@@ -30,14 +34,20 @@
 
 
 urlpatterns = [
-    path("admin/", admin.site.urls),
+    path(f'{settings.MAIN_PAGE}/admin/' if settings.MAIN_PAGE else 'admin/', admin.site.urls),
+    # signal_sets
+    path(
+        f"{settings.MAIN_PAGE}/" if settings.MAIN_PAGE else "",
+        include("signal_sets.urls"),
+    ),
     # signals
     path(
-        f"{settings.MAIN_PAGE}/" if settings.MAIN_PAGE else "", include("signals.urls")
+        f"{settings.MAIN_PAGE}/signals/" if settings.MAIN_PAGE else "signals/",
+        include("signals.urls"),
     ),
     # datasources
     path(
-        f"{settings.MAIN_PAGE}/datasources" if settings.MAIN_PAGE else "datasources",
+        f"{settings.MAIN_PAGE}/datasources/" if settings.MAIN_PAGE else "datasources",
         include("datasources.urls"),
     ),
     path("__debug__/", include("debug_toolbar.urls")),

src/signal_sets/admin.py

@@ -0,0 +1,12 @@
+from django.contrib import admin
+from import_export.admin import ImportExportModelAdmin
+
+from signal_sets.resources import SignalSetResource
+from signal_sets.models import SignalSet
+
+
+@admin.register(SignalSet)
+class SignalSetAdmin(ImportExportModelAdmin):
+    list_display = ('name', 'data_description', 'maintainer_name', 'maintainer_email', 'organization')
+    search_fields = ('name', 'maintainer_name', 'maintainer_email', 'organization')
+    resource_class = SignalSetResource

src/signal_sets/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class SignalSetsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'signal_sets'

src/signal_sets/filters.py

@@ -0,0 +1,64 @@
+import logging
+
+import django_filters
+from django_filters.widgets import QueryArrayWidget
+
+from signals.models import Pathogen, GeographicScope, Geography, SeverityPyramidRung
+from signal_sets.models import SignalSet
+from datasources.models import DataSource
+
+logger = logging.getLogger(__name__)
+
+
+class SignalSetFilter(django_filters.FilterSet):
+
+    pathogens = django_filters.ModelMultipleChoiceFilter(
+        field_name="pathogens",
+        queryset=Pathogen.objects.all(),
+        widget=QueryArrayWidget,
+    )
+
+    geographic_scope = django_filters.ModelMultipleChoiceFilter(
+        field_name="geographic_scope",
+        queryset=GeographicScope.objects.all(),
+        widget=QueryArrayWidget,
+    )
+
+    available_geographies = django_filters.ModelMultipleChoiceFilter(
+        field_name="available_geographies",
+        queryset=Geography.objects.all().order_by("display_order_number"),
+        widget=QueryArrayWidget,
+    )
+
+    severity_pyramid_rungs = django_filters.ModelMultipleChoiceFilter(
+        field_name="severity_pyramid_rungs",
+        queryset=SeverityPyramidRung.objects.all(),
+        widget=QueryArrayWidget,
+    )
+
+    data_source = django_filters.ModelMultipleChoiceFilter(
+        field_name="data_source",
+        queryset=DataSource.objects.all(),
+        widget=QueryArrayWidget,
+    )
+
+    temporal_granularity = django_filters.MultipleChoiceFilter(
+        field_name="temporal_granularity",
+        choices=[
+            ("Daily", "Daily"),
+            ("Weekly", "Weekly"),
+            ("Hourly", "Hourly"),
+        ],
+        lookup_expr="icontains",
+    )
+
+    class Meta:
+        model = SignalSet
+        fields: list[str] = [
+            "pathogens",
+            "geographic_scope",
+            "available_geographies",
+            "severity_pyramid_rungs",
+            "data_source",
+            "temporal_granularity"
+        ]

src/signal_sets/forms.py

@@ -0,0 +1,63 @@
+from django import forms
+
+from signals.models import Pathogen, GeographicScope, Geography, SeverityPyramidRung
+from signal_sets.models import SignalSet
+from datasources.models import DataSource
+
+
+class SignalSetFilterForm(forms.ModelForm):
+
+    pathogens = forms.ModelChoiceField(
+        queryset=Pathogen.objects.all(), widget=forms.CheckboxSelectMultiple()
+    )
+
+    geographic_scope = forms.ModelChoiceField(
+        queryset=GeographicScope.objects.all(), widget=forms.CheckboxSelectMultiple()
+    )
+
+    available_geographies = forms.ModelChoiceField(
+        queryset=Geography.objects.all().order_by("display_order_number"),
+        widget=forms.CheckboxSelectMultiple(),
+    )
+
+    severity_pyramid_rungs = forms.ModelChoiceField(
+        queryset=SeverityPyramidRung.objects.all(),
+        widget=forms.CheckboxSelectMultiple(),
+    )
+
+    data_source = forms.ModelChoiceField(
+        queryset=DataSource.objects.all(),
+        widget=forms.CheckboxSelectMultiple(),
+    )
+
+    temporal_granularity = forms.ChoiceField(
+        choices=[
+            ("Daily", "Daily"),
+            ("Weekly", "Weekly"),
+            ("Hourly", "Hourly"),
+        ],
+        widget=forms.CheckboxSelectMultiple(),
+    )
+
+    class Meta:
+        model = SignalSet
+        fields: list[str] = [
+            "pathogens",
+            "geographic_scope",
+            "available_geographies",
+            "severity_pyramid_rungs",
+            "data_source",
+            "temporal_granularity"
+        ]
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Initialize the form.
+        """
+        super().__init__(*args, **kwargs)
+
+        # Set required attribute to False and disable helptext for all fields
+        for field_name, field in self.fields.items():
+            field.required = False
+            field.help_text = ""
+            field.label = ""