Merge pull request #4613 from open-formulieren/feature/4542-email-ver…

…ification

Add endpoints & models to start email verification

docs/conf.py

@@ -113,6 +113,7 @@
     r"https://www\.jccsoftware\.nl/.*",  # looks like the requests user agent is blocked...
     r"https://hub\.docker\.com/r/.*",  # HTTP 429, presumably docker hub is blocking/limiting Github CI runners
     r"https://stackoverflow\.com/.*",  # SO 403s when running on github actions :/
+    r"https://sequencediagram\.org/index\.html",  # anchor are not server side
 ]
 
 extlinks = {

docs/developers/backend/core/email-verification.rst

@@ -0,0 +1,75 @@
+.. _developers_backend_core_email_verification:
+
+========================
+Core: email verification
+========================
+
+Form designers can optionally require email verification for email components. Email
+verification is disabled by default. When it's enabled, the following process takes
+place.
+
+Sequence diagram
+================
+
+The `sequence diagram`_ describes the flow. The user is the person filling out the
+submission, the SDK is the UI exposed to the user and the backend represents the API
+called by the SDK.
+
+.. image:: _assets/email_verification.png
+    :width: 100%
+
+
+.. _sequence diagram: https://sequencediagram.org/index.html#initialData=C4S2BsFMAIFEFsCGJzQG6QE4gGYgMaKgD2AdtDuMQO4BQti+wxm0AqgM5a0AOimofCD6lg0AMoARANK9+g4YlHQAQowDWkUgBN6nLAFoAfFOkAuOKKzRISFNETbtmSBw4AaaPnAF10AEQY2DgAnv7QAEYArsDMpLSmxqYWAPI8WtDwxNqI4LSkxMAw2ADmABZixDgSMhYAstm50NogHDzgiCEcNnaojs6u3UraXmR4mPCRMXG0+phJtdAAwj74fv74YyAT4SDkWTl5iUZqa1raFuLA8uhYuAREIGS0p5o6xgAUAIwAlHOX5x6yFQ1DAZVG2kgCRkAB4DAZXudUrJjsloAAlc7WTaQ6B7HgxCggSDgEZ7TKNPJzBbmSxFVg4mA4TDESa2YHQ6TGRE6CxLMqQNa3YIPEjkRmcuEIjRI6ApaTQFgBPZoXIgEaM8JYFmYfKFYogcqVapog5NbzELjdXBy6SeElcZqtdqdGyYHVAA
+
+**Description**
+
+1. The verification process starts with an explicit confirm action by the user (clicking
+   the confirm button in the modal rendered by the SDK).
+2. This triggers an API call that creates the verification request in the backend. It
+   captures (and validates) the submission, the email address and the formio component
+   key of the email component. Multiple email components can be present in the form, each
+   requiring verification. A code is generated and stored in the database, and an email
+   is sent containing the generated code. The code may not be exposed via the API.
+3. Upon successful start, the modal confirms that an email is underway (repeating the
+   email address) and displays an input field for the verification code.
+4. The user can enter the code and click a "confirm" button to verify the code. The API
+   endpoint receives submission, component key, email address and code. On success, the
+   backend updates the state to ``verified: true``.
+5. The modal closes and the user can continue with the rest of the form.
+6. Once the step is submitted, backend validation checks that the values of email
+   components have a matching verified status, if not, validation errors are displayed.
+
+Design decisions made
+=====================
+
+**No unique constraints**
+
+No unique constraints to enforce the combination of email, component key and submission
+to be unique. The reason for this is that it's possible an email does not arrive, and
+in those situations, users must be able to request another verification, which ideally
+has a different code.
+
+Additionally, doing de-duplication checks complicates the implementation, and the cost
+of allowing duplicates is low.
+
+**Multiple email addresses for the same component are allowed**
+
+Multiple (different) addresses are allowed for two reasons:
+
+* the component may have ``{"multiple": True}``, which associates multiple values with
+  a single component/field already. Each address needs to be verified to proceed.
+* the user may make a mistake, realize (because of the verification) that they made a
+  typo, and correct their input. This naturally leads to multiple values for the same
+  component, of which only one is actually relevant.
+
+**Verification codes are not globally unique**
+
+We randomly generate 6-character verification codes (excluding confusing characters),
+which has a chance of producing the same code multiple times. A code is used in the
+context of a single submission and a single form component/field, they cannot be used
+in other submissions belonging to someone else.
+
+**Brute forcing verification codes**
+
+The default rate limits and permission checks should mitigate this. If needed,
+additional protections can be added in the future.

docs/developers/backend/index.rst

@@ -45,6 +45,7 @@ Core
 
     core/index
     core/formio
+    core/email-verification
     core/submissions
     core/submission-renderer
     core/variables

src/openapi.yaml

@@ -5334,6 +5334,52 @@ paths:
               $ref: '#/components/headers/X-Is-Form-Designer'
             Content-Language:
               $ref: '#/components/headers/Content-Language'
+  /api/v2/submissions/email-verifications:
+    post:
+      operationId: submissions_email_verifications_create
+      description: |-
+        Create an email verification resource to start the verification process. A verification e-mail will be scheduled and sent to the provided email address, containing the verification code to use during verification.
+
+        Validations check that the provided component key is present in the form of the submission and that it actually is an `email` component.
+      summary: Start email verification
+      tags:
+      - submissions
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/EmailVerification'
+          application/x-www-form-urlencoded:
+            schema:
+              $ref: '#/components/schemas/EmailVerification'
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/EmailVerification'
+        required: true
+      security:
+      - cookieAuth: []
+      responses:
+        '201':
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/EmailVerification'
+          description: ''
+          headers:
+            X-Session-Expires-In:
+              $ref: '#/components/headers/X-Session-Expires-In'
+            X-CSRFToken:
+              $ref: '#/components/headers/X-CSRFToken'
+            X-Is-Form-Designer:
+              $ref: '#/components/headers/X-Is-Form-Designer'
+            Content-Language:
+              $ref: '#/components/headers/Content-Language'
+      parameters:
+      - in: header
+        name: X-CSRFToken
+        schema:
+          type: string
+        required: true
   /api/v2/submissions/files/{uuid}:
     get:
       operationId: submissions_files_retrieve
@@ -7289,6 +7335,27 @@ components:
       - form
       type: string
       description: '* `form` - form'
+    EmailVerification:
+      type: object
+      properties:
+        submission:
+          type: string
+          format: uri
+          writeOnly: true
+        componentKey:
+          type: string
+          description: Key of the email component in the submission's form.
+          pattern: ^(\w|\w[\w.\-]*\w)$
+        email:
+          type: string
+          format: email
+          title: Email address
+          description: The email address that is being verified.
+          maxLength: 254
+      required:
+      - componentKey
+      - email
+      - submission
     Exception:
       type: object
       description: |-

src/openforms/fixtures/default_admin_index.json

@@ -94,6 +94,10 @@
                 "payments",
                 "submissionpayment"
             ],
+            [
+                "submissions",
+                "emailverification"
+            ],
             [
                 "submissions",
                 "submission"

src/openforms/submissions/admin.py

@@ -26,6 +26,7 @@
 from .constants import IMAGE_COMPONENTS, PostSubmissionEvents, RegistrationStatuses
 from .exports import ExportFileTypes, export_submissions
 from .models import (
+    EmailVerification,
     Submission,
     SubmissionFileAttachment,
     SubmissionReport,
@@ -579,3 +580,18 @@ def form_key_display(self, obj):
 
     def has_add_permission(self, request, obj=None):
         return False
+
+
+@admin.register(EmailVerification)
+class EmailVerificationAdmin(admin.ModelAdmin):
+    list_display = ("email", "submission", "component_key", "is_verified")
+    search_fields = (
+        "email",
+        "component_key",
+        "submission__uuid",
+        "submission__public_registration_reference",
+    )
+
+    @admin.display(description=_("is verified"), boolean=True)
+    def is_verified(self, obj: EmailVerification) -> bool:
+        return obj.verified_on is not None

src/openforms/submissions/api/serializers.py

@@ -1,8 +1,10 @@
 import logging
 from dataclasses import dataclass
 from datetime import timedelta
+from typing import TypedDict
 
 from django.conf import settings
+from django.contrib.sessions.backends.base import SessionBase
 from django.db import transaction
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
@@ -26,9 +28,9 @@
 from openforms.forms.validators import validate_not_deleted
 from openforms.utils.urls import build_absolute_uri
 
-from ..constants import ProcessingResults, ProcessingStatuses
+from ..constants import SUBMISSIONS_SESSION_KEY, ProcessingResults, ProcessingStatuses
 from ..form_logic import check_submission_logic, evaluate_form_logic
-from ..models import Submission, SubmissionStep
+from ..models import EmailVerification, Submission, SubmissionStep
 from ..tokens import submission_resume_token_generator
 from ..utils import get_report_download_url
 from .fields import (
@@ -564,3 +566,63 @@ class SubmissionReportUrlSerializer(serializers.Serializer):
     @extend_schema_field(OpenApiTypes.URI)
     def get_report_download_url(self, submission) -> str:
         return get_report_download_url(self.context["request"], submission.report)
+
+
+class EmailVerificationData(TypedDict):
+    submission: Submission
+    component_key: str
+    email: str
+
+
+class EmailVerificationSerializer(serializers.ModelSerializer):
+    submission = serializers.HyperlinkedRelatedField(
+        view_name="api:submission-detail",
+        lookup_field="uuid",
+        queryset=Submission.objects.none(),  # Overridden dynamically
+        label=_("Submission"),
+        write_only=True,
+        required=True,
+    )
+
+    class Meta:
+        model = EmailVerification
+        fields = ("submission", "component_key", "email")
+
+    def get_fields(self):
+        fields = super().get_fields()
+        view = self.context.get("view")
+        if getattr(view, "swagger_fake_view", False):
+            return fields
+
+        session: SessionBase = self.context["request"].session
+
+        submission_field = fields["submission"]
+        assert isinstance(
+            submission_field, serializers.HyperlinkedRelatedField
+        )  # for the type checker
+        submission_field.queryset = Submission.objects.filter(
+            completed_on=None, uuid__in=session.get(SUBMISSIONS_SESSION_KEY, [])
+        )
+        return fields
+
+    def validate(self, attrs: EmailVerificationData) -> EmailVerificationData:
+        # validate that the component key is present in the submissoin form *and* points
+        # to an email component
+        config_wrapper = attrs["submission"].total_configuration_wrapper
+        key = attrs["component_key"]
+        try:
+            component = config_wrapper.component_map[key]
+            key_valid = component["type"] == "email"
+        except KeyError:
+            key_valid = False
+        if not key_valid:
+            raise serializers.ValidationError(
+                {
+                    "component_key": _(
+                        "The key '{key}' does not point to an email component in the form."
+                    ).format(key=key),
+                },
+                code="not_found",
+            )
+
+        return attrs

src/openforms/submissions/api/urls.py

@@ -1,6 +1,6 @@
 from django.urls import path
 
-from .views import TemporaryFileView
+from .views import EmailVerificationCreateView, TemporaryFileView
 
 app_name = "submissions"
 
@@ -10,4 +10,9 @@
         TemporaryFileView.as_view(),
         name="temporary-file",
     ),
+    path(
+        "email-verifications",
+        EmailVerificationCreateView.as_view(),
+        name="email-verification",
+    ),
 ]

src/openforms/submissions/api/views.py

@@ -1,17 +1,20 @@
 from django.conf import settings
+from django.db import transaction
 from django.utils.translation import gettext_lazy as _
 
 from django_sendfile import sendfile
 from djangorestframework_camel_case.render import CamelCaseJSONRenderer
 from drf_spectacular.utils import extend_schema, extend_schema_view
-from rest_framework.generics import DestroyAPIView
+from rest_framework.generics import CreateAPIView, DestroyAPIView
 
 from openforms.api.authentication import AnonCSRFSessionAuthentication
 from openforms.api.serializers import ExceptionSerializer
+from openforms.submissions.models.email_verification import EmailVerification
 
 from ..models import TemporaryFileUpload
-from .permissions import OwnsTemporaryUploadPermission
+from .permissions import AnyActiveSubmissionPermission, OwnsTemporaryUploadPermission
 from .renderers import FileRenderer
+from .serializers import EmailVerificationSerializer
 
 
 @extend_schema(
@@ -68,3 +71,28 @@ def perform_destroy(self, instance):
         # saved when trying to access the next form step
         instance.attachments.all().delete()
         instance.delete()
+
+
+@extend_schema(
+    summary=_("Start email verification"),
+    description=_(
+        "Create an email verification resource to start the verification process. "
+        "A verification e-mail will be scheduled and sent to the provided email "
+        "address, containing the verification code to use during verification.\n\n"
+        "Validations check that the provided component key is present in the form of "
+        "the submission and that it actually is an `email` component."
+    ),
+)
+class EmailVerificationCreateView(CreateAPIView):
+    authentication_classes = (AnonCSRFSessionAuthentication,)
+    permission_classes = (AnyActiveSubmissionPermission,)
+    serializer_class = EmailVerificationSerializer
+    # using none to prevent potential accidents - the view only needs to know about
+    # queryset.model anyway
+    queryset = EmailVerification.objects.none()
+
+    def perform_create(self, serializer):
+        super().perform_create(serializer)
+        verification = serializer.instance
+        assert isinstance(verification, EmailVerification)
+        transaction.on_commit(verification.send_email)