Merge pull request #112 from unicef/feature/settings

Feature/settings

.pre-commit-config.yaml

@@ -3,28 +3,28 @@ repos:
     rev: 5.13.2
     hooks:
       - id: isort
-        stages: [commit]
+        stages: [pre-commit]
   - repo: https://github.com/ambv/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
         args: [--config=pyproject.toml]
         exclude: "migrations|snapshots"
-        stages: [commit]
+        stages: [pre-commit]
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: [--config=.flake8]
 
         additional_dependencies: [flake8-bugbear==22.9.23]
-        stages: [ commit ]
+        stages: [ pre-commit ]
   - repo: https://github.com/PyCQA/bandit
-    rev: '1.7.9' # Update me!
+    rev: '1.7.10' # Update me!
     hooks:
       - id: bandit
         args: ["-c", "bandit.yaml"]
   - repo: https://github.com/twisted/towncrier
-    rev: 23.11.0
+    rev: 24.8.0
     hooks:
       - id: towncrier-check

pyproject.toml

@@ -44,6 +44,8 @@ dependencies = [
     "setuptools>=74.1.2",
     "django-smart-env>=0.1.0",
     "jsonschema>=4.23.0",
+    "django-celery-boost>=0.2.0",
+    "django-svelte-jsoneditor>=0.4.2",
 ]
 
 [build-system]

src/hope_dedup_engine/apps/api/admin/__init__.py

@@ -1,11 +1,6 @@
-from django.contrib import admin
-
 from .config import ConfigAdmin  # noqa
 from .deduplicationset import DeduplicationSetAdmin  # noqa
 from .duplicate import DuplicateAdmin  # noqa
 from .hdetoken import HDETokenAdmin  # noqa
 from .image import ImageAdmin  # noqa
-
-admin.site.site_header = "HOPE Dedup Engine"
-admin.site.site_title = "HOPE Deduplication Admin"
-admin.site.index_title = "Welcome to the HOPE Deduplication Engine Admin"
+from .jobs import DedupJob  # noqa

src/hope_dedup_engine/apps/api/admin/config.py

@@ -2,17 +2,44 @@
 from typing import Any
 
 from django.contrib import messages
-from django.contrib.admin import ModelAdmin, register
+from django.contrib.admin import ModelAdmin, register, site
+from django.core.exceptions import ValidationError
+from django.db import models
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import redirect, render
 from django.urls import path, reverse
 
+from admin_extra_buttons.api import button
+from admin_extra_buttons.mixins import ExtraButtonsMixin
+from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
+
+from hope_dedup_engine.apps.api.forms import EditSchemaForm
 from hope_dedup_engine.apps.api.models import Config
+from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
+from hope_dedup_engine.utils.security import is_root
+from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
 
 
 @register(Config)
-class ConfigAdmin(ModelAdmin):
+class ConfigAdmin(ExtraButtonsMixin, ModelAdmin):
     list_display = ("name", "settings")
+    change_list_template = "admin/api/config/change_list.html"
+
+    formfield_overrides = {
+        models.JSONField: {
+            "widget": SvelteJSONEditorWidget,
+        }
+    }
+
+    def get_changeform_initial_data(self, request: HttpRequest) -> dict[str, str]:
+        initial_data = super().get_changeform_initial_data(request)
+        initial_data["settings"] = {}
+        try:
+            schema = SchemaManager.get_or_create()
+            DefaultValidatingValidator(schema).validate(initial_data["settings"])
+        except ValidationError as e:
+            self.message_user(request, e.message, level=messages.ERROR)
+        return initial_data
 
     def get_urls(self):
         urls = super().get_urls()
@@ -22,6 +49,11 @@ def get_urls(self):
                 self.admin_site.admin_view(self.confirm_save),
                 name="confirm_save_config",
             ),
+            path(
+                "change-settings-schema/",
+                self.admin_site.admin_view(self.change_settings_schema),
+                name="change_settings_schema",
+            ),
         ]
         return custom_urls + urls
 
@@ -38,7 +70,7 @@ def response_change(self, request: HttpRequest, obj: Any) -> HttpResponse:
             return redirect(confirm_url)
         return super().response_change(request, obj)
 
-    def confirm_save(self, request, object_id):  # pragma: no cover
+    def confirm_save(self, request, object_id) -> HttpResponse:  # pragma: no cover
         obj = self.get_object(request, object_id)
         if request.method == "POST":
             form_data = request.session.get("unsaved_data", None)
@@ -59,3 +91,41 @@ def confirm_save(self, request, object_id):  # pragma: no cover
                 "form_data": request.session.get("unsaved_data"),
             },
         )
+
+    @button(permission=is_root)
+    def change_settings_schema(
+        self, request: HttpRequest
+    ) -> HttpResponse:  # pragma: no cover
+        context = {
+            "opts": self.model._meta,
+            "site_header": site.site_header,
+            "title": "Change settings shema",
+            "trail_label": "Settings schema",
+            "has_view_permission": self.has_view_permission(request),
+        }
+
+        if request.method == "POST":
+            form = EditSchemaForm(request.POST)
+            if form.is_valid():
+                try:
+                    SchemaManager.save(form.cleaned_data["schema"])
+                except ValidationError as e:
+                    self.message_user(request, e.message, level=messages.ERROR)
+                else:
+                    self.message_user(request, "Schema has been updated.")
+                    return redirect(reverse("admin:api_config_changelist"))
+        else:
+            try:
+                form = EditSchemaForm(initial={"schema": SchemaManager.get_or_create()})
+            except ValidationError as e:
+                self.message_user(request, e.message, level=messages.ERROR)
+                return redirect(reverse("admin:api_config_changelist"))
+
+        return render(
+            request,
+            "admin/api/config/change_settings_schema.html",
+            {
+                "form": form,
+                **context,
+            },
+        )

src/hope_dedup_engine/apps/api/admin/deduplicationset.py

@@ -1,35 +1,27 @@
-from uuid import UUID
-
 from django.contrib.admin import ModelAdmin, register
-from django.http import HttpRequest, HttpResponseRedirect
-from django.urls import reverse
 
-from admin_extra_buttons.api import button
-from admin_extra_buttons.mixins import ExtraButtonsMixin
 from adminfilters.dates import DateRangeFilter
 from adminfilters.filters import ChoicesFieldComboFilter, DjangoLookupFilter
 from adminfilters.mixin import AdminFiltersMixin
 
 from hope_dedup_engine.apps.api.models import DeduplicationSet
-from hope_dedup_engine.apps.api.utils.process import start_processing
-from hope_dedup_engine.utils.security import can_reprocess
 
 
 @register(DeduplicationSet)
-class DeduplicationSetAdmin(AdminFiltersMixin, ExtraButtonsMixin, ModelAdmin):
+class DeduplicationSetAdmin(AdminFiltersMixin, ModelAdmin):
     list_display = (
         "id",
         "name",
         "reference_pk",
-        "state_value",
+        "state",
         "config",
         "created_at",
         "updated_at",
         "deleted",
     )
     readonly_fields = (
         "id",
-        "state_value",
+        "state",
         "external_system",
         "created_at",
         "created_by",
@@ -39,22 +31,11 @@ class DeduplicationSetAdmin(AdminFiltersMixin, ExtraButtonsMixin, ModelAdmin):
     )
     search_fields = ("name",)
     list_filter = (
-        ("state_value", ChoicesFieldComboFilter),
+        ("state", ChoicesFieldComboFilter),
         ("created_at", DateRangeFilter),
         ("updated_at", DateRangeFilter),
         DjangoLookupFilter,
     )
-    change_form_template = "admin/api/deduplicationset/change_form.html"
 
     def has_add_permission(self, request):
         return False
-
-    @button(permission=can_reprocess)
-    def process(self, request: HttpRequest, pk: UUID) -> HttpResponseRedirect:
-        obj = self.get_object(request, pk)
-        start_processing(obj)
-        self.message_user(
-            request,
-            f"Processing for deduplication set '{obj}' has been started.",
-        )
-        return HttpResponseRedirect(reverse("admin:api_deduplicationset_changelist"))

src/hope_dedup_engine/apps/api/admin/jobs.py

@@ -0,0 +1,10 @@
+from django.contrib import admin
+
+from django_celery_boost.admin import CeleryTaskModelAdmin
+
+from hope_dedup_engine.apps.api.models.jobs import DedupJob
+
+
+@admin.register(DedupJob)
+class DedupJobAdmin(CeleryTaskModelAdmin):
+    list_display = ["deduplication_set_id", "progress"]

src/hope_dedup_engine/apps/api/config_settings_schema.json

@@ -0,0 +1,72 @@
+{
+    "type": "object",
+    "properties": {
+        "detection": {
+            "type": "object",
+            "properties": {
+                "confidence": {
+                    "type": "number",
+                    "exclusiveMinimum": 0,
+                    "maximum": 1,
+                    "default": "constance.config.FACE_DETECTION_CONFIDENCE"
+                }
+            },
+            "default": {},
+            "required": [
+                "confidence"
+            ]
+        },
+        "recognition": {
+            "type": "object",
+            "properties": {
+                "num_jitters": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "default": "constance.config.FACE_ENCODINGS_NUM_JITTERS"
+                },
+                "model": {
+                    "type": "string",
+                    "enum": [
+                        "small",
+                        "large"
+                    ],
+                    "default": "constance.config.FACE_ENCODINGS_MODEL"
+                },
+                "preprocessors": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "enum": []
+                    },
+                    "uniquItems": true,
+                    "default": []
+                }
+            },
+            "default": {},
+            "required": [
+                "num_jitters",
+                "model"
+            ]
+        },
+        "duplicates": {
+            "type": "object",
+            "properties": {
+                "tolerance": {
+                    "type": "number",
+                    "exclusiveMinimum": 0,
+                    "maximum": 1,
+                    "default": "constance.config.FACE_DISTANCE_THRESHOLD"
+                }
+            },
+            "default": {},
+            "required": [
+                "tolerance"
+            ]
+        }
+    },
+    "required": [
+        "detection",
+        "recognition",
+        "duplicates"
+    ]
+}

src/hope_dedup_engine/apps/api/deduplication/adapters.py

@@ -1,6 +1,6 @@
-from collections.abc import Generator
-from typing import Any
+from collections.abc import Callable, Generator
 
+from hope_dedup_engine.apps.api.deduplication.config import ConfigDefaults
 from hope_dedup_engine.apps.api.deduplication.registry import DuplicateKeyPair
 from hope_dedup_engine.apps.api.models import DeduplicationSet
 from hope_dedup_engine.apps.faces.services.duplication_detector import (
@@ -12,23 +12,28 @@ class DuplicateFaceFinder:
     weight = 1
 
     def __init__(self, deduplication_set: DeduplicationSet):
+        self.tracker = None
         self.deduplication_set = deduplication_set
 
-    def run(self) -> Generator[DuplicateKeyPair, None, None]:
+    def run(
+        self, tracker: Callable[[int], None] | None = None
+    ) -> Generator[DuplicateKeyPair, None, None]:
         filename_to_reference_pk = {
             filename: reference_pk
             for reference_pk, filename in self.deduplication_set.image_set.values_list(
                 "reference_pk", "filename"
             )
         }
-        ds_config: dict[str, Any] = (
-            self.deduplication_set.config and self.deduplication_set.config.settings
-        ) or {}
+        cfg = ConfigDefaults()
+        if self.deduplication_set.config:
+            cfg.apply_config_overrides(self.deduplication_set.config.settings)
         # ignored key pairs are not handled correctly in DuplicationDetector
         detector = DuplicationDetector(
-            tuple[str](filename_to_reference_pk.keys()), ds_config
+            tuple[str](filename_to_reference_pk.keys()), cfg=cfg
         )
-        for first_filename, second_filename, distance in detector.find_duplicates():
+        for first_filename, second_filename, distance in detector.find_duplicates(
+            tracker
+        ):
             yield filename_to_reference_pk[first_filename], filename_to_reference_pk[
                 second_filename
             ], 1 - distance