Skip to content

Commit 5b5d8c7

Browse files
saxixsaxix
authored
Merge pull request #112 from unicef/feature/settings
Feature/settings
2 parents 183d759 + 2859f27 commit 5b5d8c7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+1197
-733
lines changed

‎.pre-commit-config.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,28 +3,28 @@ repos:
33
rev: 5.13.2
44
hooks:
55
- id: isort
6-
stages: [commit]
6+
stages: [pre-commit]
77
- repo: https://github.com/ambv/black
8-
rev: 24.4.2
8+
rev: 24.10.0
99
hooks:
1010
- id: black
1111
args: [--config=pyproject.toml]
1212
exclude: "migrations|snapshots"
13-
stages: [commit]
13+
stages: [pre-commit]
1414
- repo: https://github.com/PyCQA/flake8
15-
rev: 7.1.0
15+
rev: 7.1.1
1616
hooks:
1717
- id: flake8
1818
args: [--config=.flake8]
1919

2020
additional_dependencies: [flake8-bugbear==22.9.23]
21-
stages: [ commit ]
21+
stages: [ pre-commit ]
2222
- repo: https://github.com/PyCQA/bandit
23-
rev: '1.7.9' # Update me!
23+
rev: '1.7.10' # Update me!
2424
hooks:
2525
- id: bandit
2626
args: ["-c", "bandit.yaml"]
2727
- repo: https://github.com/twisted/towncrier
28-
rev: 23.11.0
28+
rev: 24.8.0
2929
hooks:
3030
- id: towncrier-check

‎pdm.lock

Lines changed: 140 additions & 109 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ dependencies = [
4444
"setuptools>=74.1.2",
4545
"django-smart-env>=0.1.0",
4646
"jsonschema>=4.23.0",
47+
"django-celery-boost>=0.2.0",
48+
"django-svelte-jsoneditor>=0.4.2",
4749
]
4850

4951
[build-system]

‎src/hope_dedup_engine/apps/api/admin/__init__.py

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,6 @@
1-
from django.contrib import admin
2-
31
from .config import ConfigAdmin # noqa
42
from .deduplicationset import DeduplicationSetAdmin # noqa
53
from .duplicate import DuplicateAdmin # noqa
64
from .hdetoken import HDETokenAdmin # noqa
75
from .image import ImageAdmin # noqa
8-
9-
admin.site.site_header = "HOPE Dedup Engine"
10-
admin.site.site_title = "HOPE Deduplication Admin"
11-
admin.site.index_title = "Welcome to the HOPE Deduplication Engine Admin"
6+
from .jobs import DedupJob # noqa

‎src/hope_dedup_engine/apps/api/admin/config.py

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,44 @@
22
from typing import Any
33

44
from django.contrib import messages
5-
from django.contrib.admin import ModelAdmin, register
5+
from django.contrib.admin import ModelAdmin, register, site
6+
from django.core.exceptions import ValidationError
7+
from django.db import models
68
from django.http import HttpRequest, HttpResponse
79
from django.shortcuts import redirect, render
810
from django.urls import path, reverse
911

12+
from admin_extra_buttons.api import button
13+
from admin_extra_buttons.mixins import ExtraButtonsMixin
14+
from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget
15+
16+
from hope_dedup_engine.apps.api.forms import EditSchemaForm
1017
from hope_dedup_engine.apps.api.models import Config
18+
from hope_dedup_engine.apps.api.validators import DefaultValidatingValidator
19+
from hope_dedup_engine.utils.security import is_root
20+
from src.hope_dedup_engine.apps.api.utils.shema_manager import SchemaManager
1121

1222

1323
@register(Config)
14-
class ConfigAdmin(ModelAdmin):
24+
class ConfigAdmin(ExtraButtonsMixin, ModelAdmin):
1525
list_display = ("name", "settings")
26+
change_list_template = "admin/api/config/change_list.html"
27+
28+
formfield_overrides = {
29+
models.JSONField: {
30+
"widget": SvelteJSONEditorWidget,
31+
}
32+
}
33+
34+
def get_changeform_initial_data(self, request: HttpRequest) -> dict[str, str]:
35+
initial_data = super().get_changeform_initial_data(request)
36+
initial_data["settings"] = {}
37+
try:
38+
schema = SchemaManager.get_or_create()
39+
DefaultValidatingValidator(schema).validate(initial_data["settings"])
40+
except ValidationError as e:
41+
self.message_user(request, e.message, level=messages.ERROR)
42+
return initial_data
1643

1744
def get_urls(self):
1845
urls = super().get_urls()
@@ -22,6 +49,11 @@ def get_urls(self):
2249
self.admin_site.admin_view(self.confirm_save),
2350
name="confirm_save_config",
2451
),
52+
path(
53+
"change-settings-schema/",
54+
self.admin_site.admin_view(self.change_settings_schema),
55+
name="change_settings_schema",
56+
),
2557
]
2658
return custom_urls + urls
2759

@@ -38,7 +70,7 @@ def response_change(self, request: HttpRequest, obj: Any) -> HttpResponse:
3870
return redirect(confirm_url)
3971
return super().response_change(request, obj)
4072

41-
def confirm_save(self, request, object_id): # pragma: no cover
73+
def confirm_save(self, request, object_id) -> HttpResponse: # pragma: no cover
4274
obj = self.get_object(request, object_id)
4375
if request.method == "POST":
4476
form_data = request.session.get("unsaved_data", None)
@@ -59,3 +91,41 @@ def confirm_save(self, request, object_id): # pragma: no cover
5991
"form_data": request.session.get("unsaved_data"),
6092
},
6193
)
94+
95+
@button(permission=is_root)
96+
def change_settings_schema(
97+
self, request: HttpRequest
98+
) -> HttpResponse: # pragma: no cover
99+
context = {
100+
"opts": self.model._meta,
101+
"site_header": site.site_header,
102+
"title": "Change settings shema",
103+
"trail_label": "Settings schema",
104+
"has_view_permission": self.has_view_permission(request),
105+
}
106+
107+
if request.method == "POST":
108+
form = EditSchemaForm(request.POST)
109+
if form.is_valid():
110+
try:
111+
SchemaManager.save(form.cleaned_data["schema"])
112+
except ValidationError as e:
113+
self.message_user(request, e.message, level=messages.ERROR)
114+
else:
115+
self.message_user(request, "Schema has been updated.")
116+
return redirect(reverse("admin:api_config_changelist"))
117+
else:
118+
try:
119+
form = EditSchemaForm(initial={"schema": SchemaManager.get_or_create()})
120+
except ValidationError as e:
121+
self.message_user(request, e.message, level=messages.ERROR)
122+
return redirect(reverse("admin:api_config_changelist"))
123+
124+
return render(
125+
request,
126+
"admin/api/config/change_settings_schema.html",
127+
{
128+
"form": form,
129+
**context,
130+
},
131+
)

‎src/hope_dedup_engine/apps/api/admin/deduplicationset.py

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,27 @@
1-
from uuid import UUID
2-
31
from django.contrib.admin import ModelAdmin, register
4-
from django.http import HttpRequest, HttpResponseRedirect
5-
from django.urls import reverse
62

7-
from admin_extra_buttons.api import button
8-
from admin_extra_buttons.mixins import ExtraButtonsMixin
93
from adminfilters.dates import DateRangeFilter
104
from adminfilters.filters import ChoicesFieldComboFilter, DjangoLookupFilter
115
from adminfilters.mixin import AdminFiltersMixin
126

137
from hope_dedup_engine.apps.api.models import DeduplicationSet
14-
from hope_dedup_engine.apps.api.utils.process import start_processing
15-
from hope_dedup_engine.utils.security import can_reprocess
168

179

1810
@register(DeduplicationSet)
19-
class DeduplicationSetAdmin(AdminFiltersMixin, ExtraButtonsMixin, ModelAdmin):
11+
class DeduplicationSetAdmin(AdminFiltersMixin, ModelAdmin):
2012
list_display = (
2113
"id",
2214
"name",
2315
"reference_pk",
24-
"state_value",
16+
"state",
2517
"config",
2618
"created_at",
2719
"updated_at",
2820
"deleted",
2921
)
3022
readonly_fields = (
3123
"id",
32-
"state_value",
24+
"state",
3325
"external_system",
3426
"created_at",
3527
"created_by",
@@ -39,22 +31,11 @@ class DeduplicationSetAdmin(AdminFiltersMixin, ExtraButtonsMixin, ModelAdmin):
3931
)
4032
search_fields = ("name",)
4133
list_filter = (
42-
("state_value", ChoicesFieldComboFilter),
34+
("state", ChoicesFieldComboFilter),
4335
("created_at", DateRangeFilter),
4436
("updated_at", DateRangeFilter),
4537
DjangoLookupFilter,
4638
)
47-
change_form_template = "admin/api/deduplicationset/change_form.html"
4839

4940
def has_add_permission(self, request):
5041
return False
51-
52-
@button(permission=can_reprocess)
53-
def process(self, request: HttpRequest, pk: UUID) -> HttpResponseRedirect:
54-
obj = self.get_object(request, pk)
55-
start_processing(obj)
56-
self.message_user(
57-
request,
58-
f"Processing for deduplication set '{obj}' has been started.",
59-
)
60-
return HttpResponseRedirect(reverse("admin:api_deduplicationset_changelist"))

‎src/hope_dedup_engine/apps/api/admin/jobs.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from django.contrib import admin
2+
3+
from django_celery_boost.admin import CeleryTaskModelAdmin
4+
5+
from hope_dedup_engine.apps.api.models.jobs import DedupJob
6+
7+
8+
@admin.register(DedupJob)
9+
class DedupJobAdmin(CeleryTaskModelAdmin):
10+
list_display = ["deduplication_set_id", "progress"]

‎src/hope_dedup_engine/apps/api/config_settings_schema.json

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"type": "object",
3+
"properties": {
4+
"detection": {
5+
"type": "object",
6+
"properties": {
7+
"confidence": {
8+
"type": "number",
9+
"exclusiveMinimum": 0,
10+
"maximum": 1,
11+
"default": "constance.config.FACE_DETECTION_CONFIDENCE"
12+
}
13+
},
14+
"default": {},
15+
"required": [
16+
"confidence"
17+
]
18+
},
19+
"recognition": {
20+
"type": "object",
21+
"properties": {
22+
"num_jitters": {
23+
"type": "integer",
24+
"minimum": 1,
25+
"default": "constance.config.FACE_ENCODINGS_NUM_JITTERS"
26+
},
27+
"model": {
28+
"type": "string",
29+
"enum": [
30+
"small",
31+
"large"
32+
],
33+
"default": "constance.config.FACE_ENCODINGS_MODEL"
34+
},
35+
"preprocessors": {
36+
"type": "array",
37+
"items": {
38+
"type": "string",
39+
"enum": []
40+
},
41+
"uniquItems": true,
42+
"default": []
43+
}
44+
},
45+
"default": {},
46+
"required": [
47+
"num_jitters",
48+
"model"
49+
]
50+
},
51+
"duplicates": {
52+
"type": "object",
53+
"properties": {
54+
"tolerance": {
55+
"type": "number",
56+
"exclusiveMinimum": 0,
57+
"maximum": 1,
58+
"default": "constance.config.FACE_DISTANCE_THRESHOLD"
59+
}
60+
},
61+
"default": {},
62+
"required": [
63+
"tolerance"
64+
]
65+
}
66+
},
67+
"required": [
68+
"detection",
69+
"recognition",
70+
"duplicates"
71+
]
72+
}

‎src/hope_dedup_engine/apps/api/deduplication/adapters.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
from collections.abc import Generator
2-
from typing import Any
1+
from collections.abc import Callable, Generator
32

3+
from hope_dedup_engine.apps.api.deduplication.config import ConfigDefaults
44
from hope_dedup_engine.apps.api.deduplication.registry import DuplicateKeyPair
55
from hope_dedup_engine.apps.api.models import DeduplicationSet
66
from hope_dedup_engine.apps.faces.services.duplication_detector import (
@@ -12,23 +12,28 @@ class DuplicateFaceFinder:
1212
weight = 1
1313

1414
def __init__(self, deduplication_set: DeduplicationSet):
15+
self.tracker = None
1516
self.deduplication_set = deduplication_set
1617

17-
def run(self) -> Generator[DuplicateKeyPair, None, None]:
18+
def run(
19+
self, tracker: Callable[[int], None] | None = None
20+
) -> Generator[DuplicateKeyPair, None, None]:
1821
filename_to_reference_pk = {
1922
filename: reference_pk
2023
for reference_pk, filename in self.deduplication_set.image_set.values_list(
2124
"reference_pk", "filename"
2225
)
2326
}
24-
ds_config: dict[str, Any] = (
25-
self.deduplication_set.config and self.deduplication_set.config.settings
26-
) or {}
27+
cfg = ConfigDefaults()
28+
if self.deduplication_set.config:
29+
cfg.apply_config_overrides(self.deduplication_set.config.settings)
2730
# ignored key pairs are not handled correctly in DuplicationDetector
2831
detector = DuplicationDetector(
29-
tuple[str](filename_to_reference_pk.keys()), ds_config
32+
tuple[str](filename_to_reference_pk.keys()), cfg=cfg
3033
)
31-
for first_filename, second_filename, distance in detector.find_duplicates():
34+
for first_filename, second_filename, distance in detector.find_duplicates(
35+
tracker
36+
):
3237
yield filename_to_reference_pk[first_filename], filename_to_reference_pk[
3338
second_filename
3439
], 1 - distance

0 commit comments

Comments
 (0)