Add essential files

pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
     "flower>=2.0.1",
     "setuptools>=74.1.2",
     "django-smart-env>=0.1.0",
+    "django-celery-boost>=0.2.0",
 ]
 
 [build-system]

src/hope_dedup_engine/apps/api/admin/__init__.py

@@ -5,6 +5,7 @@
 from .duplicate import DuplicateAdmin  # noqa
 from .hdetoken import HDETokenAdmin  # noqa
 from .image import ImageAdmin  # noqa
+from .jobs import DedupJob  # noqa
 
 admin.site.site_header = "HOPE Dedup Engine"
 admin.site.site_title = "HOPE Deduplication Admin"

src/hope_dedup_engine/apps/api/admin/jobs.py

@@ -0,0 +1,10 @@
+from django.contrib import admin
+
+from django_celery_boost.admin import CeleryTaskModelAdmin
+
+from hope_dedup_engine.apps.api.models.jobs import DedupJob
+
+
+@admin.register(DedupJob)
+class AsyncJobAdmin(CeleryTaskModelAdmin):
+    pass

src/hope_dedup_engine/apps/api/migrations/0009_alter_config_face_distance_threshold.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.7 on 2024-10-30 12:32
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0008_ignoredfilenamepair"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="config",
+            name="face_distance_threshold",
+            field=models.FloatField(
+                null=True,
+                validators=[
+                    django.core.validators.MinValueValidator(0.1),
+                    django.core.validators.MaxValueValidator(1.0),
+                ],
+            ),
+        ),
+    ]

src/hope_dedup_engine/apps/api/migrations/0010_dedupjob.py

@@ -0,0 +1,81 @@
+# Generated by Django 5.0.7 on 2024-10-30 12:45
+
+import concurrency.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0009_alter_config_face_distance_threshold"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DedupJob",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "version",
+                    concurrency.fields.AutoIncVersionField(
+                        default=0, help_text="record revision number"
+                    ),
+                ),
+                (
+                    "curr_async_result_id",
+                    models.CharField(
+                        blank=True,
+                        editable=False,
+                        help_text="Current (active) AsyncResult is",
+                        max_length=36,
+                        null=True,
+                    ),
+                ),
+                (
+                    "last_async_result_id",
+                    models.CharField(
+                        blank=True,
+                        editable=False,
+                        help_text="Latest executed AsyncResult is",
+                        max_length=36,
+                        null=True,
+                    ),
+                ),
+                (
+                    "celery_history",
+                    models.JSONField(blank=True, default=dict, editable=False),
+                ),
+                (
+                    "celery_result",
+                    models.CharField(
+                        blank=True,
+                        default="",
+                        editable=False,
+                        max_length=100,
+                        null=True,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+                "default_permissions": (
+                    "add",
+                    "change",
+                    "delete",
+                    "view",
+                    "queue",
+                    "terminate",
+                    "inspect",
+                    "revoke",
+                ),
+            },
+        ),
+    ]

src/hope_dedup_engine/apps/api/models/__init__.py

@@ -5,3 +5,4 @@
     Duplicate,
     Image,
 )
+from hope_dedup_engine.apps.api.models.jobs import DedupJob  # noqa: F401

src/hope_dedup_engine/apps/api/models/jobs.py

@@ -0,0 +1,7 @@
+from django_celery_boost.models import CeleryTaskModel
+
+
+class DedupJob(CeleryTaskModel):
+    celery_task_name = (
+        "hope_dedup_engine.apps.api.deduplication.process.find_duplicates"
+    )

src/hope_dedup_engine/config/settings.py

@@ -44,6 +44,7 @@
     "hope_dedup_engine.apps.faces",
     "storages",
     "smart_env",
+    "django_celery_boost",
 )
 
 MIDDLEWARE = (