feat: preserve metadata when handling blobs; refactor storage_utils

ietf/api/tests.py

@@ -49,6 +49,7 @@
     'ietf.secr.proceedings',
     'ietf.ipr',
     'ietf.status',
+    'ietf.blobdb',
 )
 
 class CustomApiTests(TestCase):
@@ -1453,8 +1454,6 @@ def test_all_model_resources_exist(self):
         top = r.json()
         for name in self.apps:
             app_name = self.apps[name]
-            if app_name == "ietf.blobdb":
-                continue
             app = import_module(app_name)
             self.assertEqual("/api/v1/%s/"%name, top[name]["list_endpoint"])
             r = client.get(top[name]["list_endpoint"])

ietf/blobdb/factories.py

@@ -0,0 +1,13 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+import factory
+
+from .models import Blob
+
+
+class BlobFactory(factory.django.DjangoModelFactory):
+    class Meta:
+        model = Blob
+
+    name = factory.Faker("file_path")
+    bucket = factory.Faker("word")
+    content = factory.Faker("binary", length=32)  # careful, default length is 1e6

ietf/blobdb/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.19 on 2025-03-05 22:44
+# Copyright The IETF Trust 2025, All Rights Reserved
 
 from django.db import migrations, models
 import django.utils.timezone
@@ -38,7 +38,7 @@ class Migration(migrations.Migration):
                     "modified",
                     models.DateTimeField(
                         default=django.utils.timezone.now,
-                        help_text="Last modification time",
+                        help_text="Last modification time of the blob",
                     ),
                 ),
                 ("content", models.BinaryField(help_text="Content of the blob")),
@@ -50,6 +50,23 @@ class Migration(migrations.Migration):
                         max_length=96,
                     ),
                 ),
+                (
+                    "mtime",
+                    models.DateTimeField(
+                        blank=True,
+                        default=None,
+                        help_text="mtime associated with the blob as a filesystem object",
+                        null=True,
+                    ),
+                ),
+                (
+                    "content_type",
+                    models.CharField(
+                        blank=True,
+                        help_text="content-type header value for the blob contents",
+                        max_length=1024,
+                    ),
+                ),
             ],
         ),
         migrations.AddConstraint(

ietf/blobdb/models.py

@@ -11,12 +11,23 @@ class Blob(models.Model):
         max_length=1024, help_text="Name of the bucket containing this blob"
     )
     modified = models.DateTimeField(
-        default=timezone.now, help_text="Last modification time"
+        default=timezone.now, help_text="Last modification time of the blob"
     )
     content = models.BinaryField(help_text="Content of the blob")
     checksum = models.CharField(
         max_length=96, help_text="SHA-384 digest of the content", editable=False
     )
+    mtime = models.DateTimeField(
+        default=None,
+        blank=True,
+        null=True,
+        help_text="mtime associated with the blob as a filesystem object",
+    )
+    content_type = models.CharField(
+        max_length=1024,
+        blank=True,
+        help_text="content-type header value for the blob contents",
+    )
 
     class Meta:
         constraints = [

ietf/blobdb/storage.py

@@ -5,9 +5,21 @@
 from django.utils.deconstruct import deconstructible
 from django.utils import timezone
 
+from ietf.utils.storage import MetadataFile
 from .models import Blob
 
 
+class BlobFile(MetadataFile):
+
+    def __init__(self, content, name=None, mtime=None, content_type=""):
+        super().__init__(
+            file=ContentFile(content),
+            name=name,
+            mtime=mtime,
+            content_type=content_type,
+        )
+
+
 @deconstructible
 class BlobdbStorage(Storage):
 
@@ -43,12 +55,22 @@ def _open(self, name, mode="rb"):
             raise FileNotFoundError(
                 f"No object '{name}' exists in bucket '{self.bucket_name}'"
             )
-        return ContentFile(content=blob.content, name=blob.name)
+        return BlobFile(
+            content=blob.content,
+            name=blob.name,
+            mtime=blob.mtime or blob.modified,  # fall back to modified time
+            content_type=blob.content_type,
+        )
 
     def _save(self, name, content):
         Blob.objects.update_or_create(
             name=name,
             bucket=self.bucket_name,
-            defaults={"content": content.read(), "modified": timezone.now()},
+            defaults={
+                "content": content.read(),
+                "modified": timezone.now(),
+                "mtime": getattr(content, "mtime", None),
+                "content_type": getattr(content, "content_type", ""),
+            },
         )
         return name

ietf/blobdb/tests.py

@@ -1,3 +1,80 @@
-from django.test import TestCase
+# Copyright The IETF Trust 2025, All Rights Reserved
+import datetime
 
-# Create your tests here.
+from django.core.files.base import ContentFile
+
+from ietf.utils.test_utils import TestCase
+from .factories import BlobFactory
+from .models import Blob
+from .storage import BlobFile, BlobdbStorage
+
+
+class StorageTests(TestCase):
+    def test_save(self):
+        storage = BlobdbStorage(bucket_name="my-bucket")
+        timestamp = datetime.datetime(
+            2025,
+            3,
+            17,
+            1,
+            2,
+            3,
+            tzinfo=datetime.timezone.utc,
+        )
+        # Create file to save
+        my_file = BlobFile(
+            content=b"These are my bytes.",
+            mtime=timestamp,
+            content_type="application/x-my-content-type",
+        )
+        # save the file
+        saved_name = storage.save("myfile.txt", my_file)
+        # validate the outcome
+        self.assertEqual(saved_name, "myfile.txt")
+        blob = Blob.objects.filter(bucket="my-bucket", name="myfile.txt").first()
+        self.assertIsNotNone(blob)  # validates bucket and name
+        self.assertEqual(bytes(blob.content), b"These are my bytes.")
+        self.assertEqual(blob.mtime, timestamp)
+        self.assertEqual(blob.content_type, "application/x-my-content-type")
+
+    def test_save_naive_file(self):
+        storage = BlobdbStorage(bucket_name="my-bucket")
+        my_naive_file = ContentFile(content=b"These are my naive bytes.")
+        # save the file
+        saved_name = storage.save("myfile.txt", my_naive_file)
+        # validate the outcome
+        self.assertEqual(saved_name, "myfile.txt")
+        blob = Blob.objects.filter(bucket="my-bucket", name="myfile.txt").first()
+        self.assertIsNotNone(blob)  # validates bucket and name
+        self.assertEqual(bytes(blob.content), b"These are my naive bytes.")
+        self.assertIsNone(blob.mtime)
+        self.assertEqual(blob.content_type, "")
+
+    def test_open(self):
+        """BlobdbStorage open yields a BlobFile with specific mtime and content_type"""
+        mtime = datetime.datetime(2021, 1, 2, 3, 45, tzinfo=datetime.timezone.utc)
+        blob = BlobFactory(mtime=mtime, content_type="application/x-oh-no-you-didnt")
+        storage = BlobdbStorage(bucket_name=blob.bucket)
+        with storage.open(blob.name, "rb") as f:
+            self.assertTrue(isinstance(f, BlobFile))
+            assert isinstance(f, BlobFile)  # redundant, narrows type for linter
+            self.assertEqual(f.read(), bytes(blob.content))
+            self.assertEqual(f.mtime, mtime)
+            self.assertEqual(f.content_type, "application/x-oh-no-you-didnt")
+
+    def test_open_null_mtime(self):
+        """BlobdbStorage open yields a BlobFile with default mtime and content_type"""
+        blob = BlobFactory(content_type="application/x-oh-no-you-didnt")  # does not set mtime
+        storage = BlobdbStorage(bucket_name=blob.bucket)
+        with storage.open(blob.name, "rb") as f:
+            self.assertTrue(isinstance(f, BlobFile))
+            assert isinstance(f, BlobFile)  # redundant, narrows type for linter
+            self.assertEqual(f.read(), bytes(blob.content))
+            self.assertIsNotNone(f.mtime)
+            self.assertEqual(f.mtime, blob.modified)
+            self.assertEqual(f.content_type, "application/x-oh-no-you-didnt")
+
+    def test_open_file_not_found(self):
+        storage = BlobdbStorage(bucket_name="not-a-bucket")
+        with self.assertRaises(FileNotFoundError):
+            storage.open("definitely/not-a-file.txt")