Merge pull request #86 from octue/fix-maxsize-bytes

Fix signature mismatch bug due to max size bytes header

Author: thclark

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM windpioneers/gdal-python:little-gecko-gdal-2.4.1-python-3.9-dev
+FROM windpioneers/gdal-python:rational-swordtail-gdal-3.10.0-python-3.13-dev
 
 # Bust any pre-existing apt cache
 #     Solves: https://askubuntu.com/questions/1388000/failed-to-fetch-deb-debian-org-debian-pool-main-l-linux-linux-libc-dev-5-10-70-1

.pre-commit-config.yaml

@@ -20,11 +20,18 @@ repos:
       - id: check-json5
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.9
+    rev: v0.11.0
     hooks:
       - id: ruff
+        name: Ruff lint
         args: [--fix, --exit-non-zero-on-fix]
+
+      - id: ruff
+        name: Ruff isort
+        args: [check, --select, I, --fix]
+
       - id: ruff-format
+        name: Ruff format
 
   - repo: https://github.com/windpioneers/pre-commit-hooks
     rev: 0.0.5

django_gcp/events/signals.py

@@ -1,4 +1,3 @@
 from django.dispatch import Signal
 
-
 event_received = Signal()

django_gcp/events/utils.py

@@ -1,13 +1,14 @@
 import base64
+from datetime import timezone
 import json
 import logging
-from datetime import timezone
+
 from dateutil.parser import isoparse
 from django.conf import settings
 from django.urls import reverse
 from django.utils.http import urlencode
-from django_gcp.exceptions import InvalidPubSubMessageError
 
+from django_gcp.exceptions import InvalidPubSubMessageError
 
 logger = logging.getLogger(__name__)
 

django_gcp/events/views.py

@@ -1,5 +1,6 @@
 import json
 import logging
+
 from django.conf import settings
 from django.http import HttpResponse
 from django.utils.decorators import method_decorator
@@ -8,7 +9,6 @@
 
 from .signals import event_received
 
-
 logger = logging.getLogger(__name__)
 
 

django_gcp/logs/error_reporting.py

@@ -2,6 +2,7 @@
 # pylint: disable=broad-except
 
 from logging import StreamHandler
+
 from django.conf import settings
 from google.cloud import error_reporting
 

django_gcp/logs/structured_logs.py

@@ -1,4 +1,5 @@
 import json
+
 from google.cloud.logging_v2.handlers import StructuredLogHandler
 
 

django_gcp/management/commands/_base.py

@@ -1,4 +1,5 @@
 import abc
+
 from django.apps import apps
 from django.core.management.base import BaseCommand as DjangoBaseCommand
 

django_gcp/management/commands/cleanup_tmp_files.py

@@ -1,5 +1,7 @@
 from datetime import datetime, timedelta
+
 from django.core.management.base import BaseCommand
+
 from django_gcp.storage import GoogleCloudStorage
 
 

django_gcp/metadata/__init__.py

@@ -1,4 +1,3 @@
 from .metadata import ENDPOINTS, CloudRunMetadata
 
-
 __all__ = ["ENDPOINTS", "CloudRunMetadata"]

django_gcp/metadata/metadata.py

@@ -4,7 +4,6 @@
 
 from ..exceptions import NotOnCloudRunError
 
-
 ENDPOINTS = {
     "project_id": "computeMetadata/v1/project/project-id",
     "project_number": "/computeMetadata/v1/project/numeric-project-id",

django_gcp/signals.py

@@ -1,6 +1,5 @@
 import logging
 
-
 logger = logging.getLogger(__name__)
 
 

django_gcp/static/django_gcp/cloud_object_widget.js

@@ -191,12 +191,17 @@ $(document).ready(function($) {
       // var field = wrapper.find("input[type=hidden]");
       var file = wrapper.find(".gcp-file-input").get(0).files[0];
       const signed_ingress_url = wrapper.data("signed-ingress-url")
-      // const ingress_path = wrapper.data("ingress-path")
+      const max_size_bytes = wrapper.data("max-size-bytes")
+      const headers = {}
+      if (max_size_bytes !== 0) {
+        headers["X-Goog-Content-Length-Range"] = `0,${max_size_bytes}`
+      }
       $.ajax({
         url: signed_ingress_url,
         type: "PUT",
         data: file,
         contentType: "application/octet-stream",
+        headers: headers,
         success: function() {
           // field.val(JSON.stringify({_tmp_path: ingress_path, name: file.name, content_type: file.type}));
           window.gcp_number_of_files--;

django_gcp/storage/compress.py

@@ -1,7 +1,7 @@
-import io
-import zlib
 from gzip import GzipFile
+import io
 from typing import Optional
+import zlib
 
 from .utils import to_bytes
 

django_gcp/storage/fields.py

@@ -12,16 +12,11 @@
 
 from .forms import CloudObjectFormField
 from .gcloud import GoogleCloudStorage
-from .operations import blob_exists, copy_blob, get_signed_upload_url
+from .operations import UNLIMITED_MAX_SIZE, blob_exists, copy_blob, get_signed_upload_url
 from .widgets import DEFAULT_ACCEPT_MIMETYPE, CloudObjectWidget
 
 logger = logging.getLogger(__name__)
 
-
-# 32 megabyte default size chosen to be consistent with Cloud Run's
-# largest request size
-DEFAULT_MAX_SIZE_BYTES = 32 * 1024 * 1024
-
 DEFAULT_OVERWRITE_MODE = "never"
 
 OVERWRITE_MODES = [
@@ -102,8 +97,8 @@ def __init__(
         ingress_to="_tmp/",
         store_key="media",
         accept_mimetype=DEFAULT_ACCEPT_MIMETYPE,
-        max_size_bytes=DEFAULT_MAX_SIZE_BYTES,
         overwrite_mode=DEFAULT_OVERWRITE_MODE,
+        max_size_bytes=None,
         on_change=None,
         update_attributes=None,
         **kwargs,
@@ -114,6 +109,7 @@ def __init__(
         self._on_commit_valid = None
         self._versioning_enabled = None
         self._temporary_path = None
+        self._max_size_bytes = max_size_bytes
         self._primary_key_set_explicitly = "primary_key" in kwargs
         self._choices_set_explicitly = "choices" in kwargs
         self.overwrite_mode = overwrite_mode
@@ -123,7 +119,6 @@ def __init__(
         self.ingress_path = None
         self.store_key = store_key
         self.accept_mimetype = accept_mimetype
-        self.max_size_bytes = max_size_bytes
         self.on_change = on_change
         kwargs["default"] = kwargs.pop("default", None)
         kwargs["help_text"] = kwargs.pop("help_text", "GCP cloud storage object")
@@ -178,6 +173,7 @@ def formfield(self, **kwargs):
             widget = CloudObjectWidget(
                 accept_mimetype=self.accept_mimetype,
                 signed_ingress_url=self._get_signed_ingress_url(),
+                max_size_bytes=self.max_size_bytes,
                 ingress_path=self._get_temporary_path(),
             )
 
@@ -195,6 +191,23 @@ def override_blobfield_value(self):
             DEFAULT_OVERRIDE_BLOBFIELD_VALUE,
         )
 
+    @property
+    def max_size_bytes(self):
+        """Shortcut to determine the max size in bytes allowable as an upload
+
+        Defaults to unlimited upload size (which is dangerous, but since you're
+        signing a URL for the user to upload we assume you've established trust
+        in the user and know what you're doing!).
+        """
+        if self._max_size_bytes is not None:
+            return self._max_size_bytes
+
+        return getattr(
+            settings,
+            "GCP_STORAGE_BLOBFIELD_MAX_SIZE_BYTES",
+            UNLIMITED_MAX_SIZE,
+        )
+
     def clean(self, value, model_instance, skip_validation=False):
         """Clean the value to ensure correct state and on_commit hooks.
         This method is called during clean_fields on model save.

django_gcp/storage/forms.py

@@ -1,9 +1,9 @@
 import logging
+
 from django.forms.fields import JSONField
 
 from .widgets import CloudObjectWidget
 
-
 logger = logging.getLogger(__name__)
 
 

django_gcp/storage/gcloud.py

@@ -1,6 +1,7 @@
 import logging
 import mimetypes
 from tempfile import SpooledTemporaryFile
+
 from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
 from django.core.files.base import File
@@ -15,7 +16,6 @@
 from .settings import StorageSettings
 from .utils import clean_name, get_available_overwrite_name, safe_join, to_bytes
 
-
 CONTENT_ENCODING = "content_encoding"
 CONTENT_TYPE = "content_type"
 

django_gcp/storage/operations.py

@@ -11,6 +11,8 @@
 
 logger = logging.getLogger(__name__)
 
+UNLIMITED_MAX_SIZE = 0
+
 
 def blob_exists(bucket, blob_name):
     """Quick check that a blob with a given name exists in a bucket"""
@@ -182,7 +184,7 @@ def get_generations(bucket, blob_name):
     return list(bucket.client.list_blobs(bucket.name, versions=True, prefix=blob_name))
 
 
-def get_signed_upload_url(bucket, blob_name, timedelta=None, max_size_bytes=None, **kwargs):
+def get_signed_upload_url(bucket, blob_name, timedelta=None, max_size_bytes=UNLIMITED_MAX_SIZE, **kwargs):
     """Get a signed URL for uploading a blob to GCS
 
     :param google.cloud.storage.Bucket bucket: The bucket to which the blob will be uploaded
@@ -195,7 +197,8 @@ def get_signed_upload_url(bucket, blob_name, timedelta=None, max_size_bytes=None
         timedelta = datetime.timedelta(minutes=60)
 
     blob = bucket.blob(blob_name)
-    if max_size_bytes is not None:
+
+    if max_size_bytes != 0:
         content_length_range = f"0,{max_size_bytes}"
         headers = kwargs.pop("headers", {})
         headers["X-Goog-Content-Length-Range"] = content_length_range

django_gcp/storage/settings.py

@@ -1,9 +1,9 @@
 from datetime import timedelta
+
 from django.conf import settings as django_settings
 from django.core.exceptions import ImproperlyConfigured
 from django.core.signals import setting_changed
 
-
 DEFAULT_GZIP_CONTENT_TYPES = (
     "text/css",
     "text/javascript",

django_gcp/storage/utils.py

@@ -1,5 +1,6 @@
 import os
 import posixpath
+
 from django.core.exceptions import SuspiciousFileOperation
 from django.utils.encoding import force_bytes
 

django_gcp/storage/widgets.py

@@ -3,6 +3,8 @@
 from django.conf import settings
 from django.forms import Widget
 
+from .operations import UNLIMITED_MAX_SIZE
+
 DEFAULT_ACCEPT_MIMETYPE = "*/*"
 
 
@@ -28,6 +30,7 @@ def __init__(
         self,
         *args,
         signed_ingress_url=None,
+        max_size_bytes=UNLIMITED_MAX_SIZE,
         ingress_path=None,
         accept_mimetype=DEFAULT_ACCEPT_MIMETYPE,
         **kwargs,
@@ -37,14 +40,15 @@ def __init__(
         self._kwargs = kwargs
         self.accept_mimetype = accept_mimetype
         self.signed_ingress_url = signed_ingress_url
+        self.max_size_bytes = max_size_bytes
         self.ingress_path = ingress_path
 
         if "unfold" in settings.INSTALLED_APPS:
             self.template_name = "django_gcp/contrib/unfold/cloud_object_widget.html"
 
     def get_context(self, name, value, attrs):
         context = super().get_context(name, value, attrs)
-
+        context["max_size_bytes"] = self.max_size_bytes
         context["signed_ingress_url"] = self.signed_ingress_url
         context["ingress_path"] = self.ingress_path
         context["accept_mimetype"] = self.accept_mimetype

django_gcp/tasks/_pilot/base.py

@@ -3,6 +3,7 @@
 import logging
 import os
 from typing import Any, Callable, Dict, Generator, List, Tuple, Union
+
 from google import auth
 from google.auth import iam
 from google.auth.credentials import Credentials
@@ -16,7 +17,6 @@
 
 from . import exceptions
 
-
 DEFAULT_PROJECT = os.environ.get("GCP_PROJECT", None)
 DEFAULT_LOCATION = os.environ.get("GCP_LOCATION", None)
 DEFAULT_SERVICE_ACCOUNT = os.environ.get("GCP_SERVICE_ACCOUNT", None)

django_gcp/tasks/_pilot/mocker.py

@@ -1,6 +1,7 @@
 from contextlib import ExitStack
 from functools import wraps
 from unittest.mock import patch
+
 from google.oauth2.service_account import Credentials
 
 

django_gcp/tasks/_pilot/pubsub.py

@@ -1,8 +1,9 @@
 # https://googleapis.dev/python/pubsub/latest/index.html
 import base64
-import json
 from dataclasses import dataclass
+import json
 from typing import Any, AsyncIterator, Callable, Dict, Union
+
 from google.api_core.exceptions import AlreadyExists, NotFound
 from google.cloud import pubsub_v1
 from google.protobuf.field_mask_pb2 import FieldMask

django_gcp/tasks/_pilot/scheduler.py

@@ -1,12 +1,12 @@
 # More Information <https://cloud.google.com/scheduler/docs/reference/rest>
 import os
 from typing import Dict, Generator
+
 from google.api_core.exceptions import NotFound
 from google.cloud import scheduler
 
 from .base import AppEngineBasedService, GoogleCloudPilotAPI
 
-
 DEFAULT_TIMEZONE = os.environ.get("TIMEZONE", "Europe/London")  # UTC
 MAX_TIMEOUT = 30 * 60  # max allowed to HTTP endpoints is 30 minutes
 

django_gcp/tasks/_pilot/tasks.py

@@ -1,7 +1,8 @@
 # Reference: https://googleapis.dev/python/cloudtasks/latest/tasks_v2/cloud_tasks.html
-import uuid
 from datetime import datetime, timedelta
 from typing import Dict
+import uuid
+
 from google.api_core.exceptions import FailedPrecondition
 from google.cloud import tasks_v2
 from google.protobuf import timestamp_pb2

django_gcp/tasks/helpers.py

@@ -1,4 +1,5 @@
 import asyncio
+
 from asgiref.sync import async_to_sync