deps: update shared (use new minio storage service) (#944)

Author: joseph-sentry

app.py

@@ -1,8 +1,6 @@
 import logging
 import logging.config
-import os
 
-import django
 from celery import Celery, signals
 
 from helpers.logging_config import get_logging_config_dict
@@ -13,11 +11,6 @@
 _config_dict = get_logging_config_dict()
 logging.config.dictConfig(_config_dict)
 
-# we're moving this before we create the Celery object
-# so that celery can detect Django is being used
-# using the Django fixup will help fix some database issues
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_scaffold.settings")
-django.setup()
 
 celery_app = Celery("tasks")
 celery_app.config_from_object("celery_config:CeleryWorkerConfig")

conftest.py

@@ -270,10 +270,10 @@ def mock_redis(mocker):
 
 @pytest.fixture
 def mock_storage(mocker):
-    m = mocker.patch("services.storage._cached_get_storage_client")
+    m = mocker.patch("shared.storage.get_appropriate_storage_service")
     storage_server = MemoryStorageService({})
     m.return_value = storage_server
-    yield storage_server
+    return storage_server
 
 
 @pytest.fixture

main.py

@@ -1,22 +1,32 @@
-# -*- coding: utf-8 -*-
-import logging
 import os
-import sys
-import typing
-
-import click
-from celery.signals import worker_process_shutdown
-from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
-from shared.celery_config import BaseCeleryConfig
-from shared.config import get_config
-from shared.license import startup_license_logging
-from shared.metrics import start_prometheus
-from shared.storage.exceptions import BucketAlreadyExistsError
-
-import app
-from helpers.environment import get_external_dependencies_folder
-from helpers.version import get_current_version
-from services.storage import get_storage_client
+
+import django
+
+# we're moving this before we create the Celery object
+# so that celery can detect Django is being used
+# using the Django fixup will help fix some database issues
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_scaffold.settings")
+django.setup()
+
+# -*- coding: utf-8 -*-
+import logging  # noqa: E402
+import os  # noqa: E402
+import sys  # noqa: E402
+import typing  # noqa: E402
+
+import click  # noqa: E402
+import shared.storage  # noqa: E402
+from celery.signals import worker_process_shutdown  # noqa: E402
+from prometheus_client import REGISTRY, CollectorRegistry, multiprocess  # noqa: E402
+from shared.celery_config import BaseCeleryConfig  # noqa: E402
+from shared.config import get_config  # noqa: E402
+from shared.license import startup_license_logging  # noqa: E402
+from shared.metrics import start_prometheus  # noqa: E402
+from shared.storage.exceptions import BucketAlreadyExistsError  # noqa: E402
+
+import app  # noqa: E402
+from helpers.environment import get_external_dependencies_folder  # noqa: E402
+from helpers.version import get_current_version  # noqa: E402
 
 log = logging.getLogger(__name__)
 
@@ -69,7 +79,9 @@ def setup_worker():
 
     start_prometheus(9996, registry=registry)  # 9996 is an arbitrary port number
 
-    storage_client = get_storage_client()
+    # this storage client is only used to create the bucket so it doesn't need to be
+    # aware of the repoid
+    storage_client = shared.storage.get_appropriate_storage_service()
     minio_config = get_config("services", "minio")
     bucket_name = get_config("services", "minio", "bucket", default="archive")
     auto_create_bucket = get_config(

requirements.in

@@ -1,5 +1,5 @@
 https://github.com/codecov/test-results-parser/archive/c840502d1b4dd7d05b2efc2c1328affaf2acd27c.tar.gz#egg=test-results-parser
-https://github.com/codecov/shared/archive/45252f75524c38172b991a960bac964ec7b6f7d1.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/8c5de3fadd3c987304043c6cfd64864f372d674f.tar.gz#egg=shared
 https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
 asgiref>=3.7.2
 analytics-python==1.3.0b1

requirements.txt

@@ -328,7 +328,7 @@ sentry-sdk==2.13.0
     #   shared
 setuptools==75.6.0
     # via nodeenv
-shared @ https://github.com/codecov/shared/archive/45252f75524c38172b991a960bac964ec7b6f7d1.tar.gz#egg=shared
+shared @ https://github.com/codecov/shared/archive/8c5de3fadd3c987304043c6cfd64864f372d674f.tar.gz#egg=shared
     # via -r requirements.in
 six==1.16.0
     # via
@@ -399,4 +399,6 @@ wrapt==1.16.0
 yarl==1.9.4
     # via vcrpy
 zstandard==0.23.0
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   shared

services/archive.py

@@ -7,11 +7,11 @@
 from uuid import uuid4
 
 import sentry_sdk
+import shared.storage
 from shared.config import get_config
 from shared.utils.ReportEncoder import ReportEncoder
 
 from helpers.metrics import metrics
-from services.storage import get_storage_client
 
 log = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def __init__(self, repository, bucket=None) -> None:
             self.root = get_config("services", "minio", "bucket", default="archive")
         else:
             self.root = bucket
-        self.storage = get_storage_client()
+        self.storage = shared.storage.get_appropriate_storage_service(repository.repoid)
         log.debug("Getting archive hash")
         self.storage_hash = self.get_archive_hash(repository)
 

services/bundle_analysis/comparison.py

@@ -1,10 +1,10 @@
 from functools import cached_property
 
+import shared.storage
 from shared.bundle_analysis import (
     BundleAnalysisComparison,
     BundleAnalysisReportLoader,
 )
-from shared.storage import get_appropriate_storage_service
 
 from database.enums import ReportType
 from database.models.core import Commit, Repository
@@ -69,7 +69,9 @@ def head_commit_report(self) -> CommitReport:
 
     def get_comparison(self) -> BundleAnalysisComparison:
         loader = BundleAnalysisReportLoader(
-            storage_service=get_appropriate_storage_service(),
+            storage_service=shared.storage.get_appropriate_storage_service(
+                self.repository.repoid
+            ),
             repo_key=ArchiveService.get_archive_hash(self.repository),
         )
 

services/bundle_analysis/notify/contexts/__init__.py

@@ -3,6 +3,7 @@
 from typing import Generic, Literal, Self, TypeVar
 
 import sentry_sdk
+import shared.storage
 from shared.bundle_analysis import BundleAnalysisReport, BundleAnalysisReportLoader
 from shared.torngit.base import TorngitBaseAdapter
 from shared.validation.types import BundleThreshold
@@ -18,7 +19,6 @@
     NotificationUserConfig,
 )
 from services.repository import get_repo_provider_service
-from services.storage import get_storage_client
 
 T = TypeVar("T")
 
@@ -167,7 +167,9 @@ def load_bundle_analysis_report(self) -> "NotificationContextBuilder":
         repo_hash = ArchiveService.get_archive_hash(
             self._notification_context.repository
         )
-        storage_service = get_storage_client()
+        storage_service = shared.storage.get_appropriate_storage_service(
+            self._notification_context.repository.repoid
+        )
         analysis_report_loader = BundleAnalysisReportLoader(storage_service, repo_hash)
         bundle_analysis_report = analysis_report_loader.load(
             self._notification_context.commit_report.external_id

services/bundle_analysis/notify/contexts/tests/test_comment_context.py

@@ -290,10 +290,7 @@ def test_build_context(self, dbsession, mocker, mock_storage):
         builder = BundleAnalysisPRCommentContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         assert context.commit_report == head_commit_report
         assert context.bundle_analysis_report.session_count() == 18

services/bundle_analysis/notify/contexts/tests/test_commit_status_context.py

@@ -235,10 +235,7 @@ def test_build_context(self, dbsession, mocker, mock_storage):
         builder = CommitStatusNotificationContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         assert context.commit_report == head_commit_report
         assert context.bundle_analysis_report.session_count() == 18