deps: update shared (use new minio storage service) (#944)

Author: joseph-sentry

Diff for: app.py

@@ -1,8 +1,6 @@
 import logging
 import logging.config
-import os
 
-import django
 from celery import Celery, signals
 
 from helpers.logging_config import get_logging_config_dict
@@ -13,11 +11,6 @@
 _config_dict = get_logging_config_dict()
 logging.config.dictConfig(_config_dict)
 
-# we're moving this before we create the Celery object
-# so that celery can detect Django is being used
-# using the Django fixup will help fix some database issues
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_scaffold.settings")
-django.setup()
 
 celery_app = Celery("tasks")
 celery_app.config_from_object("celery_config:CeleryWorkerConfig")

Diff for: conftest.py

@@ -270,10 +270,10 @@ def mock_redis(mocker):
 
 @pytest.fixture
 def mock_storage(mocker):
-    m = mocker.patch("services.storage._cached_get_storage_client")
+    m = mocker.patch("shared.storage.get_appropriate_storage_service")
     storage_server = MemoryStorageService({})
     m.return_value = storage_server
-    yield storage_server
+    return storage_server
 
 
 @pytest.fixture

Diff for: main.py

@@ -1,22 +1,32 @@
-# -*- coding: utf-8 -*-
-import logging
 import os
-import sys
-import typing
-
-import click
-from celery.signals import worker_process_shutdown
-from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
-from shared.celery_config import BaseCeleryConfig
-from shared.config import get_config
-from shared.license import startup_license_logging
-from shared.metrics import start_prometheus
-from shared.storage.exceptions import BucketAlreadyExistsError
-
-import app
-from helpers.environment import get_external_dependencies_folder
-from helpers.version import get_current_version
-from services.storage import get_storage_client
+
+import django
+
+# we're moving this before we create the Celery object
+# so that celery can detect Django is being used
+# using the Django fixup will help fix some database issues
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "django_scaffold.settings")
+django.setup()
+
+# -*- coding: utf-8 -*-
+import logging  # noqa: E402
+import os  # noqa: E402
+import sys  # noqa: E402
+import typing  # noqa: E402
+
+import click  # noqa: E402
+import shared.storage  # noqa: E402
+from celery.signals import worker_process_shutdown  # noqa: E402
+from prometheus_client import REGISTRY, CollectorRegistry, multiprocess  # noqa: E402
+from shared.celery_config import BaseCeleryConfig  # noqa: E402
+from shared.config import get_config  # noqa: E402
+from shared.license import startup_license_logging  # noqa: E402
+from shared.metrics import start_prometheus  # noqa: E402
+from shared.storage.exceptions import BucketAlreadyExistsError  # noqa: E402
+
+import app  # noqa: E402
+from helpers.environment import get_external_dependencies_folder  # noqa: E402
+from helpers.version import get_current_version  # noqa: E402
 
 log = logging.getLogger(__name__)
 
@@ -69,7 +79,9 @@ def setup_worker():
 
     start_prometheus(9996, registry=registry)  # 9996 is an arbitrary port number
 
-    storage_client = get_storage_client()
+    # this storage client is only used to create the bucket so it doesn't need to be
+    # aware of the repoid
+    storage_client = shared.storage.get_appropriate_storage_service()
     minio_config = get_config("services", "minio")
     bucket_name = get_config("services", "minio", "bucket", default="archive")
     auto_create_bucket = get_config(

Diff for: requirements.in

@@ -1,5 +1,5 @@
 https://github.com/codecov/test-results-parser/archive/c840502d1b4dd7d05b2efc2c1328affaf2acd27c.tar.gz#egg=test-results-parser
-https://github.com/codecov/shared/archive/45252f75524c38172b991a960bac964ec7b6f7d1.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/8c5de3fadd3c987304043c6cfd64864f372d674f.tar.gz#egg=shared
 https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
 asgiref>=3.7.2
 analytics-python==1.3.0b1

Diff for: requirements.txt

@@ -328,7 +328,7 @@ sentry-sdk==2.13.0
     #   shared
 setuptools==75.6.0
     # via nodeenv
-shared @ https://github.com/codecov/shared/archive/45252f75524c38172b991a960bac964ec7b6f7d1.tar.gz#egg=shared
+shared @ https://github.com/codecov/shared/archive/8c5de3fadd3c987304043c6cfd64864f372d674f.tar.gz#egg=shared
     # via -r requirements.in
 six==1.16.0
     # via
@@ -399,4 +399,6 @@ wrapt==1.16.0
 yarl==1.9.4
     # via vcrpy
 zstandard==0.23.0
-    # via -r requirements.in
+    # via
+    #   -r requirements.in
+    #   shared

Diff for: services/archive.py

@@ -7,11 +7,11 @@
 from uuid import uuid4
 
 import sentry_sdk
+import shared.storage
 from shared.config import get_config
 from shared.utils.ReportEncoder import ReportEncoder
 
 from helpers.metrics import metrics
-from services.storage import get_storage_client
 
 log = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def __init__(self, repository, bucket=None) -> None:
             self.root = get_config("services", "minio", "bucket", default="archive")
         else:
             self.root = bucket
-        self.storage = get_storage_client()
+        self.storage = shared.storage.get_appropriate_storage_service(repository.repoid)
         log.debug("Getting archive hash")
         self.storage_hash = self.get_archive_hash(repository)
 

Diff for: services/bundle_analysis/comparison.py

@@ -1,10 +1,10 @@
 from functools import cached_property
 
+import shared.storage
 from shared.bundle_analysis import (
     BundleAnalysisComparison,
     BundleAnalysisReportLoader,
 )
-from shared.storage import get_appropriate_storage_service
 
 from database.enums import ReportType
 from database.models.core import Commit, Repository
@@ -69,7 +69,9 @@ def head_commit_report(self) -> CommitReport:
 
     def get_comparison(self) -> BundleAnalysisComparison:
         loader = BundleAnalysisReportLoader(
-            storage_service=get_appropriate_storage_service(),
+            storage_service=shared.storage.get_appropriate_storage_service(
+                self.repository.repoid
+            ),
             repo_key=ArchiveService.get_archive_hash(self.repository),
         )
 

Diff for: services/bundle_analysis/notify/contexts/__init__.py

@@ -3,6 +3,7 @@
 from typing import Generic, Literal, Self, TypeVar
 
 import sentry_sdk
+import shared.storage
 from shared.bundle_analysis import BundleAnalysisReport, BundleAnalysisReportLoader
 from shared.torngit.base import TorngitBaseAdapter
 from shared.validation.types import BundleThreshold
@@ -18,7 +19,6 @@
     NotificationUserConfig,
 )
 from services.repository import get_repo_provider_service
-from services.storage import get_storage_client
 
 T = TypeVar("T")
 
@@ -167,7 +167,9 @@ def load_bundle_analysis_report(self) -> "NotificationContextBuilder":
         repo_hash = ArchiveService.get_archive_hash(
             self._notification_context.repository
         )
-        storage_service = get_storage_client()
+        storage_service = shared.storage.get_appropriate_storage_service(
+            self._notification_context.repository.repoid
+        )
         analysis_report_loader = BundleAnalysisReportLoader(storage_service, repo_hash)
         bundle_analysis_report = analysis_report_loader.load(
             self._notification_context.commit_report.external_id

Diff for: services/bundle_analysis/notify/contexts/tests/test_comment_context.py

@@ -290,10 +290,7 @@ def test_build_context(self, dbsession, mocker, mock_storage):
         builder = BundleAnalysisPRCommentContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         assert context.commit_report == head_commit_report
         assert context.bundle_analysis_report.session_count() == 18

Diff for: services/bundle_analysis/notify/contexts/tests/test_commit_status_context.py

@@ -235,10 +235,7 @@ def test_build_context(self, dbsession, mocker, mock_storage):
         builder = CommitStatusNotificationContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         assert context.commit_report == head_commit_report
         assert context.bundle_analysis_report.session_count() == 18

Diff for: services/bundle_analysis/notify/messages/tests/test_comment.py

@@ -48,10 +48,7 @@ def test_build_message_from_samples(self, dbsession, mocker, mock_storage):
         builder = BundleAnalysisPRCommentContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         message = BundleAnalysisCommentMarkdownStrategy().build_message(context)
         assert message == dedent("""\

Diff for: services/bundle_analysis/notify/messages/tests/test_commit_status.py

@@ -102,10 +102,7 @@ def test_build_message_from_samples_negative_changes(
         builder = CommitStatusNotificationContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         message = CommitStatusMessageStrategy().build_message(context)
         assert message == expected
@@ -157,10 +154,7 @@ def test_build_message_from_samples(
         builder = CommitStatusNotificationContextBuilder().initialize(
             head_commit, user_yaml, GITHUB_APP_INSTALLATION_DEFAULT_NAME
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         context = builder.build_context().get_result()
         message = CommitStatusMessageStrategy().build_message(context)
         assert message == expected
@@ -178,10 +172,7 @@ def _setup_send_message_tests(
             "services.bundle_analysis.notify.contexts.get_repo_provider_service",
             return_value=fake_repo_provider,
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
+
         head_commit, base_commit = get_commit_pair(dbsession)
         head_commit.parent_commit_id = base_commit.commitid
         dbsession.add_all([head_commit, base_commit])

Diff for: services/bundle_analysis/notify/tests/test_notify_service.py

@@ -135,10 +135,6 @@ def test_create_context_success(self, dbsession, mock_storage, mocker):
         enriched_pull = get_enriched_pull_setting_up_mocks(
             dbsession, mocker, (head_commit, base_commit)
         )
-        mocker.patch(
-            "services.bundle_analysis.comparison.get_appropriate_storage_service",
-            return_value=mock_storage,
-        )
         service = BundleAnalysisNotifyService(head_commit, current_yaml)
         result = service.create_context_for_notification(
             BaseBundleAnalysisNotificationContext(

Diff for: services/bundle_analysis/report.py

@@ -5,6 +5,7 @@
 from typing import Any, Dict, Optional
 
 import sentry_sdk
+import shared.storage
 from shared.bundle_analysis import BundleAnalysisReport, BundleAnalysisReportLoader
 from shared.bundle_analysis.models import AssetType, MetadataKey
 from shared.bundle_analysis.storage import get_bucket_name
@@ -25,7 +26,6 @@
 from database.models.timeseries import Measurement, MeasurementName
 from services.archive import ArchiveService
 from services.report import BaseReportService
-from services.storage import get_storage_client
 from services.timeseries import repository_datasets_query
 
 log = logging.getLogger(__name__)
@@ -234,7 +234,7 @@ def process_upload(
         """
         commit_report: CommitReport = upload.report
         repo_hash = ArchiveService.get_archive_hash(commit_report.commit.repository)
-        storage_service = get_storage_client()
+        storage_service = shared.storage.get_appropriate_storage_service(commit.repoid)
         bundle_loader = BundleAnalysisReportLoader(storage_service, repo_hash)
 
         # fetch existing bundle report from storage
@@ -383,7 +383,9 @@ def save_measurements(
         try:
             commit_report: CommitReport = upload.report
             repo_hash = ArchiveService.get_archive_hash(commit_report.commit.repository)
-            storage_service = get_storage_client()
+            storage_service = shared.storage.get_appropriate_storage_service(
+                commit.repoid
+            )
             bundle_loader = BundleAnalysisReportLoader(storage_service, repo_hash)
 
             # fetch existing bundle report from storage