Connect to microservice when looking for existing raws.

The microservice maps metadata to filenames without requiring that the
latter be a deterministic function of the former, which for Rubin
instruments it isn't.

python/activator/raw.py

@@ -41,6 +41,9 @@
 import os
 import re
 import time
+import urllib.parse
+
+import requests
 
 from lsst.obs.lsst import LsstCam, LsstComCam, LsstComCamSim
 from lsst.obs.lsst.translators.lsst import LsstBaseTranslator
@@ -186,6 +189,17 @@ def check_for_snap(
         was found. If multiple files match, this function logs an error
         but returns one of the files anyway.
     """
+    if microservice:
+        try:
+            return _query_microservice(microservice=microservice,
+                                       instrument=instrument,
+                                       group=group,
+                                       detector=detector,
+                                       snap=snap,
+                                       )
+        except RuntimeError:
+            _log.exception("Could not query microservice, falling back to prefix algorithm.")
+
     prefix = get_prefix_from_snap(instrument, group, detector, snap)
     if not prefix:
         return None
@@ -208,29 +222,78 @@ def get_prefix_from_snap(
 
     Parameters
     ----------
-    instrument: `str`
+    instrument : `str`
         The name of the instrument taking the image.
-    group: `str`
+    group : `str`
         The group id from the visit, associating the snaps making up the visit.
-    detector: `int`
+    detector : `int`
         The integer detector id for the image being sought.
-    snap: `int`
+    snap : `int`
         The snap number within the group for the visit.
 
     Returns
     -------
-    prefix: `str` or None
+    prefix : `str` or `None`
         The prefix to a path to the corresponding raw image object.  If it
         can be calculated, then the prefix may be the entire path.  If no
-        prefix can be calculated, None is returned.
+        prefix can be calculated, `None` is returned.
     """
-
     if instrument not in _LSST_CAMERA_LIST:
         return f"{instrument}/{detector}/{group}/{snap}/"
-    # TODO DM-39022: use a microservice to determine paths for LSST cameras.
     return None
 
 
+def _query_microservice(
+    microservice: str, instrument: str, group: str, detector: int, snap: int
+) -> str | None:
+    """Look up a raw image's location from the raw image microservice.
+
+    Parameters
+    ----------
+    microservice : `str`
+        The URI of the microservice to query.
+    instrument : `str`
+        The name of the instrument taking the image.
+    group : `str`
+        The group id from the visit, associating the snaps making up the visit.
+    detector : `int`
+        The integer detector id for the image being sought.
+    snap : `int`
+        The snap number within the group for the visit.
+
+    Returns
+    -------
+    key : `str` or `None`
+        The raw's object key within its bucket, or `None` if no image was found.
+
+    Raises
+    ------
+    RuntimeError
+        Raised if this function could not connect to the microservice, or if the
+        microservice encountered an error.
+    """
+    detector_name = _DETECTOR_FROM_INT[instrument][detector]
+    uri = f"{microservice}/{instrument}/{group}/{snap}/{detector_name}"
+    try:
+        response = requests.get(uri, timeout=1.0)
+        response.raise_for_status()
+        unpacked = response.json()
+    except requests.Timeout as e:
+        raise RuntimeError("Timed out connecting to raw microservice.") from e
+    except requests.RequestException as e:
+        raise RuntimeError("Could not query raw microservice.") from e
+
+    if unpacked["error"]:
+        raise RuntimeError(f"Raw microservice had an internal error: {unpacked['message']}")
+    if unpacked["present"]:
+        # Need to return just the key, without the bucket
+        path = urllib.parse.urlparse(unpacked["uri"], allow_fragments=False).path
+        # Valid key does not start with a /
+        return path.lstrip("/")
+    else:
+        return None
+
+
 def get_exp_id_from_oid(oid: str) -> int:
     """Calculate an exposure id from an image object's pathname.
 

tests/test_raw.py

@@ -131,15 +131,89 @@ def test_writeread(self):
         self.assertEqual(get_exp_id_from_oid(path), self.exposure)
 
     def test_get_prefix(self):
-        """Test that get_prefix_from_snap returns None for now."""
+        """Test that get_prefix_from_snap returns None for LSST cameras."""
         prefix = get_prefix_from_snap(self.instrument, self.group, self.detector, self.snap)
         self.assertIsNone(prefix)
 
+    def test_check_for_snap_present(self):
+        microservice = "http://fake_host/fake_app"
+        path = get_raw_path(self.instrument, self.detector, self.group, self.snap, self.exposure, self.filter)
+        message = {"error": False, "present": True, "uri": f"s3://{self.bucket}/{path}"}
+
+        fits_path = ResourcePath(f"s3://{self.bucket}").join(path)
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "S3 does not support flushing objects", UserWarning)
+            with fits_path.open("wb"):
+                pass  # Empty file is just fine
+
+        with unittest.mock.patch("requests.get", **{"return_value.json.return_value": message}):
+            oid = check_for_snap(boto3.client("s3"),
+                                 self.bucket,
+                                 instrument=self.instrument,
+                                 microservice=microservice,
+                                 group=self.group,
+                                 snap=self.snap,
+                                 detector=self.detector,
+                                 )
+        self.assertEqual(oid, path)
+
+    def test_check_for_snap_noservice(self):
+        path = get_raw_path(self.instrument, self.detector, self.group, self.snap, self.exposure, self.filter)
+        fits_path = ResourcePath(f"s3://{self.bucket}").join(path)
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", "S3 does not support flushing objects", UserWarning)
+            with fits_path.open("wb"):
+                pass  # Empty file is just fine
+
+        oid = check_for_snap(boto3.client("s3"),
+                             self.bucket,
+                             instrument=self.instrument,
+                             microservice="",
+                             group=self.group,
+                             snap=self.snap,
+                             detector=self.detector,
+                             )
+        self.assertEqual(oid, None)
+
+    def test_check_for_snap_absent(self):
+        microservice = "http://fake_host/fake_app"
+        message = {"error": False, "present": False}
+
+        with unittest.mock.patch("requests.get", **{"return_value.json.return_value": message}):
+            oid = check_for_snap(boto3.client("s3"),
+                                 self.bucket,
+                                 instrument=self.instrument,
+                                 microservice=microservice,
+                                 group=self.group,
+                                 snap=self.snap,
+                                 detector=self.detector,
+                                 )
+        self.assertEqual(oid, None)
+
+    def test_check_for_snap_error(self):
+        microservice = "http://fake_host/fake_app"
+        error_msg = "Microservice on strike"
+        message = {"error": True, "message": error_msg}
+
+        with unittest.mock.patch("requests.get", **{"return_value.json.return_value": message}), \
+                self.assertLogs(level="WARNING") as recorder:
+            oid = check_for_snap(boto3.client("s3"),
+                                 self.bucket,
+                                 instrument=self.instrument,
+                                 microservice=microservice,
+                                 group=self.group,
+                                 snap=self.snap,
+                                 detector=self.detector,
+                                 )
+        self.assertEqual(oid, None)
+        self.assertTrue(any(error_msg in line for line in recorder.output))
+
 
 class LatissTest(LsstBase, unittest.TestCase):
     def setUp(self):
         self.instrument = "LATISS"
         self.detector = 0
+        self.detector_name = "R00_S00"
         self.snap = 0
         self.exposure = 2022032100002
         super().setUp()
@@ -157,6 +231,7 @@ class LsstComCamTest(LsstBase, unittest.TestCase):
     def setUp(self):
         self.instrument = "LSSTComCam"
         self.detector = 4
+        self.detector_name = "R22_S11"
         self.snap = 1
         self.exposure = 2022032100003
         super().setUp()
@@ -174,6 +249,7 @@ class LsstCamTest(LsstBase, unittest.TestCase):
     def setUp(self):
         self.instrument = "LSSTCam"
         self.detector = 42
+        self.detector_name = "R11_S20"
         self.snap = 0
         self.exposure = 2022032100004
         super().setUp()