Add secret manager in weather-dl.

Configuration.md

@@ -198,16 +198,15 @@ target_template=gs://ecmwf-downloads/hres-single-level/{}.nc
 partition_keys=
     date
 [parameters.deepmind]
-api_key=KKKKK1
-api_url=UUUUU1
+secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1
 [parameters.research]
-api_key=KKKKK2
-api_url=UUUUU2
+secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1
 [parameters.cloud]
-api_key=KKKKK3
-api_url=UUUUU3
+secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1
 ```
 
+Note: Here, secret_key is the [secret-manager](https://cloud.google.com/secret-manager) key with value like this: {"api_url": "URL", "api_key": "KEY"}
+
 ## `selection` Section
 
 _Parameters used to select desired data_

setup.py

@@ -28,6 +28,7 @@
     "google-cloud-spanner==1.19.3",
     "google-cloud-videointelligence==1.16.3",
     "google-cloud-vision==1.0.2",
+    "google-cloud-secret-manager==2.16.2",
     "apache-beam[gcp]==2.40.0",
 ]
 

weather_dl/download_pipeline/parsers.py

@@ -23,6 +23,7 @@
 import typing as t
 import numpy as np
 from collections import OrderedDict
+from google.cloud import secretmanager
 from urllib.parse import urlparse
 
 from .clients import CLIENTS
@@ -460,6 +461,23 @@ def prepare_target_name(config: Config) -> str:
     return target
 
 
+def get_secret(secret_key: str) -> t.Dict:
+    """Retrieve the secret value from the Google Cloud Secret Manager.
+
+    Parameters:
+        secret_key (str): The name or identifier of the secret in the Google
+                        Cloud Secret Manager.
+
+    Returns:
+        dict: A dictionary containing the retrieved secret data.
+    """
+    client = secretmanager.SecretManagerServiceClient()
+    response = client.access_secret_version(request={"name": secret_key})
+    payload = response.payload.data.decode("UTF-8")
+    secret_dict = json.loads(payload)
+    return secret_dict
+
+
 def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]:
     """Collect parameter subsections from main configuration.
 
@@ -471,17 +489,14 @@ def get_subsections(config: Config) -> t.List[t.Tuple[str, t.Dict]]:
     For example:
     ```
       [parameters.alice]
-      api_key=KKKKK1
-      api_url=UUUUU1
+      secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1
       [parameters.bob]
-      api_key=KKKKK2
-      api_url=UUUUU2
+      secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1
       [parameters.eve]
-      api_key=KKKKK3
-      api_url=UUUUU3
+      secret_key=projects/PROJECT_NAME/secrets/SECRET_NAME/versions/1
     ```
     """
-    return [(name, params) for name, params in config.kwargs.items()
+    return [(name, get_secret(params.get('secret_key'))) for name, params in config.kwargs.items()
             if isinstance(params, dict)] or [('default', {})]
 
 

weather_dl/download_pipeline/parsers_test.py

@@ -13,11 +13,14 @@
 # limitations under the License.
 import datetime
 import io
+import json
 import unittest
+from unittest.mock import patch, MagicMock
 
 from .manifest import MockManifest, Location
 from .parsers import (
     date,
+    get_secret,
     parse_config,
     process_config,
     _number_of_replacements,
@@ -538,6 +541,29 @@ def test_cfg_parses_parameter_subsections(self):
                 },
             })
 
+    @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient")
+    def test_get_secret_success(self, mock_secretmanager):
+        secret_data = {
+            "api_url": "https://example.com/api",
+            "api_key": "my_secret_api_key"
+        }
+        mock_response = MagicMock()
+        mock_response.payload.data.decode.return_value = json.dumps(secret_data)
+        mock_secretmanager.return_value.access_secret_version.return_value = (
+            mock_response)
+
+        api_key = "projects/my-project/secrets/my-secret/versions/latest"
+        result = get_secret(api_key)
+        self.assertEqual(result, secret_data)
+
+    @patch("weather_dl.download_pipeline.parsers.secretmanager.SecretManagerServiceClient")
+    def test_get_secret_failure(self, mock_secretmanager):
+        mock_secretmanager.return_value.access_secret_version.side_effect = (
+            Exception("Error retrieving secret"))
+        api_key = "projects/my-project/secrets/my-secret/versions/latest"
+        with self.assertRaises(Exception):
+            get_secret(api_key)
+
 
 class HelpersTest(unittest.TestCase):
 

weather_dl/setup.py

@@ -28,6 +28,7 @@
     "google-cloud-spanner==1.19.3",
     "google-cloud-videointelligence==1.16.3",
     "google-cloud-vision==1.0.2",
+    "google-cloud-secret-manager==2.16.2",
     "apache-beam[gcp]==2.40.0",
 ]