Merge pull request #238 from ARGA-Genomes/alaProfiles

Ala Profiles Update

Author: christopher-mangion

dataSources/ala/profiles/config.json

@@ -3,17 +3,16 @@
     "subsections": {
         "kamilaroi": {},
         "noongar": {},
-        "southeastarnhemland": {},
-        "mangrovewatch": {},
+        "southEastarnHemland": {},
+        "mangroveWatch": {},
         "weeds-australia": {}
     },
     "downloading": {
         "path": "./processing.py",
         "function": "collect",
         "args": [
             "{OUT-DIR}",
-            "<SUB>",
-            "../token.json"
+            "<SUB>"
         ],
         "output": "<SUB>.csv"
     },

dataSources/ala/profiles/scripts/processing.py

@@ -1,34 +1,46 @@
-import json
 import requests
+import pandas as pd
 from pathlib import Path
+from lib.secrets import secrets
+import lib.downloading as dl
+from lib.progressBar import ProgressBar
+import logging
 import lib.dataframes as dff
-import pandas as pd
 
-def collect(outputDir: Path, profile: str, tokenFilePath: Path) -> None:
-    with open(tokenFilePath) as fp:
-        token = json.load(fp)
+def collect(outputDir: Path, profile: str) -> None:
+    session = requests.session()
+
+    response = session.post(
+        "https://auth.ala.org.au/cas/oidc/oidcAccessToken",
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+        data="grant_type=client_credentials&scope=openid email profile roles",
+        auth=dl.buildAuth(secrets.ala.id, secrets.ala.secret)
+    )
+
+    accessToken = response.json()["access_token"]
 
-    bearerToken = token["access_token"]
     baseURL = "https://api.ala.org.au/profiles"
     endpoint = f"/api/opus/{profile}/profile?pageSize=1000"
-    response = requests.get(baseURL + endpoint, headers={"Authorization": f"Bearer {bearerToken}"})
+    response = requests.get(baseURL + endpoint, headers={"Authorization": f"Bearer {accessToken}"})
     data = response.json()
 
     if "message" in data and "not authorized" in data["message"]:
-        print("Failed to authorize, please make sure bearer token is valid.")
+        logging.error("Failed to authorize, please make sure bearer token is valid.")
         return
 
-    print(f"Accessing profile: {profile}")
+    logging.info(f"Accessing profile: {profile}")
+
+    dataLength = len(data)
+    logging.info(f"Found {dataLength} records")
 
+    progress = ProgressBar(dataLength)
     records = []
-    for idx, entry in enumerate(data, start=1):
+    for entry in data:
         uuid = entry["uuid"]
-        print(f"At record: {idx}", end="\r")
-
-        response = requests.get(baseURL + f"/api/opus/{profile}/profile/{uuid}", headers={"Authorization": f"Bearer {bearerToken}"})
+        response = requests.get(baseURL + f"/api/opus/{profile}/profile/{uuid}", headers={"Authorization": f"Bearer {accessToken}"})
         records.append(response.json())
-    print()
+        progress.update()
 
     df = pd.DataFrame.from_records(records)
     df = dff.removeSpaces(df)
-    df.to_csv(outputDir / f"{profile}.csv", index=False)
+    df.to_csv(outputDir / f"{profile}.csv", index=False)

src/lib/data/sources.py

@@ -25,7 +25,7 @@ def _buildSourceName(self, locationName: str, databaseName: str, subsectionName:
         return f"{locationName}{self._divider}{databaseName}" + (f"{self._divider}{subsectionName}" if subsectionName else "")
 
     def _splitSourceName(self, sourceName: str) -> tuple[str, str, str]:
-        sourceSections = sourceName.split(self._divider)
+        sourceSections = sourceName.split(self._divider, 2)
 
         if len(sourceSections) <= 1:
             return sourceName, "", ""