Merge pull request #237 from ARGA-Genomes/alaBiocache

ALA Biocache

Author: christopher-mangion

dataSources/ala/avh/config.json

@@ -0,0 +1,50 @@
+{
+    "datasetID": "0007001",
+    "retrieveType": "script",
+    "downloading": {
+        "path": "./processing.py",
+        "function": "collectBiocache",
+        "args": [
+            {
+                "q": "*:*",
+                "fq": "(basis_of_record:\"PRESERVED_SPECIMEN\" OR basis_of_record:\"MATERIAL_SAMPLE\" OR basis_of_record:\"LIVING_SPECIMEN\" OR basis_of_record:\"MATERIAL_CITATION\")",
+                "disableAllQualityFilters": true,
+                "qualityProfile": "ALA",
+                "qc": "-_nest_parent_:*"
+            },
+            "{OUT-PATH}"
+        ],
+        "output": "biocache.zip"
+    },
+    "processing": {
+        "linear": [
+            {
+                "path": ".lib/zipping.py",
+                "function": "extract",
+                "args": [
+                    "{IN-PATH}",
+                    "{OUT-DIR}"
+                ],
+                "output": "biocache"
+            },
+            {
+                "path": "./processing.py",
+                "function": "cleanup",
+                "args": [
+                    "{IN-PATH}",
+                    "{OUT-PATH}"
+                ],
+                "output": "biocache.csv"
+            }
+        ]
+    },
+    "conversion": {
+        "mapColumnName": "ala-biocache mappings"
+    },
+    "update": {
+        "type": "weekly",
+        "day": "sunday",
+        "time": 9,
+        "repeat": 2
+    }
+}

dataSources/ala/avh/scripts/processing.py

@@ -0,0 +1,62 @@
+import requests
+from pathlib import Path
+import lib.downloading as dl
+import logging
+from lib.secrets import secrets
+import lib.bigFiles as bf
+
+def collectBiocache(queryParamters: dict, outputFilePath: Path) -> None:
+    paramters = {
+        "email": secrets.general.email,
+        "emailNotify": False
+    }
+
+    baseURL = "https://api.ala.org.au/occurrences/occurrences/offline/download?"
+    url = dl.urlBuilder(baseURL, paramters | queryParamters)
+
+    response = requests.get(url)
+    data = response.json()
+
+    statusURL = data["statusUrl"]
+    totalRecords = data["totalRecords"]
+    logging.info(f"Found {totalRecords} total records")
+
+    dl.asyncRunner(statusURL, "status", "finished", "downloadUrl", outputFilePath)
+
+def cleanup(folderPath: Path, outputFilePath: Path) -> None:
+    extraFiles = [
+        "citation.csv",
+        "headings.csv",
+        "README.html"
+    ]
+
+    for fileName in extraFiles:
+        path = folderPath / fileName
+        path.unlink(missing_ok=True)
+
+    bf.combineDirectoryFiles(outputFilePath, folderPath)
+
+# status = {
+#     "inQueue": [
+#         "totalRecords",
+#         "queueSize",
+#         "statusUrl",
+#         "cancelUrl",
+#         "searchUrl"
+#     ],
+#     "running": [
+#         "totalRecords",
+#         "records",
+#         "statusUrl",
+#         "cancelUrl",
+#         "searchUrl"
+#     ],
+#     "finished": [
+#         "totalRecords",
+#         "queueSize",
+#         "downloadUrl",
+#         "statusUrl",
+#         "cancelUrl",
+#         "searchUrl"
+#     ]
+# }

dataSources/ala/biocache/config.json

@@ -4,6 +4,8 @@
 from requests.exceptions import HTTPError
 import logging
 from lib.progressBar import ProgressBar
+from urllib.parse import quote
+import time
 
 class RepeatDownloader:
     def __init__(self, headers: dict = {}, username: str = "", password: str = "", chunkSize: int = 1024*1024, verbose: bool = False):
@@ -57,3 +59,63 @@ def download(url: str, filePath: Path, chunkSize: int = 1024*1024, verbose: bool
                     print(f"Downloaded chunk: {idx}", end="\r")
 
     return True
+
+def urlBuilder(url: str, parameters: dict) -> str:
+    def encode(key: str, value: any) -> str:
+        if isinstance(value, bool):
+            value = str(value).lower()
+
+        if isinstance(value, int):
+            value = str(value)
+
+        return f"{key}={quote(value)}"
+
+    flatParams = []
+    for paramterName, parameterValue in parameters.items():
+        if isinstance(parameterValue, list):
+            for item in parameterValue:
+                flatParams.append(encode(paramterName, item))
+            continue
+
+        flatParams.append(encode(paramterName, parameterValue))
+
+    return f"{url}" + "&".join(flatParams)
+
+def asyncRunner(checkURL: str, statusField: str, completedStr: str, downloadField: str, outputFilePath: Path, recheckDelay: int = 10) -> bool:
+    session = requests.Session()
+
+    def getCompleted() -> tuple[bool, str, str]:
+        response = session.get(checkURL)
+        if response.status_code != 200:
+            logging.warning(f"Failed to retrieve {checkURL}, received status code {response.status_code}. Reason: {response.reason}")
+            return True, None, None
+        
+        data = response.json()
+
+        statusValue = data.get(statusField, "Unknown")
+        downloadURL = data.get(downloadField, "")
+
+        return statusValue == completedStr, statusValue, downloadURL
+    
+    loading = "|/-\\"
+    totalChecks = 0
+    recheckDelay = max(recheckDelay, 5)
+    reprintsPerSecond = 2
+
+    logging.info(f"Polling {checkURL} for status...")
+    completed, status, downloadURL = getCompleted()
+    while not completed:
+        for _ in range(recheckDelay):
+            for _ in range(reprintsPerSecond):
+                print(f"> ({loading[totalChecks % len(loading)]}) Status: {status}", end="\r")
+                time.sleep(1 / reprintsPerSecond)
+
+            totalChecks += 1
+
+        completed, status, downloadURL = getCompleted()
+
+    if status is None:
+        logging.error("Failed to check status of download.")
+        return False
+
+    return download(downloadURL, outputFilePath, verbose=True)