Merge pull request #166 from ARGA-Genomes/conversionTimestamp

Conversion timestamp

Author: christopher-mangion

src/lib/data/database.py

@@ -176,6 +176,8 @@ def update(self) -> bool:
         for step in (Step.DOWNLOAD, Step.PROCESSING, Step.CONVERSION):
             self.create(step, (True, True), True)
 
+        self.package()
+
     def _verboseFlags(self, flags: list[Flag]) -> str:
         return " | ".join(f"{flag.value}={flag in flags}" for flag in Flag)
 

src/lib/systemManagers/baseManager.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 import time
 from enum import Enum
+from typing import Any
 
 class Metadata(Enum):
     OUTPUT = "output"
@@ -96,8 +97,13 @@ def updateTotalTime(self, totalTime: float) -> None:
         self.metadata[Metadata.TOTAL_TIME.value] = totalTime
         self._syncMetadata()
 
-    def getLastUpdate(self) -> datetime | None:
+    def getMetadata(self, stepIndex: int, property: Metadata) -> None | Any:
         if not self.metadata:
             return None
 
-        return datetime.fromisoformat(self.metadata[self.taskName][0][Metadata.TIMESTAMP.value])
+        return self.metadata[self.taskName][stepIndex][property.value]
+
+    def getLastUpdate(self) -> datetime | None:
+        timestamp = self.getMetadata(0, Metadata.TIMESTAMP)
+        if timestamp is not None:
+            return datetime.fromisoformat(timestamp)

src/lib/systemManagers/conversion.py

@@ -1,5 +1,4 @@
 import pandas as pd
-import numpy as np
 from pathlib import Path
 import lib.common as cmn
 from lib.bigFileWriter import BigFileWriter
@@ -10,7 +9,7 @@
 import logging
 import gc
 import time
-from datetime import datetime
+from datetime import datetime, date
 import lib.zipping as zp
 
 class ConversionManager(SystemManager):
@@ -20,12 +19,23 @@ def __init__(self, dataDir: Path, datasetID: str, location: str, database: str,
         super().__init__(dataDir.parent, self.stepName, "tasks")
 
         self.conversionDir = dataDir / self.stepName
-        self.location = location
+
         self.datasetID = datasetID
+        self.location = location
+        self.database = database
+        self.subsection = subsection
 
-        self.output = StackedFile(self.conversionDir / (f"{location}-{database}" + (f"-{subsection}" if subsection else "")))
+        self.file = None
+
+    def _generateFileName(self, withTimestamp: bool) -> StackedFile:
+        sourceName = f"{self.location}-{self.database}"
+        if self.subsection:
+            sourceName += f"-{self.subsection}"
+        
+        if withTimestamp:
+            sourceName += date.today().strftime("-%Y-%m-%d")
 
-        self.fileLoaded = False
+        return StackedFile(self.conversionDir / sourceName)
 
     def loadFile(self, file: File, properties: dict, mapDir: Path) -> None:
         self.file = file
@@ -40,13 +50,16 @@ def loadFile(self, file: File, properties: dict, mapDir: Path) -> None:
         self.skipRemap = properties.pop("skipRemap", [])
         self.preserveDwC = properties.pop("preserveDwC", False)
         self.prefixUnmapped = properties.pop("prefixUnmapped", True)
+
+        timestamp = properties.pop("timestamp", True)
+        self.output = self._generateFileName(timestamp)
+
         self.augments = [FunctionScript(self.baseDir, augProperties) for augProperties in properties.pop("augment", [])]
 
         self.remapper = Remapper(mapDir, self.mapID, self.customMapID, self.customMapPath, self.location, self.preserveDwC, self.prefixUnmapped)
-        self.fileLoaded = True
 
     def convert(self, overwrite: bool = False, verbose: bool = True, ignoreRemapErrors: bool = True, forceRetrieve: bool = False) -> bool:
-        if not self.fileLoaded:
+        if self.file is None:
             logging.error("No file loaded for conversion, exiting...")
             return False
 
@@ -145,8 +158,14 @@ def applyAugments(self, df: pd.DataFrame) -> pd.DataFrame | None:
 
         return df
 
-    def package(self, compressLocation: Path) -> Path:
-        renamedFile = self.metadataPath.rename(self.output.filePath / self.metadataPath.name)
-        outputPath = zp.compress(self.output.filePath, compressLocation)
+    def package(self, compressLocation: Path) -> Path | None:
+        outputFileName = self.getMetadata(-1, Metadata.OUTPUT)
+        if outputFileName is None:
+            return
+        
+        outputFilePath = self.conversionDir / outputFileName
+        renamedFile = self.metadataPath.rename(outputFilePath / self.metadataPath.name)
+        outputPath = zp.compress(outputFilePath, compressLocation)
         renamedFile.rename(self.metadataPath)
+        
         return outputPath