OSN-680. Import full previous month on the first import of the current month for AWS

Author: nk-hystax

diworker/diworker/importers/aws.py

@@ -11,7 +11,9 @@
 from datetime import datetime, timedelta, timezone
 from functools import cached_property
 
-from diworker.diworker.importers.base import CSVBaseReportImporter
+from diworker.diworker.importers.base import (
+    CSVBaseReportImporter, CSV_REWRITE_DAYS
+)
 import tools.optscale_time as opttime
 import pyarrow.parquet as pq
 
@@ -175,9 +177,22 @@ def get_update_fields(self):
             '_rec_n'
         ]
 
+    @staticmethod
+    def _is_first_import_in_month(last_import_dt: datetime):
+        now = opttime.utcnow()
+        if (last_import_dt.month + 1 == now.month and
+                last_import_dt.year == now.year) or (
+                    now.month == 1 and last_import_dt.year + 1 == now.year):
+            return True
+
     def get_current_reports(self, reports_groups, last_import_modified_at):
         current_reports = defaultdict(list)
         reports_count = 0
+        # during first report in the current month download all reports
+        # from the previous month to do full reimport
+        if self._is_first_import_in_month(last_import_modified_at):
+            last_import_modified_at = opttime.startmonth(
+                last_import_modified_at)
         for date, reports in reports_groups.items():
             for report in reports:
                 if report.get('LastModified', -1) > last_import_modified_at:
@@ -188,6 +203,18 @@ def get_current_reports(self, reports_groups, last_import_modified_at):
         LOG.info('Selected %s reports', reports_count)
         return current_reports
 
+    @cached_property
+    def min_date_import_threshold(self) -> datetime:
+        last_import_dt = datetime.fromtimestamp(
+            self.cloud_acc.get('last_import_modified_at', 0), tz=timezone.utc)
+        last_import_dt = last_import_dt.replace(
+            hour=0, minute=0, second=0, microsecond=0
+        )
+        if self._is_first_import_in_month(last_import_dt):
+            # import full previous month on the first import in month
+            return last_import_dt.replace(day=1)
+        return last_import_dt - timedelta(days=CSV_REWRITE_DAYS)
+
     def get_raw_upsert_filters(self, expense):
         filters = super().get_raw_upsert_filters(expense)
         filters.update({

tools/optscale_time/optscale_time.py

@@ -16,3 +16,7 @@ def utcfromtimestamp(timestamp) -> datetime:
 
 def startday(date) -> datetime:
     return date.replace(hour=0, minute=0, second=0, microsecond=0)
+
+
+def startmonth(date) -> datetime:
+    return date.replace(day=1, hour=0, minute=0, second=0, microsecond=0)