Remove --delete-source option from consolidate scripts.

Author: ropable

consolidator.py

@@ -8,7 +8,7 @@
 import unicodecsv as csv
 from dotenv import load_dotenv
 
-from utils import configure_logging, delete_logs, download_logs, upload_log
+from utils import configure_logging, download_logs, upload_log
 
 # Load environment variables.
 load_dotenv()
@@ -103,9 +103,8 @@ def consolidate_logs(
     container_src="access-logs-json",
     container_dest="access-logs",
     container_dest_errors="error-logs",
-    delete_source=False,
 ):
-    """Download logs for the specified timestamp and services, consolidate, upload and optionally delete the source logs."""
+    """Download logs for the specified timestamp and services, consolidate and upload the source logs."""
     if SENTRY_CRON_URL:
         LOGGER.info("Signalling Sentry monitor (in progress)")
         requests.get(f"{SENTRY_CRON_URL}?status=in_progress")
@@ -124,9 +123,6 @@ def consolidate_logs(
         out_log_errors = consolidate_json_errors(timestamp, temp_dir.name, temp_dir.name)
         # Upload consolidated CSV errors to blob storage.
         upload_log(out_log_errors, container_dest_errors, CONN_STR, slow_connection=True)
-        # Optionally deleting JSON logs from blob storage.
-        if delete_source:
-            delete_logs(timestamp, hosts, container_src, CONN_STR, True)
 
     if SENTRY_CRON_URL:
         LOGGER.info("Signalling Sentry monitor (completed)")
@@ -175,18 +171,11 @@ def consolidate_logs(
         action="store",
         required=False,
     )
-    parser.add_argument(
-        "--delete-source",
-        help="Delete the source CSV after processing (optional)",
-        action="store_true",
-        required=False,
-    )
     args = parser.parse_args()
     consolidate_logs(
         timestamp=args.timestamp,
         hosts=args.hosts,
         container_src=args.container,
         container_dest=args.destination_container,
         container_dest_errors=args.destination_container_errors,
-        delete_source=args.delete_source,
     )

consolidator_fastly.py

@@ -5,7 +5,7 @@
 import unicodecsv as csv
 from dotenv import load_dotenv
 
-from utils import configure_logging, delete_logs, download_logs, upload_log
+from utils import configure_logging, download_logs, upload_log
 
 # Load environment variables.
 load_dotenv()
@@ -47,8 +47,8 @@ def consolidate_logfiles(timestamp, source_dir, destination_dir):
     return out_log
 
 
-def consolidate_fastly_logs(timestamp, services, container_src="fastly", container_dest="fastly", delete_source=False):
-    """Download logs for the specified timestamp and services, consolidate, upload and optionally delete the source logs."""
+def consolidate_fastly_logs(timestamp, services, container_src="fastly", container_dest="fastly"):
+    """Download logs for the specified timestamp and services, consolidate and upload the source logs."""
     # Use a temporary directory to download logs.
     temp_dir = TemporaryDirectory()
     # Download Fastly logs.
@@ -58,9 +58,6 @@ def consolidate_fastly_logs(timestamp, services, container_src="fastly", contain
         out_log = consolidate_logfiles(timestamp, temp_dir.name, temp_dir.name)
         # Upload consolidated CSV log to blob storage.
         upload_log(out_log, container_dest, CONN_STR)
-        # Optionally deleting source logs from blob storage.
-        if delete_source:
-            delete_logs(timestamp, services, container_src, CONN_STR)
 
 
 if __name__ == "__main__":
@@ -97,17 +94,10 @@ def consolidate_fastly_logs(timestamp, services, container_src="fastly", contain
         action="store",
         required=False,
     )
-    parser.add_argument(
-        "--delete-source",
-        help="Delete the source logs after processing (optional, default false)",
-        action="store_true",
-        required=False,
-    )
     args = parser.parse_args()
     consolidate_fastly_logs(
         timestamp=args.timestamp,
         services=args.services,
         container_src=args.container,
         container_dest=args.destination_container,
-        delete_source=args.delete_source,
     )

utils.py

@@ -96,37 +96,6 @@ def download_logs(
     return True
 
 
-def delete_logs(timestamp, hosts, container_name, conn_str, nginx_host_log=False, enable_logging=True):
-    """Given the passed in timestamp and hosts list, delete blobs from the container."""
-    if enable_logging:
-        logger = logging.getLogger()
-
-    container_client = ContainerClient.from_connection_string(conn_str, container_name)
-    log_list = []
-    hosts_list = hosts.split(",")
-
-    for host in hosts_list:
-        if nginx_host_log:
-            blob_list = container_client.list_blobs(name_starts_with=f"{host}/nginx_access.{timestamp}")
-        else:
-            blob_list = container_client.list_blobs(name_starts_with=f"{host}/{timestamp}")
-        log_list += [b for b in blob_list]
-
-    for blob in log_list:
-        blob_client = BlobClient.from_connection_string(conn_str, container_name, blob.name)
-        if enable_logging:
-            logger.info(f"Deleting blob {blob.name} from {container_name} container")
-        try:
-            blob_client.delete_blob()
-        except Exception as e:
-            if enable_logging:
-                logger.error(f"Exception during deletion of {blob.name}, aborting")
-                logger.exception(e)
-            return
-
-    return True
-
-
 def upload_log(source_path, container_name, conn_str, overwrite=True, enable_logging=True, blob_name="", slow_connection=False):
     """Upload a single log at `source_path` to Azure blob storage (`blob_name` destination name is optional)."""
     if not blob_name: