Initial version of asking approval feature

Removed deprecated HomePageView

Author: juanpablosalas

DMOps/file_invalidation_server/fi_manager/models.py

@@ -18,7 +18,10 @@ class FileInvalidationRequests(models.Model):
     reason = models.TextField(blank=True, null=True)
     job_id = models.CharField(max_length=8,null=True,blank=True)
     logs = models.TextField()
+    rse = models.TextField()
     global_invalidate_last_replicas = models.BooleanField(default=False)
+    request_user = models.TextField()
+    approve_user = models.TextField()
 
     class Meta:
         managed = False

DMOps/file_invalidation_server/fi_manager/process_jobs.py

@@ -73,11 +73,10 @@ def fetch_and_process():
 
             delete_opts = client.V1DeleteOptions(propagation_policy='Foreground')
 
-            #batch_v1.delete_namespaced_job(
-            #        name=job_name,
-            #        namespace=namespace,
-            #           body=delete_opts)
-            logger.info(f"Job {job_name} would be deleted but is being kept for dev purposes.")
+            batch_v1.delete_namespaced_job(
+                    name=job_name,
+                    namespace=namespace,
+                       body=delete_opts)
 
 def parse_job_logs(logs: str):
     if "Error running shell script" in logs:

DMOps/file_invalidation_server/fi_manager/templates/core/home.html

@@ -1,8 +1,9 @@
 from django.urls import path
-from .views import FileInvalidationRequestsView, FileQueryView
+from .views import FileInvalidationRequestsView, FileQueryView, InvalidationApproval
 
 
 urlpatterns = [
     path("upload/", FileInvalidationRequestsView.as_view(), name="file-upload"),
     path("query/", FileQueryView.as_view(), name="file-query"),
+    path("approve/<request_id:request_id>", InvalidationApproval.as_view(), name="invalidation-approval"),
 ]

DMOps/file_invalidation_server/fi_manager/urls.py

@@ -12,8 +12,15 @@
 src_dir = os.path.dirname(current_dir)
 yaml_path = os.path.join(src_dir, 'controllers', 'job.yaml')
 
+def get_cern_username(request):
+        return (
+            request.META.get("HTTP_X_FORWARDED_PREFERRED_USERNAME")
+            or request.META.get("HTTP_X_FORWARDED_USER")
+            or request.META.get("REMOTE_USER")
+        )
 
-def process_invalidation(request_id, reason, dry_run=True,mode='global',rse=None,to_process='queued',global_invalidate_last_replicas=False):
+
+def process_invalidation(request_id, reason, dry_run=True,mode='global',rse=None,to_process='approved',global_invalidate_last_replicas=False):
     file_records = FileInvalidationRequests.objects.filter(request_id=request_id,status=to_process)
     if file_records.count()==0:
         raise ValueError(f'There are no records in the db for the request_id: {request_id}')

…_invalidation_server/fi_manager/tasks.py …_invalidation_server/fi_manager/utils.pyDMOps/file_invalidation_server/fi_manager/tasks.py renamed to DMOps/file_invalidation_server/fi_manager/utils.py DMOps/file_invalidation_server/fi_manager/tasks.py renamed to DMOps/file_invalidation_server/fi_manager/utils.py

@@ -5,13 +5,12 @@
 from .models import FileInvalidationRequests
 import base64
 from django.http import HttpResponse
-from .tasks import process_invalidation
+from .utils import process_invalidation, get_cern_username
 from django.views.generic import TemplateView
 from django.db.models import Count, CharField, Value, F, Case, When
 from django.db.models.functions import StrIndex, Substr
 import logging
 import uuid
-dids_limit = 1000
 
 class FileInvalidationRequestSerializer(serializers.Serializer):
     reason = serializers.CharField(required=True,help_text="Enter the reason for the file invalidation request.")
@@ -81,40 +80,32 @@ def post(self, request, *args, **kwargs):
 
         file_lines = file_lines.splitlines()
         request_id = uuid.uuid4()
+        logging.info({k: v for k, v in request.META.items() if k.startswith("HTTP_") or k == "REMOTE_USER"})
+        user = get_cern_username(request)
 
         cnt = 0
-        if len(file_lines)>dids_limit:
-            return Response({"message": f"Request aborted. The file exceeds DIDs limit ({len(file_lines)}>{dids_limit})"}, status=status.HTTP_400_BAD_REQUEST)
 
         raw_file_message = ""
         already_serviced_files = ""
         for fn in file_lines:
             fn = fn.strip()
             fn = fn.replace('cms:/store','/store')
             obj = FileInvalidationRequests.objects.filter(file_name=fn).filter(dry_run=False).first()
-            if '/RAW/' in fn:
-                raw_file_message = raw_file_message +f'\nRequest aborted for file {fn}. File contains RAW data.'
-                input_vals = {'request_id':request_id,'file_name':fn,'status':'aborted','mode':mode,'dry_run':dry_run,'reason':reason,'logs':f'Request aborted for file {fn}. File contains RAW data.'}
-                file_record = FileInvalidationRequests.objects.create(**input_vals)
-            elif obj:
+            if obj:
                 already_serviced_files = already_serviced_files + f'\nRequest was not created for file {fn} because it has already been submitted and it is currently in status: {obj.status}.'
                 if obj.status=="in_progress":
                     already_serviced_files = already_serviced_files + ' Please wait 30min for the CronJob to update it on the database'
+                elif obj.status=="waiting_approval":
+                    already_serviced_files = already_serviced_files + ' Please ask DMOps to approve the invalidation.'
             else:
-                input_vals = {'request_id':request_id,'file_name':fn,'status':'queued','mode':mode,'dry_run':dry_run,'reason':reason,'global_invalidate_last_replicas':global_invalidate_last_replicas}
+                input_vals = {'request_id':request_id,'file_name':fn,'status':'waiting_approval','mode':mode,'dry_run':dry_run,'reason':reason,'global_invalidate_last_replicas':global_invalidate_last_replicas,'request_user':user, 'rse': rse}
                 file_record = FileInvalidationRequests.objects.create(**input_vals)
                 cnt += 1
 
-        logging.info(f'{cnt} of {len(file_lines)} files were created in the database with queued status.')
+        logging.info(f'{cnt} of {len(file_lines)} files were created in the database with waiting_approval status.')
 
-        response_message = ""
-        if cnt>0:
-            logging.info(f'Processing request id {request_id}...')
-            response_message = process_invalidation(request_id, reason,dry_run=dry_run,mode=mode,rse=rse,to_process='queued',global_invalidate_last_replicas=global_invalidate_last_replicas)
-        else:
-            return Response({"message": f"None of the files could be invalidated. {raw_file_message}, {already_serviced_files}"}, status=status.HTTP_400_BAD_REQUEST)
 
-        response_message = response_message + raw_file_message + already_serviced_files
+        response_message = raw_file_message + already_serviced_files
 
         return Response({"message": response_message,
                          "redirect_url":f"https://file-invalidation.app.cern.ch/api/query/?request_id={request_id}",
@@ -174,9 +165,43 @@ def get(self, request, *args, **kwargs):
         )
 
 
-class HomePageView(TemplateView):
-    template_name = 'core/home.html'
+class InvalidationApproval(APIView):
+    
+    def get(self, request, request_id):
+        files = FileInvalidationRequests.objects.filter(request_id=request_id)
+
+        return Response(
+            [{"request_id": f.request_id, "file_name": f.file_name, "status": f.status,"mode":f.mode,"dry_run":f.dry_run,"reason":f.reason,"job_id":f.job_id,"logs":f.logs,"request_user":f.request_user} for f in files],
+            status=status.HTTP_200_OK
+        )
+
+    
+    def post(self, request, request_id):
+        files = FileInvalidationRequests.objects.filter(request_id=request_id)
+
+        if not files.exists():
+            return Response({"message":f"Files with request_id {request_id} not found"},
+                            status=status.HTTP_404_NOT_FOUND)
+        
+        approval_user = get_cern_username(request)
+        request_user = files.first().request_user
 
-    def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
-        return context
+        if approval_user==request_user:
+            return Response({"message":f"Approval user cannot be the same as request user"},
+                            status=status.HTTP_403_FORBIDDEN)
+        
+        updated = files.update(status="approved",approval_user=approval_user)
+        reason = updated.first().reason
+        dry_run = updated.first().dry_run
+        mode = updated.first().mode
+        rse = updated.first().rse
+        global_invalidate_last_replicas = updated.first().global_invalidate_last_replicas
+
+        try:
+            response_message = process_invalidation(request_id, reason, dry_run=dry_run, mode=mode, rse=rse,to_process="approved",global_invalidate_last_replicas=global_invalidate_last_replicas)
+            return Response({"message": response_message,
+                             "redirect_url":f"https://file-invalidation.app.cern.ch/api/query/?request_id={request_id}",
+                            "redirect_description":"View request_id details"}, status=status.HTTP_201_CREATED)
+
+        except Exception as e:
+            return Response({"message":f"None of the files could be invalidated - {str(e)}"})

DMOps/file_invalidation_server/fi_manager/views.py

@@ -16,10 +16,8 @@
 """
 from django.contrib import admin
 from django.urls import path, include
-from fi_manager.views import HomePageView
 
 urlpatterns = [
     path('admin/', admin.site.urls),
     path("api/", include("fi_manager.urls")),
-    path("", HomePageView.as_view(), name='home')
 ]