Merge pull request #930 from emcgrady/master

Merging. We can work to turn it on.

Author: ericvaandering

docker/wf-dashboard/Dockerfile

@@ -0,0 +1,17 @@
+# syntax=docker/dockerfile:1
+FROM almalinux:9
+ENV WDIR=/data
+RUN dnf -y install python3 python3-pip
+RUN pip install opensearch-py pandas pycurl rucio-clients tqdm
+RUN mkdir $WDIR 
+WORKDIR $WDIR
+ENV X509_USER_PROXY=/opt/proxy
+RUN dnf -y install git
+RUN git clone https://github.com/emcgrady/WFDashboard.git
+RUN git clone https://github.com/dmwm/CMSSpark.git
+RUN pip install -r CMSSpark/requirements.txt
+ENV PYTHONPATH="${PYTHONPATH}:CMSSpark/src/python/CMSSpark"
+RUN cp WFDashboard/monit_pull.py $WDIR
+ADD rucio.cfg /opt/rucio/etc/
+RUN chmod +x monit_pull.py
+ENTRYPOINT ["python3", "monit_pull.py"]

docker/wf-dashboard/build_image.sh

@@ -0,0 +1,4 @@
+#! /bin/sh
+
+podman build . -t registry.cern.ch/cmsrucio/wf-dashboard:release-0.9
+podman push registry.cern.ch/cmsrucio/wf-dashboard:release-0.9

docker/wf-dashboard/rucio.cfg

@@ -0,0 +1,17 @@
+# Copyright European Organization for Nuclear Research (CERN)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Authors:
+# - Vincent Garonne, <vgaronne@gmail.com>, 2017
+# - Eric Vaandering, <ewv@fnal.gov>, 2018
+
+[common]
+[client]
+rucio_host = http://cms-rucio.cern.ch
+auth_host = https://cms-rucio-auth.cern.ch
+auth_type = x509_proxy
+ca_cert = /cvmfs/cms.cern.ch/grid/etc/grid-security/certificates/
+request_retries = 3

helm/rucio-cron-jobs/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.2.1
+version: 3.0.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/rucio-cron-jobs/templates/cronjobs.yaml

@@ -123,3 +123,41 @@ spec:
 {{- end }}
               command: ['/root/CMSRucio/docker/rucio_client/scripts/k8s_sync_users_links.sh']
           restartPolicy: Never
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Release.Name }}-wf-dashboard
+spec:
+  schedule: "20 4 * * *"
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 7200
+      backoffLimit: 0
+      template:
+        spec:
+          volumes:
+          - name: proxy-volume
+            secret:
+              defaultMode: 420
+              secretName: server-rucio-x509up
+          {{- range $key, $val := .Values.persistentVolumes }}
+          - name: {{ $key }}
+            persistentVolumeClaim:
+              claimName: {{ $val.name }}
+          {{- end}}
+          containers:
+            - name: {{ .Release.Name }}-wf-dashboard
+              image: registry.cern.ch/cmsrucio/wf-dashboard:release-0.9
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              volumeMounts:
+                - mountPath: /opt/proxy
+                  name: proxy-volume
+              env: 
+                - name: X509_USER_PROXY
+                  value: "/opt/proxy"
+{{- with .Values.cronjobSettings.additionalEnvs }}
+{{ toYaml . | indent 16 }}
+{{- end }}
+              command: ['python3', 'monit_pull.py']
+          restartPolicy: Never

helm/rucio-cron-jobs/values.yaml

@@ -90,3 +90,7 @@ nodeSelector: {}
 tolerations: []
 
 affinity: {}
+
+wfDashboard: false
+  # when true, the workflow dashboard will be run in integration and production
+  # when false, the workflow dashboard will only run in production