Merge pull request #969 from scientist-softserv/i852-new-worker-to-run-resource-intensive-jobs

bkiahstroud · web-flow · commit 5b6c954e5fe9 · 2024-02-13T17:17:37.000-08:00
i852 New worker to run resource intensive jobs
diff --git a/.env b/.env
@@ -1,3 +1,4 @@
+AUXILIARY_WORKER_THREAD_COUNT=1
 CAPYBARA_SERVER=web
 CHROME_HOSTNAME=seleniarm-hub
 DB_ADAPTER=postgresql
diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb
@@ -2,15 +2,15 @@
 
 # CreateLargeDerivativesJob is intended to be used for resource-intensive derivative
 # generation (e.g. video processing). It is functionally similar to CreateDerivativesJob,
-# except that it queues jobs in the :resource_intensive queue.
+# except that it queues jobs in the :auxiliary queue.
 #
-# The worker responsible for processing jobs in the :resource_intensive queue should be
+# The worker responsible for processing jobs in the :auxiliary queue should be
 # configured to have more resources dedicated to it, especially CPU. Otherwise, the
 # `ffmpeg` commands that this job class eventually triggers could be throttled.
 #
 # @see CreateDerivativesJobDecorator
 # @see Hydra::Derivatives::Processors::Ffmpeg
 # @see https://github.com/scientist-softserv/palni-palci/issues/852
 class CreateLargeDerivativesJob < CreateDerivativesJob
-  queue_as :resource_intensive
+  queue_as :auxiliary
 end
diff --git a/bin/helm_deploy b/bin/helm_deploy
@@ -23,7 +23,7 @@ WORKER_IMAGE="${WORKER_IMAGE:-ghcr.io/samvera/hyku/worker}"
 DEPLOY_TAG="${DEPLOY_TAG:-latest}"
 WORKER_TAG="${WORKER_TAG:-$DEPLOY_TAG}"
 
-helm pull oci://ghcr.io/samvera/charts/hyrax --version 2.0.0 --untar --untardir charts
+helm pull oci://ghcr.io/samvera/charts/hyrax --version 3.5.1 --untar --untardir charts
 
 helm repo update
 
diff --git a/bin/worker b/bin/worker
@@ -9,4 +9,8 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec sidekiq"
+if ENV['SIDEKIQ_CONFIG']
+  exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}"
+else
+  exec "echo $DATABASE_URL && bundle exec sidekiq"
+end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
@@ -3,4 +3,3 @@
   - default
   - import
   - export
-  - resource_intensive
diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml
@@ -0,0 +1,7 @@
+---
+:concurrency: <%= ENV['AUXILIARY_WORKER_THREAD_COUNT'] %>
+:queues:
+  - auxiliary
+  - default
+  - import
+  - export
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,6 +20,35 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  build:
+    context: .
+    target: hyku-worker
+    args:
+      - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
+      - HYKU_BULKRAX_ENABLED=true
+    cache_from:
+      - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
+      - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  command: sh -l -c 'bundle && bundle exec sidekiq'
+  depends_on:
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    zoo:
+      condition: service_started
+
 volumes:
   assets:
   cache:
@@ -145,6 +174,8 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_auxiliary:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     # ports:
@@ -153,33 +184,11 @@ services:
       - 3000
 
   worker:
-    <<: *app
-    build:
-      context: .
-      target: hyku-worker
-      args:
-        - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
-        - HYKU_BULKRAX_ENABLED=true
-      cache_from:
-        - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
-        - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    command: sh -l -c 'bundle && bundle exec sidekiq'
-    depends_on:
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      zoo:
-        condition: service_started
+    <<: *app-worker
+
+  worker_auxiliary:
+    <<: *app-worker
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_auxiliary.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
@@ -47,14 +47,15 @@ ingress:
     - host: commons-archive.org
       paths:
         - path: /
+          pathType: ImplementationSpecific
     - host: "*.commons-archive.org"
       paths:
         - path: /
-  annotations: {
-    kubernetes.io/ingress.class: "nginx",
-    nginx.ingress.kubernetes.io/proxy-body-size: "0",
-    cert-manager.io/cluster-issuer: letsencrypt-production-dns
-  }
+          pathType: ImplementationSpecific
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    cert-manager.io/cluster-issuer: "letsencrypt-production-dns"
   tls:
     - hosts:
         - commons-archive.org
@@ -191,6 +192,167 @@ worker:
     runAsGroup: 101
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
+
+# When adding/removing key-value pairs to this block, ensure the
+# corresponding changes are made in the `extraDeploy` block below.
+workerAuxiliary:
+  replicaCount: 1
+  resources:
+    limits:
+      memory: "12Gi"
+      cpu: "4"
+    requests:
+      memory: "4Gi"
+      cpu: "2"
+  extraEnvVars:
+    - name: AUXILIARY_WORKER_THREAD_COUNT
+      value: "1"
+    - name: SIDEKIQ_CONFIG
+      value: "config/sidekiq_auxiliary.yml"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-auxiliary-worker
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.workerAuxiliary.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101
@@ -208,11 +370,16 @@ fcrepo:
 postgresql:
   enabled: false
 redis:
-  cluster:
-    enabled: false
-  password: $REDIS_PASSWORD
+  enabled: true
+  architecture: standalone
+  auth:
+    password: $REDIS_PASSWORD
 solr:
   enabled: false
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
 
 externalPostgresql:
   host: postgres-cluster-alpha-ha.postgres.svc.cluster.local
@@ -226,7 +393,7 @@ externalSolrCollection: demo-palni-palci
 externalSolrPassword: $SOLR_ADMIN_PASSWORD
 
 global:
-  hyraxName: palni-palci-demo-pals
+  hyraxHostName: palni-palci-demo-pals
 
 nginx:
   enabled: true
@@ -238,7 +405,7 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxName }};
+      server {{ .Values.global.hyraxHostName }};
     }
 
     map ${DOLLAR}status ${DOLLAR}loggable {
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml
diff --git a/spec/jobs/create_large_derivatives_job_spec.rb b/spec/jobs/create_large_derivatives_job_spec.rb

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+AUXILIARY_WORKER_THREAD_COUNT=1`
`1`	`2`	`CAPYBARA_SERVER=web`
`2`	`3`	`CHROME_HOSTNAME=seleniarm-hub`
`3`	`4`	`DB_ADAPTER=postgresql`
-Original file line number
+Diff line change
   - default
   - import
   - export
 -  - resource_intensive