From 36755a97c08c5214f70b1e118de8c5e769b02198 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 12:25:56 -0800
Subject: [PATCH 01/21] new worker to run resource intensive jobs

---
 config/sidekiq.yml                    |  1 -
 config/sidekiq_resource_intensive.yml |  7 +++
 docker-compose.production.yml         | 43 ++++++++++--------
 docker-compose.yml                    | 63 +++++++++++++++------------
 4 files changed, 69 insertions(+), 45 deletions(-)
 create mode 100644 config/sidekiq_resource_intensive.yml

diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index fb2531c82..a72d01223 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -3,4 +3,3 @@
   - default
   - import
   - export
-  - resource_intensive
diff --git a/config/sidekiq_resource_intensive.yml b/config/sidekiq_resource_intensive.yml
new file mode 100644
index 000000000..9865178ef
--- /dev/null
+++ b/config/sidekiq_resource_intensive.yml
@@ -0,0 +1,7 @@
+---
+:concurrency: 1
+:queues:
+  - default
+  - import
+  - export
+  - resource_intensive
diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index 8412c2ad5..e0b369661 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -14,6 +14,25 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
+  depends_on:
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    zoo:
+      condition: service_started
+
 volumes:
   assets:
   cache:
@@ -133,29 +152,19 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_resource_intensive:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     expose:
       - 3000
 
   worker:
-    <<: *app
-    image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
-    depends_on:
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      zoo:
-        condition: service_started
+    <<: *app-worker
+
+  worker_resource_intensive:
+    <<: *app-worker
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows
diff --git a/docker-compose.yml b/docker-compose.yml
index 900b85592..f79d8daa0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -20,6 +20,35 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  build:
+    context: .
+    target: hyku-worker
+    args:
+      - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
+      - HYKU_BULKRAX_ENABLED=true
+    cache_from:
+      - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
+      - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq.yml'
+  depends_on:
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    zoo:
+      condition: service_started
+
 volumes:
   assets:
   cache:
@@ -145,6 +174,8 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_resource_intensive:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     # ports:
@@ -153,33 +184,11 @@ services:
       - 3000
 
   worker:
-    <<: *app
-    build:
-      context: .
-      target: hyku-worker
-      args:
-        - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
-        - HYKU_BULKRAX_ENABLED=true
-      cache_from:
-        - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
-        - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    command: sh -l -c 'bundle && bundle exec sidekiq'
-    depends_on:
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      zoo:
-        condition: service_started
+    <<: *app-worker
+
+  worker_resource_intensive:
+    <<: *app-worker
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows

From 38f52d02f0319e2eaa89c44bfb271eaba2284753 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 12:27:35 -0800
Subject: [PATCH 02/21] deploy resource intensive worker with 4x the normal
 resources

---
 ops/demo-deploy.tmpl.yaml       | 18 ++++++++++++++++++
 ops/production-deploy.tmpl.yaml | 10 ++++++++++
 2 files changed, 28 insertions(+)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 2080c4523..6787eb63d 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -191,6 +191,24 @@ worker:
     runAsGroup: 101
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
+
+worker_resource_intensive:
+  replicaCount: 1
+  resources:
+    limits:
+      memory: "16Gi"
+      cpu: "1200m"
+    requests:
+      memory: "8Gi"
+      cpu: "600m"
+  extraVolumeMounts: *volMounts
+  extraEnvVars: *envVars
+  podSecurityContext:
+    runAsUser: 1001
+    runAsGroup: 101
+    fsGroup: 101
+    fsGroupChangePolicy: "OnRootMismatch"
+
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 175e703a5..4f21d651e 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -201,6 +201,16 @@ worker:
       memory: "2Gi"
       cpu: "150m"
 
+worker_resource_intensive:
+  replicaCount: 1
+  resources:
+    limits:
+      memory: "16Gi"
+      cpu: "1200m"
+    requests:
+      memory: "8Gi"
+      cpu: "600m"
+
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars
   podSecurityContext:

From 692f16ad56baf9df99228a18ead55ead6f97c9f6 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 12:35:55 -0800
Subject: [PATCH 03/21] no need to specify default file location

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index f79d8daa0..2bd13f130 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -32,7 +32,7 @@ x-app-worker: &app-worker
       - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
       - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
   image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-  command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq.yml'
+  command: sh -l -c 'bundle && bundle exec sidekiq'
   depends_on:
     check_volumes:
       condition: service_completed_successfully

From a97e895d62a35a8acdaf743396a18b7e65686ada Mon Sep 17 00:00:00 2001
From: Rob Kaufman <rob@notch8.com>
Date: Fri, 5 Jan 2024 13:14:19 -0800
Subject: [PATCH 04/21] helm deployment of intensive worker

---
 bin/helm_deploy           |   2 +-
 bin/worker                |   6 +-
 ops/demo-deploy.tmpl.yaml | 166 +++++++++++++++++++++++++++++++++++---
 3 files changed, 161 insertions(+), 13 deletions(-)

diff --git a/bin/helm_deploy b/bin/helm_deploy
index 631b57b9f..9972877c6 100755
--- a/bin/helm_deploy
+++ b/bin/helm_deploy
@@ -23,7 +23,7 @@ WORKER_IMAGE="${WORKER_IMAGE:-ghcr.io/samvera/hyku/worker}"
 DEPLOY_TAG="${DEPLOY_TAG:-latest}"
 WORKER_TAG="${WORKER_TAG:-$DEPLOY_TAG}"
 
-helm pull oci://ghcr.io/samvera/charts/hyrax --version 2.0.0 --untar --untardir charts
+helm pull oci://ghcr.io/samvera/charts/hyrax --version 3.5.1 --untar --untardir charts
 
 helm repo update
 
diff --git a/bin/worker b/bin/worker
index b7605486b..a369ffc11 100755
--- a/bin/worker
+++ b/bin/worker
@@ -9,4 +9,8 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec sidekiq"
+if ENV['SIDEKIQ_CONFIG']
+  exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}"
+else
+  exec "echo $DATABASE_URL && bundle exec sidekiq"
+end
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 6787eb63d..8ed13f011 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -47,9 +47,11 @@ ingress:
     - host: commons-archive.org
       paths:
         - path: /
+          pathType: ImplementationSpecific
     - host: "*.commons-archive.org"
       paths:
         - path: /
+          pathType: ImplementationSpecific
   annotations: {
     kubernetes.io/ingress.class: "nginx",
     nginx.ingress.kubernetes.io/proxy-body-size: "0",
@@ -192,7 +194,7 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
 
-worker_resource_intensive:
+workerIntensive:
   replicaCount: 1
   resources:
     limits:
@@ -201,13 +203,152 @@ worker_resource_intensive:
     requests:
       memory: "8Gi"
       cpu: "600m"
-  extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
-  podSecurityContext:
-    runAsUser: 1001
-    runAsGroup: 101
-    fsGroup: 101
-    fsGroupChangePolicy: "OnRootMismatch"
+  extraEnvVars:
+    - name: SIDEKIQ_CONFIG
+      value: "config/sidekiq_resource_intensive.yml"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-intensive-worker
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.workerIntensive.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.workerIntensive.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - db-wait.sh "$REDIS_HOST:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.workerIntensive.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.workerIntensive.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
 
 podSecurityContext:
   runAsUser: 1001
@@ -231,6 +372,10 @@ redis:
   password: $REDIS_PASSWORD
 solr:
   enabled: false
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
 
 externalPostgresql:
   host: postgres-cluster-alpha-ha.postgres.svc.cluster.local
@@ -244,7 +389,7 @@ externalSolrCollection: demo-palni-palci
 externalSolrPassword: $SOLR_ADMIN_PASSWORD
 
 global:
-  hyraxName: palni-palci-demo-pals
+  hyraxHostName: palni-palci-demo-pals
 
 nginx:
   enabled: true
@@ -256,8 +401,7 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxName }};
-    }
+      server {{ .Values.global.hyraxHostName }};}
 
     map ${DOLLAR}status ${DOLLAR}loggable {
         ~^444  0;

From 8a8ce547a3f821241fcd94c70cf36feda39abf6f Mon Sep 17 00:00:00 2001
From: Rob Kaufman <rob@notch8.com>
Date: Fri, 5 Jan 2024 13:17:01 -0800
Subject: [PATCH 05/21] resource adjustments

---
 ops/demo-deploy.tmpl.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 8ed13f011..9fb5dd83c 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -198,11 +198,11 @@ workerIntensive:
   replicaCount: 1
   resources:
     limits:
-      memory: "16Gi"
-      cpu: "1200m"
+      memory: "12Gi"
+      cpu: "4"
     requests:
-      memory: "8Gi"
-      cpu: "600m"
+      memory: "4Gi"
+      cpu: "2"
   extraEnvVars:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"

From 3c0543f59ca0b87e0bbff41f8a6f0c73a1efd6df Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 15:35:13 -0800
Subject: [PATCH 06/21] undo changes to unused file

---
 docker-compose.production.yml | 43 ++++++++++++++---------------------
 1 file changed, 17 insertions(+), 26 deletions(-)

diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index e0b369661..8412c2ad5 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -14,25 +14,6 @@ x-app: &app
   networks:
     internal:
 
-x-app-worker: &app-worker
-  <<: *app
-  image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
-  depends_on:
-    check_volumes:
-      condition: service_completed_successfully
-    initialize_app:
-      condition: service_completed_successfully
-    db:
-      condition: service_started
-    solr:
-      condition: service_started
-    fcrepo:
-      condition: service_started
-    redis:
-      condition: service_started
-    zoo:
-      condition: service_started
-
 volumes:
   assets:
   cache:
@@ -152,19 +133,29 @@ services:
         condition: service_started
       worker:
         condition: service_started
-      worker_resource_intensive:
-        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     expose:
       - 3000
 
   worker:
-    <<: *app-worker
-
-  worker_resource_intensive:
-    <<: *app-worker
-    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
+    <<: *app
+    image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
+    depends_on:
+      check_volumes:
+        condition: service_completed_successfully
+      initialize_app:
+        condition: service_completed_successfully
+      db:
+        condition: service_started
+      solr:
+        condition: service_started
+      fcrepo:
+        condition: service_started
+      redis:
+        condition: service_started
+      zoo:
+        condition: service_started
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows

From 3900d39af8a20d8180e96812d6b3e8ca73554551 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 16:04:10 -0800
Subject: [PATCH 07/21] use ENV var for intensive worker thread count

---
 .env                                  | 1 +
 config/sidekiq_resource_intensive.yml | 2 +-
 ops/demo-deploy.tmpl.yaml             | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.env b/.env
index d61ae725a..07a6e9335 100644
--- a/.env
+++ b/.env
@@ -23,6 +23,7 @@ PASSENGER_APP_ENV=development
 RAILS_LOG_TO_STDOUT=true
 REDIS_HOST=redis
 SECRET_KEY_BASE=asdf
+SIDEKIQ_INTENSIVE_THREAD_COUNT=1
 SOLR_ADMIN_PASSWORD=SolrRocks
 SOLR_ADMIN_USER=solr
 SOLR_COLLECTION_NAME=hydra-development
diff --git a/config/sidekiq_resource_intensive.yml b/config/sidekiq_resource_intensive.yml
index 9865178ef..fc520467e 100644
--- a/config/sidekiq_resource_intensive.yml
+++ b/config/sidekiq_resource_intensive.yml
@@ -1,5 +1,5 @@
 ---
-:concurrency: 1
+:concurrency: <%= ENV['SIDEKIQ_INTENSIVE_THREAD_COUNT'] %>
 :queues:
   - default
   - import
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 9fb5dd83c..a1c6c7491 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -401,7 +401,8 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxHostName }};}
+      server {{ .Values.global.hyraxHostName }};
+    }
 
     map ${DOLLAR}status ${DOLLAR}loggable {
         ~^444  0;

From bcdb041fce3ec127c5e91dc1e6a878e392d116a5 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 16:41:52 -0800
Subject: [PATCH 08/21] adjust intensive worker prod resources

Co-authored-by: Rob Kaufman <rob@notch8.com>
---
 ops/demo-deploy.tmpl.yaml       |  2 ++
 ops/production-deploy.tmpl.yaml | 15 ++++++++++-----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index a1c6c7491..a599b9663 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -206,6 +206,8 @@ workerIntensive:
   extraEnvVars:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"
+    - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
+      value: 1
 
 extraDeploy:
   - |-
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 4f21d651e..fc6c2d724 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -201,15 +201,20 @@ worker:
       memory: "2Gi"
       cpu: "150m"
 
-worker_resource_intensive:
+workerIntensive:
   replicaCount: 1
   resources:
     limits:
-      memory: "16Gi"
-      cpu: "1200m"
+      memory: "12Gi"
+      cpu: "4"
     requests:
-      memory: "8Gi"
-      cpu: "600m"
+      memory: "4Gi"
+      cpu: "2"
+  extraEnvVars:
+    - name: SIDEKIQ_CONFIG
+      value: "config/sidekiq_resource_intensive.yml"
+    - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
+      value: 1
 
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars

From 72410d83e602c15014dd0877b09d3379ec2c5340 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 17:04:01 -0800
Subject: [PATCH 09/21] fix disarranged YAML

---
 ops/production-deploy.tmpl.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index fc6c2d724..d922328bd 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -200,6 +200,13 @@ worker:
     requests:
       memory: "2Gi"
       cpu: "150m"
+  extraVolumeMounts: *volMounts
+  extraEnvVars: *envVars
+  podSecurityContext:
+    runAsUser: 1001
+    runAsGroup: 101
+    fsGroup: 101
+    fsGroupChangePolicy: "OnRootMismatch"
 
 workerIntensive:
   replicaCount: 1
@@ -216,13 +223,6 @@ workerIntensive:
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
       value: 1
 
-  extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
-  podSecurityContext:
-    runAsUser: 1001
-    runAsGroup: 101
-    fsGroup: 101
-    fsGroupChangePolicy: "OnRootMismatch"
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101

From 8a22e71d842a0e05307fc6ae585c445db9e948d5 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 17:42:16 -0800
Subject: [PATCH 10/21] update prod to Hyrax helm chart v3.5.1

---
 ops/production-deploy.tmpl.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index d922328bd..37e1e78a0 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -47,9 +47,11 @@ ingress:
     - host: hykucommons.org
       paths:
         - path: /
+          pathType: ImplementationSpecific
     - host: "*.hykucommons.org"
       paths:
         - path: /
+          pathType: ImplementationSpecific
   annotations: {
     kubernetes.io/ingress.class: "nginx",
     nginx.ingress.kubernetes.io/proxy-body-size: "0",
@@ -245,6 +247,10 @@ redis:
   password: $REDIS_PASSWORD
 solr:
   enabled: false
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
 
 externalPostgresql:
   host: postgres-cluster-alpha-ha.postgres.svc.cluster.local
@@ -258,7 +264,7 @@ externalSolrCollection: production-palni-palci
 externalSolrPassword: $SOLR_ADMIN_PASSWORD
 
 global:
-  hyraxName: palni-palci-production-pals
+  hyraxHostName: palni-palci-production-pals
 
 nginx:
   enabled: true
@@ -270,7 +276,7 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxName }};
+      server {{ .Values.global.hyraxHostName }};
     }
 
     map ${DOLLAR}status ${DOLLAR}loggable {

From fee32b2f3060e6efee99eb5b14cc3cc5bd99b019 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 9 Jan 2024 10:52:43 -0800
Subject: [PATCH 11/21] fix demo redis deploy issue

Change pulled from:
- https://github.com/samvera/hyku/pull/2145
---
 ops/demo-deploy.tmpl.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index a599b9663..d48c772ff 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -369,9 +369,10 @@ fcrepo:
 postgresql:
   enabled: false
 redis:
-  cluster:
-    enabled: false
-  password: $REDIS_PASSWORD
+  enabled: true
+  architecture: standalone
+  auth:
+    password: $REDIS_PASSWORD
 solr:
   enabled: false
 fits:

From 789a226acec7111570bb12e142b02cab3f668dc9 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:09:20 -0800
Subject: [PATCH 12/21] env vars need to be strings

---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index d48c772ff..97cd13bf9 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -207,7 +207,7 @@ workerIntensive:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
-      value: 1
+      value: "1"
 
 extraDeploy:
   - |-

From c4b9cc31bc33ec42ddc806234b453c2f955258d7 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 23 Jan 2024 17:06:10 -0800
Subject: [PATCH 13/21] correctly parse Redis host value

Because of the special way `extraDeploy` gets interpreted, fetching ENV
values using `$` will not work.
---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 97cd13bf9..2c4456d0c 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -246,7 +246,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh "$REDIS_HOST:6379"
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 495d807f5005e7ec4116963634f549c9c73666ee Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 13:54:48 -0800
Subject: [PATCH 14/21] use more generic name for extra worker

We will be reusing this pattern across several projects. Not every extra
worker will need increased resource limits, so we should use a more
generic name that can apply to various requirements.
---
 .env                                           |  2 +-
 app/jobs/create_large_derivatives_job.rb       |  2 +-
 config/sidekiq_auxiliary.yml                   |  7 +++++++
 config/sidekiq_resource_intensive.yml          |  7 -------
 docker-compose.yml                             |  6 +++---
 ops/demo-deploy.tmpl.yaml                      | 16 ++++++++--------
 ops/production-deploy.tmpl.yaml                |  6 +++---
 spec/jobs/create_large_derivatives_job_spec.rb |  4 ++--
 8 files changed, 25 insertions(+), 25 deletions(-)
 create mode 100644 config/sidekiq_auxiliary.yml
 delete mode 100644 config/sidekiq_resource_intensive.yml

diff --git a/.env b/.env
index 07a6e9335..172682ab1 100644
--- a/.env
+++ b/.env
@@ -23,7 +23,7 @@ PASSENGER_APP_ENV=development
 RAILS_LOG_TO_STDOUT=true
 REDIS_HOST=redis
 SECRET_KEY_BASE=asdf
-SIDEKIQ_INTENSIVE_THREAD_COUNT=1
+SIDEKIQ_AUXILIARY_THREAD_COUNT=1
 SOLR_ADMIN_PASSWORD=SolrRocks
 SOLR_ADMIN_USER=solr
 SOLR_COLLECTION_NAME=hydra-development
diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb
index 6f678c40f..869203446 100644
--- a/app/jobs/create_large_derivatives_job.rb
+++ b/app/jobs/create_large_derivatives_job.rb
@@ -12,5 +12,5 @@
 # @see Hydra::Derivatives::Processors::Ffmpeg
 # @see https://github.com/scientist-softserv/palni-palci/issues/852
 class CreateLargeDerivativesJob < CreateDerivativesJob
-  queue_as :resource_intensive
+  queue_as :auxiliary
 end
diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml
new file mode 100644
index 000000000..f4ed01050
--- /dev/null
+++ b/config/sidekiq_auxiliary.yml
@@ -0,0 +1,7 @@
+---
+:concurrency: <%= ENV['SIDEKIQ_AUXILIARY_THREAD_COUNT'] %>
+:queues:
+  - default
+  - import
+  - export
+  - auxiliary
diff --git a/config/sidekiq_resource_intensive.yml b/config/sidekiq_resource_intensive.yml
deleted file mode 100644
index fc520467e..000000000
--- a/config/sidekiq_resource_intensive.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-:concurrency: <%= ENV['SIDEKIQ_INTENSIVE_THREAD_COUNT'] %>
-:queues:
-  - default
-  - import
-  - export
-  - resource_intensive
diff --git a/docker-compose.yml b/docker-compose.yml
index 2bd13f130..a6d040d7a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -174,7 +174,7 @@ services:
         condition: service_started
       worker:
         condition: service_started
-      worker_resource_intensive:
+      worker_auxiliary:
         condition: service_started
       initialize_app:
         condition: service_completed_successfully
@@ -186,9 +186,9 @@ services:
   worker:
     <<: *app-worker
 
-  worker_resource_intensive:
+  worker_auxiliary:
     <<: *app-worker
-    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_auxiliary.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 2c4456d0c..dbebcf09d 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -194,7 +194,7 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
 
-workerIntensive:
+workerAuxiliary:
   replicaCount: 1
   resources:
     limits:
@@ -205,8 +205,8 @@ workerIntensive:
       cpu: "2"
   extraEnvVars:
     - name: SIDEKIQ_CONFIG
-      value: "config/sidekiq_resource_intensive.yml"
-    - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
+      value: "config/sidekiq_auxiliary.yml"
+    - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
       value: "1"
 
 extraDeploy:
@@ -214,11 +214,11 @@ extraDeploy:
       apiVersion: apps/v1
       kind: Deployment
       metadata:
-        name: {{ include "hyrax.fullname" . }}-intensive-worker
+        name: {{ include "hyrax.fullname" . }}-auxiliary-worker
         labels:
           {{- include "hyrax.labels" . | nindent 4 }}
       spec:
-        replicas: {{ .Values.workerIntensive.replicaCount }}
+        replicas: {{ .Values.workerAuxiliary.replicaCount }}
         selector:
           matchLabels:
             {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
@@ -241,7 +241,7 @@ extraDeploy:
                   - secretRef:
                       name: {{ template "hyrax.fullname" . }}
                 env:
-                  {{- toYaml .Values.workerIntensive.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                 command:
                   - sh
@@ -276,7 +276,7 @@ extraDeploy:
                     {{- toYaml . | nindent 12 }}
                   {{- end }}
                 env:
-                  {{- toYaml .Values.workerIntensive.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
                   {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
                 {{- if .Values.worker.readinessProbe.enabled }}
                 readinessProbe:
@@ -306,7 +306,7 @@ extraDeploy:
                   {{- toYaml . | nindent 12 }}
                   {{- end }}
                 resources:
-                  {{- toYaml .Values.workerIntensive.resources | nindent 12 }}
+                  {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }}
                 {{- with .Values.extraContainerConfiguration }}
                 {{- toYaml . | nindent 10 }}
                 {{- end }}
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 37e1e78a0..2d741b181 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -210,7 +210,7 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
 
-workerIntensive:
+workerAuxiliary:
   replicaCount: 1
   resources:
     limits:
@@ -221,8 +221,8 @@ workerIntensive:
       cpu: "2"
   extraEnvVars:
     - name: SIDEKIQ_CONFIG
-      value: "config/sidekiq_resource_intensive.yml"
-    - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
+      value: "config/sidekiq_auxiliary.yml"
+    - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
       value: 1
 
 podSecurityContext:
diff --git a/spec/jobs/create_large_derivatives_job_spec.rb b/spec/jobs/create_large_derivatives_job_spec.rb
index b39f6d82a..1ed67ef2c 100644
--- a/spec/jobs/create_large_derivatives_job_spec.rb
+++ b/spec/jobs/create_large_derivatives_job_spec.rb
@@ -19,10 +19,10 @@
     allow(file_set).to receive(:update_index)
   end
 
-  it 'runs in the :resource_intensive queue' do
+  it 'runs in the :auxiliary queue' do
     expect { described_class.perform_later(file_set, file.id) }
       .to have_enqueued_job(described_class)
-      .on_queue('resource_intensive')
+      .on_queue('auxiliary')
   end
 
   # @see CreateDerivativesJobDecorator#perform

From ab11894cd73689364bc8c2bad985685996da2118 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 14:38:17 -0800
Subject: [PATCH 15/21] add extraDeploy hint for workerAuxiliary

---
 ops/demo-deploy.tmpl.yaml       | 2 ++
 ops/production-deploy.tmpl.yaml | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index dbebcf09d..6c9ede08a 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -194,6 +194,8 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
 
+# When adding/removing key-value pairs to this block, ensure the
+# corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   replicaCount: 1
   resources:
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 2d741b181..e4e436f4f 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -210,6 +210,8 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
 
+# When adding/removing key-value pairs to this block, ensure the
+# corresponding changes are made in the `extraDeploy` block below.
 workerAuxiliary:
   replicaCount: 1
   resources:

From 04bd455f9e141ea6caea8e17cf2e7bc3698a0a73 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 14:39:51 -0800
Subject: [PATCH 16/21] add extra worker deploy to prod, update redis conf

---
 ops/production-deploy.tmpl.yaml | 152 +++++++++++++++++++++++++++++++-
 1 file changed, 148 insertions(+), 4 deletions(-)

diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index e4e436f4f..8653ec468 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -225,7 +225,150 @@ workerAuxiliary:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_auxiliary.yml"
     - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
-      value: 1
+      value: "1"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-auxiliary-worker
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.workerAuxiliary.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
 
 podSecurityContext:
   runAsUser: 1001
@@ -244,9 +387,10 @@ fcrepo:
 postgresql:
   enabled: false
 redis:
-  cluster:
-    enabled: false
-  password: $REDIS_PASSWORD
+  enabled: true
+  architecture: standalone
+  auth:
+    password: $REDIS_PASSWORD
 solr:
   enabled: false
 fits:

From 874746e002ca88a2254e571df56dcc50f733322f Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 14:40:50 -0800
Subject: [PATCH 17/21] don't mix JSON and YAML syntax

---
 ops/demo-deploy.tmpl.yaml       | 9 ++++-----
 ops/production-deploy.tmpl.yaml | 9 ++++-----
 ops/staging-deploy.tmpl.yaml    | 9 ++++-----
 3 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 6c9ede08a..aca252f04 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -52,11 +52,10 @@ ingress:
       paths:
         - path: /
           pathType: ImplementationSpecific
-  annotations: {
-    kubernetes.io/ingress.class: "nginx",
-    nginx.ingress.kubernetes.io/proxy-body-size: "0",
-    cert-manager.io/cluster-issuer: letsencrypt-production-dns
-  }
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    cert-manager.io/cluster-issuer: "letsencrypt-production-dns"
   tls:
     - hosts:
         - commons-archive.org
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 8653ec468..0f9610568 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -52,11 +52,10 @@ ingress:
       paths:
         - path: /
           pathType: ImplementationSpecific
-  annotations: {
-    kubernetes.io/ingress.class: "nginx",
-    nginx.ingress.kubernetes.io/proxy-body-size: "0",
-    cert-manager.io/cluster-issuer: letsencrypt-production-dns
-  }
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    cert-manager.io/cluster-issuer: "letsencrypt-production-dns"
   tls:
     - hosts:
         - hykucommons.org
diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml
index 3c6d0ebd2..76d6e0311 100644
--- a/ops/staging-deploy.tmpl.yaml
+++ b/ops/staging-deploy.tmpl.yaml
@@ -50,11 +50,10 @@ ingress:
     - host: "*.palni-palci-staging.notch8.cloud"
       paths:
         - path: /
-  annotations: {
-    kubernetes.io/ingress.class: "nginx",
-    nginx.ingress.kubernetes.io/proxy-body-size: "0",
-    cert-manager.io/cluster-issuer: letsencrypt-production-dns
-  }
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    cert-manager.io/cluster-issuer: "letsencrypt-production-dns"
   tls:
     - hosts:
         - palni-palci-staging.notch8.cloud

From d7047868c6e43c2be8b07cb90b9681a5b112e9c0 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 14:57:35 -0800
Subject: [PATCH 18/21] staging deploy: add workerAuxiliary & update to Hyrax
 helm chart v3.5.1

---
 ops/staging-deploy.tmpl.yaml | 175 ++++++++++++++++++++++++++++++++++-
 1 file changed, 171 insertions(+), 4 deletions(-)

diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml
index 76d6e0311..b332ca629 100644
--- a/ops/staging-deploy.tmpl.yaml
+++ b/ops/staging-deploy.tmpl.yaml
@@ -47,9 +47,11 @@ ingress:
     - host: palni-palci-staging.notch8.cloud
       paths:
         - path: /
+          pathType: ImplementationSpecific
     - host: "*.palni-palci-staging.notch8.cloud"
       paths:
         - path: /
+          pathType: ImplementationSpecific
   annotations:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/proxy-body-size: "0"
@@ -181,7 +183,6 @@ worker:
     limits:
       memory: "2Gi"
       cpu: "1000m"
-
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars
   podSecurityContext:
@@ -189,6 +190,167 @@ worker:
     runAsGroup: 101
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
+
+# When adding/removing key-value pairs to this block, ensure the
+# corresponding changes are made in the `extraDeploy` block below.
+workerAuxiliary:
+  replicaCount: 1
+  resources:
+    limits:
+      memory: "12Gi"
+      cpu: "4"
+    requests:
+      memory: "4Gi"
+      cpu: "2"
+  extraEnvVars:
+    - name: SIDEKIQ_CONFIG
+      value: "config/sidekiq_auxiliary.yml"
+    - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
+      value: "1"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-auxiliary-worker
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.workerAuxiliary.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.workerAuxiliary.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101
@@ -205,11 +367,16 @@ fcrepo:
 postgresql:
   enabled: false
 redis:
-  cluster:
-    enabled: false
-  password: $REDIS_PASSWORD
+  enabled: true
+  architecture: standalone
+  auth:
+    password: $REDIS_PASSWORD
 solr:
   enabled: false
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
 
 externalFcrepoHost: fcrepo.staging-fcrepo.svc.cluster.local
 externalPostgresql:

From e4a57a8d4dc00380ce2960185d6aceee28b83e9c Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 15:04:28 -0800
Subject: [PATCH 19/21] prioritize auxiliary queue

Sidekiq queues without explicit weights are prioritized in the order
they are listed. Our auxiliary worker's primary goal is to process jobs
in the auxiliary queue, thus we want it to be the top priority.

[Sidekiq
docs](https://github.com/sidekiq/sidekiq/wiki/Advanced-Options#queues)
---
 config/sidekiq_auxiliary.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml
index f4ed01050..861299213 100644
--- a/config/sidekiq_auxiliary.yml
+++ b/config/sidekiq_auxiliary.yml
@@ -1,7 +1,7 @@
 ---
 :concurrency: <%= ENV['SIDEKIQ_AUXILIARY_THREAD_COUNT'] %>
 :queues:
+  - auxiliary
   - default
   - import
   - export
-  - auxiliary

From b26ed4de90b4a923140d34d7ee5039c38d4ee1b2 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 16:09:14 -0800
Subject: [PATCH 20/21] rename env var to be more generic

---
 .env                            | 2 +-
 config/sidekiq_auxiliary.yml    | 2 +-
 ops/demo-deploy.tmpl.yaml       | 4 ++--
 ops/production-deploy.tmpl.yaml | 4 ++--
 ops/staging-deploy.tmpl.yaml    | 4 ++--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.env b/.env
index 172682ab1..441e0cba1 100644
--- a/.env
+++ b/.env
@@ -1,3 +1,4 @@
+AUXILIARY_WORKER_THREAD_COUNT=1
 CAPYBARA_SERVER=web
 CHROME_HOSTNAME=seleniarm-hub
 DB_ADAPTER=postgresql
@@ -23,7 +24,6 @@ PASSENGER_APP_ENV=development
 RAILS_LOG_TO_STDOUT=true
 REDIS_HOST=redis
 SECRET_KEY_BASE=asdf
-SIDEKIQ_AUXILIARY_THREAD_COUNT=1
 SOLR_ADMIN_PASSWORD=SolrRocks
 SOLR_ADMIN_USER=solr
 SOLR_COLLECTION_NAME=hydra-development
diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml
index 861299213..24eee633e 100644
--- a/config/sidekiq_auxiliary.yml
+++ b/config/sidekiq_auxiliary.yml
@@ -1,5 +1,5 @@
 ---
-:concurrency: <%= ENV['SIDEKIQ_AUXILIARY_THREAD_COUNT'] %>
+:concurrency: <%= ENV['AUXILIARY_WORKER_THREAD_COUNT'] %>
 :queues:
   - auxiliary
   - default
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index aca252f04..43656223f 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -205,10 +205,10 @@ workerAuxiliary:
       memory: "4Gi"
       cpu: "2"
   extraEnvVars:
+    - name: AUXILIARY_WORKER_THREAD_COUNT
+      value: "1"
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_auxiliary.yml"
-    - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
-      value: "1"
 
 extraDeploy:
   - |-
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 0f9610568..3338bc92e 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -221,10 +221,10 @@ workerAuxiliary:
       memory: "4Gi"
       cpu: "2"
   extraEnvVars:
+    - name: AUXILIARY_WORKER_THREAD_COUNT
+      value: "1"
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_auxiliary.yml"
-    - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
-      value: "1"
 
 extraDeploy:
   - |-
diff --git a/ops/staging-deploy.tmpl.yaml b/ops/staging-deploy.tmpl.yaml
index b332ca629..b93ee50ea 100644
--- a/ops/staging-deploy.tmpl.yaml
+++ b/ops/staging-deploy.tmpl.yaml
@@ -203,10 +203,10 @@ workerAuxiliary:
       memory: "4Gi"
       cpu: "2"
   extraEnvVars:
+    - name: AUXILIARY_WORKER_THREAD_COUNT
+      value: "1"
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_auxiliary.yml"
-    - name: SIDEKIQ_AUXILIARY_THREAD_COUNT
-      value: "1"
 
 extraDeploy:
   - |-

From d9f5488d587c6ed23ed1fb9dabebd7deec63072d Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Mon, 12 Feb 2024 16:17:53 -0800
Subject: [PATCH 21/21] update comments to reflect new queue name

---
 app/jobs/create_large_derivatives_job.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb
index 869203446..c068d5152 100644
--- a/app/jobs/create_large_derivatives_job.rb
+++ b/app/jobs/create_large_derivatives_job.rb
@@ -2,9 +2,9 @@
 
 # CreateLargeDerivativesJob is intended to be used for resource-intensive derivative
 # generation (e.g. video processing). It is functionally similar to CreateDerivativesJob,
-# except that it queues jobs in the :resource_intensive queue.
+# except that it queues jobs in the :auxiliary queue.
 #
-# The worker responsible for processing jobs in the :resource_intensive queue should be
+# The worker responsible for processing jobs in the :auxiliary queue should be
 # configured to have more resources dedicated to it, especially CPU. Otherwise, the
 # `ffmpeg` commands that this job class eventually triggers could be throttled.
 #