diff --git a/bin/worker b/bin/worker index a369ffc11..f39bf9ebc 100755 --- a/bin/worker +++ b/bin/worker @@ -4,13 +4,9 @@ if ENV['GOOGLE_OAUTH_PRIVATE_KEY_VALUE'] && !ENV['GOOGLE_OAUTH_PRIVATE_KEY_VALUE end if ENV['DATABASE_URL'] && !ENV['DATABASE_URL'].empty? - ENV['DATABASE_URL'] = ENV['DATABASE_URL'].gsub('pool=5', 'pool=30') + ENV['DATABASE_URL'] = ENV['DATABASE_URL'].gsub('pool=5', "pools=#{ENV.fetch('WORKER_CONCURENCY'), 30}") else puts 'DATABASE_URL not set, no pool change needed' end -if ENV['SIDEKIQ_CONFIG'] - exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}" -else - exec "echo $DATABASE_URL && bundle exec sidekiq" -end +exec "echo $DATABASE_URL && RAILS_MAX_THREADS=$WORKER_CONCURENCY bundle exec sidekiq $WORKER_ARGS" diff --git a/config/sidekiq.yml b/config/sidekiq.yml deleted file mode 100644 index a72d01223..000000000 --- a/config/sidekiq.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -:queues: - - default - - import - - export diff --git a/config/sidekiq_auxiliary.yml b/config/sidekiq_auxiliary.yml deleted file mode 100644 index 24eee633e..000000000 --- a/config/sidekiq_auxiliary.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -:concurrency: <%= ENV['AUXILIARY_WORKER_THREAD_COUNT'] %> -:queues: - - auxiliary - - default - - import - - export diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml index 43656223f..015f4d585 100644 --- a/ops/demo-deploy.tmpl.yaml +++ b/ops/demo-deploy.tmpl.yaml @@ -187,6 +187,9 @@ worker: cpu: "150m" extraVolumeMounts: *volMounts extraEnvVars: *envVars + extraEnvFrom: + - configMapRef: + name: worker-env podSecurityContext: runAsUser: 1001 runAsGroup: 101 @@ -204,14 +207,35 @@ workerAuxiliary: requests: memory: "4Gi" cpu: "2" - extraEnvVars: - - name: AUXILIARY_WORKER_THREAD_COUNT - value: "1" - - name: SIDEKIQ_CONFIG - value: "config/sidekiq_auxiliary.yml" + extraEnvVars: *envVars + extraEnvFrom: + - configMapRef: + name: auxilary-worker-env + extraDeploy: - |- + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: worker-env + labels: + {{- include "hyrax.labels" . | nindent 4 }} + data: + WORKER_CONCURENCY: "5" + WORKER_ARGS: -q default -q import -q export + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: auxiliary-worker-env + labels: + {{- include "hyrax.labels" . | nindent 4 }} + data: + WORKER_CONCURENCY: "1" + WORKER_ARGS: -q auxiliary -q default -q import -q export + --- apiVersion: apps/v1 kind: Deployment metadata: @@ -241,9 +265,11 @@ extraDeploy: name: {{ include "hyrax.fullname" . }}-env - secretRef: name: {{ template "hyrax.fullname" . }} + {{- with .Values.workerAuxiliary.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} env: {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} - {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} command: - sh - -c @@ -273,12 +299,11 @@ extraDeploy: - secretRef: name: {{ .Values.solrExistingSecret }} {{- end }} - {{- with .Values.worker.extraEnvFrom }} + {{- with .Values.workerAuxiliary.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} env: {{- toYaml .Values.workerAuxiliary.extraEnvVars | nindent 12 }} - {{- toYaml .Values.worker.extraEnvVars | nindent 12 }} {{- if .Values.worker.readinessProbe.enabled }} readinessProbe: exec: diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml index 3338bc92e..c494d48ad 100644 --- a/ops/production-deploy.tmpl.yaml +++ b/ops/production-deploy.tmpl.yaml @@ -203,6 +203,9 @@ worker: cpu: "150m" extraVolumeMounts: *volMounts extraEnvVars: *envVars + extraEnvFrom: + - configMapRef: + name: worker-env podSecurityContext: runAsUser: 1001 runAsGroup: 101 @@ -220,14 +223,34 @@ workerAuxiliary: requests: memory: "4Gi" cpu: "2" - extraEnvVars: - - name: AUXILIARY_WORKER_THREAD_COUNT - value: "1" - - name: SIDEKIQ_CONFIG - value: "config/sidekiq_auxiliary.yml" + extraEnvVars: *envVars + extraEnvFrom: + - configMapRef: + name: auxilary-worker-env extraDeploy: - |- + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: worker-env + labels: + {{- include "hyrax.labels" . | nindent 4 }} + data: + WORKER_CONCURENCY: "5" + WORKER_ARGS: -q default -q import -q export + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: auxiliary-worker-env + labels: + {{- include "hyrax.labels" . | nindent 4 }} + data: + WORKER_CONCURENCY: "1" + WORKER_ARGS: -q auxiliary -q default -q import -q export + --- apiVersion: apps/v1 kind: Deployment metadata: