Merge branch 'main' into ci-dassie

samvera · Aug 24, 2023 · dc0bba0 · dc0bba0
2 parents 133a5eb + f8b08f8
commit dc0bba0
Show file tree

Hide file tree

Showing 15 changed files with 113 additions and 613 deletions.
diff --git a/.dassie/.env b/.dassie/.env
@@ -9,6 +9,7 @@ FCREPO_HOST=fcrepo
 FCREPO_PORT=8080
 FCREPO_REST_PATH=rest
 FCREPO_TEST_BASE_PATH=/test
+FITS_SERVLET_URL=http://fits:8080/fits
 HUB_URL=http://chrome:4444/wd/hub
 HYRAX_ANALYTICS=false
 HYRAX_ANALYTICS_PROVIDER=google

diff --git a/.dassie/config/initializers/hyrax.rb b/.dassie/config/initializers/hyrax.rb
@@ -18,6 +18,8 @@
   config.work_requires_files = false
   config.citations = true
 
+  config.characterization_options = { ch12n_tool: ENV.fetch('CH12N_TOOL', 'fits').to_sym }
+
   # Returns a URL that resolves to an image provided by a IIIF image server
   config.iiif_image_url_builder = lambda do |file_id, base_url, size, format|
     Riiif::Engine.routes.url_helpers.image_url(file_id, host: base_url, size: size)

diff --git a/.koppie/.env b/.koppie/.env
@@ -7,6 +7,7 @@ DB_NAME=koppie
 DB_PASSWORD=hyrax_password
 DB_PORT=5432
 DB_USERNAME=hyrax_user
+FITS_SERVLET_URL=http://fits:8080/fits
 HUB_URL=http://chrome:4444/wd/hub
 HYRAX_DERIVATIVES_PATH=/app/samvera/hyrax-webapp/derivatives/
 HYRAX_ENGINE_PATH=/app/samvera/hyrax-engine

diff --git a/.koppie/config/initializers/hyrax.rb b/.koppie/config/initializers/hyrax.rb
@@ -7,6 +7,8 @@
 
   config.disable_wings = true # not needed if ENV includes HYRAX_SKIP_WINGS=true
 
+  config.characterization_options = { ch12n_tool: ENV.fetch('CH12N_TOOL', 'fits').to_sym }
+
   # Register roles that are expected by your implementation.
   # @see Hyrax::RoleRegistry for additional details.
   # @note there are magical roles as defined in Hyrax::RoleRegistry::MAGIC_ROLES

diff --git a/app/services/hyrax/characterization/valkyrie_characterization_service.rb b/app/services/hyrax/characterization/valkyrie_characterization_service.rb
@@ -27,25 +27,27 @@ def self.run(metadata:, file:, user: ::User.system_user, **options)
   # @!attribute [rw] source
   #   @return [Valkyrie::StorageAdapter::StreamFile]
   # @!attribute [rw] tools
-  #   @todo clarify what this is for. it gets passed to {#characterize}
-  #     on the characterizer, but it's not clear how to customize
-  #     effectively.
+  #   can be :fits, :fits_servlet, :ffprobe or any other service added to HydraFileCharacterization
+  #   note that ffprope is faster but only works on AV files.
   #   @return [Symbol]
   attr_accessor :mapping, :metadata, :parser, :source, :tools
 
   ##
   # @api private
-  def initialize(metadata:, # rubocop:disable Metrics/ParameterLists
-                 file:,
-                 characterizer: Hydra::FileCharacterization,
-                 parser_mapping: Hydra::Works::Characterization.mapper,
-                 parser: Hydra::Works::Characterization::FitsDocument.new)
+  def initialize( # rubocop:disable Metrics/ParameterLists
+    metadata:,
+    file:,
+    characterizer: Hydra::FileCharacterization,
+    parser_mapping: Hydra::Works::Characterization.mapper,
+    parser: Hydra::Works::Characterization::FitsDocument.new,
+    ch12n_tool: :fits
+  )
     @characterizer = characterizer
     @metadata      = metadata
     @source        = file
     @mapping       = parser_mapping
     @parser        = parser
-    @tools         = :fits
+    @tools         = ch12n_tool
   end
 
   ##

diff --git a/app/services/hyrax/collections/migration_service.rb b/app/services/hyrax/collections/migration_service.rb
diff --git a/chart/hyrax/Chart.yaml b/chart/hyrax/Chart.yaml
@@ -33,3 +33,7 @@ dependencies:
     version: 13.2.30
     repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
     condition: nginx.enabled
+  - name: fits
+    version: 0.1.0
+    repository:  https://samvera-labs.github.io/fits-charts
+    condition: fits.enabled
diff --git a/chart/hyrax/README.md b/chart/hyrax/README.md
@@ -35,28 +35,30 @@ production environments.
 
 The chart populates the following environment variables:
 
-| Variable          | Description                    | Condition              |
-|-------------------|--------------------------------|------------------------|
-| DB_PASSWORD       | Postgresql password            | n/a                    |
-| DB_PORT           | Postgresql service port        | n/a                    |
-| DB_HOST           | Postgresql hostname            | n/a                    |
-| DB_USERNAME       | Postgresql username            | n/a                    |
-| MEMCACHED_HOST    | Memcached host                 | `memcached.enabled`    |
-| RACK_ENV          | app environment ('production') | n/a                    |
-| RAILS_ENV         | app environment ('production') | n/a                    |
-| REDIS_HOST        | Redis service host             | `redis.enabled`        |
-| FCREPO_BASE_PATH  | Fedora Commons root path       | n/a                    |
-| FCREPO_HOST       | Fedora Commons host            | n/a                    |
-| FCREPO_PORT       | Fedora Commons port            | n/a                    |
-| FCREPO_REST_PATH  | Fedora Commons REST endpoint   | n/a                    |
-| SKIP_HYRAX_ENGINE_SEED   | Flag to load Hyrax engine seed file | n/a                    |
-| SOLR_ADMIN_USER   | Solr user for basic auth       | n/a                    |
-| SOLR_ADMIN_PASSWORD | Solr password for basic auth | n/a                    |
-| SOLR_COLLECTION_NAME | The name of the solr collection to use | n/a         |
+| Variable          | Description                    | Condition                                |
+|-------------------|--------------------------------|------------------------------------------|
+| CH12N_TOOL        | Tool for characterization      | `fits.enabled` or `externalFits.enabled` |
+| DB_PASSWORD       | Postgresql password            | n/a                                      |
+| DB_PORT           | Postgresql service port        | n/a                                      |
+| DB_HOST           | Postgresql hostname            | n/a                                      |
+| DB_USERNAME       | Postgresql username            | n/a                                      |
+| MEMCACHED_HOST    | Memcached host                 | `memcached.enabled`                      |
+| RACK_ENV          | app environment ('production') | n/a                                      |
+| RAILS_ENV         | app environment ('production') | n/a                                      |
+| REDIS_HOST        | Redis service host             | `redis.enabled`                          |
+| FCREPO_BASE_PATH  | Fedora Commons root path       | n/a                                      |
+| FCREPO_HOST       | Fedora Commons host            | n/a                                      |
+| FCREPO_PORT       | Fedora Commons port            | n/a                                      |
+| FCREPO_REST_PATH  | Fedora Commons REST endpoint   | n/a                                      |
+| FITS_SERVLET_URL  | URL for FITS servlet           | `fits.enabled` or `externalFits.enabled` |
+| SKIP_HYRAX_ENGINE_SEED   | Flag to load Hyrax engine seed file | n/a                          |
+| SOLR_ADMIN_USER   | Solr user for basic auth       | n/a                                      |
+| SOLR_ADMIN_PASSWORD | Solr password for basic auth | n/a                                      |
+| SOLR_COLLECTION_NAME | The name of the solr collection to use | n/a                           |
 | SOLR_CONFIGSET_NAME | The name of the solr configset to use for config management tasks | n/a |
-| SOLR_HOST         | Solr service host              | n/a                    |
-| SOLR_PORT         | Solr service port              | n/a                    |
-| SOLR_URL          | Solr service full URL          | n/a                    |
+| SOLR_HOST         | Solr service host              | n/a                                      |
+| SOLR_PORT         | Solr service port              | n/a                                      |
+| SOLR_URL          | Solr service full URL          | n/a                                      |
 
 ## With an external SolrCloud
 
@@ -89,6 +91,17 @@ To use an existing or external `fcrepo` instance, use the chart values:
   - `fcrepo.enabled`: false
   - `externalFcrepoHost`: "myfedora.hostname.example.com"
 
+
+# Optional Fits Service
+
+Running fits as a servlet will result in much faster fits responses. This is because the loading cost of the
+various Java components far out weighs the processing time of those components.
+
+To enable fits as a service:
+
+  - ``fits.enabled: true` or `externalFits.enabled: true` and specify the
+    existing `externalFits.url`
+
 ## For DevOps:
 
 For those interested in trying out or contributing to this Chart, it's helpful

diff --git a/chart/hyrax/templates/_helpers.tpl b/chart/hyrax/templates/_helpers.tpl
@@ -86,6 +86,23 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end }}
 {{- end -}}
 
+
+{{/*
+Create default fully qualified service names.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "hyrax.fits.fullname" -}}
+{{- printf "%s-%s" .Release.Name "fits" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "hyrax.fits.host" -}}
+{{- if .Values.fits.enabled }}
+{{- include "hyrax.fits.fullname" . }}
+{{- else }}
+{{- .Values.externalFitsHost | default "NO_FITS_HOST_DEFINED" }}
+{{- end }}
+{{- end -}}
+
 {{- define "hyrax.memcached.fullname" -}}
 {{- printf "%s-%s" .Release.Name "memcached" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

diff --git a/chart/hyrax/templates/configmap-env.yaml b/chart/hyrax/templates/configmap-env.yaml
@@ -8,6 +8,11 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
+  {{- if (or .Values.fits.enabled .Values.externalFits.enabled) }}
+  CH12N_TOOL: "fits_servlet"
+  {{- else }}
+  CH12N_TOOL: "fits"
+  {{- end }}
   DB_HOST: {{ template "hyrax.postgresql.host" . }}
   DB_PORT: "5432"
   DB_USERNAME: {{ template "hyrax.postgresql.database" . }}
@@ -29,6 +34,11 @@ data:
   FCREPO_BASE_PATH: {{ .Values.fcrepoBasePathOverride | default (printf "/%s" (include "hyrax.fullname" .)) | quote }}
   FCREPO_PORT: {{ .Values.fcrepo.servicePort | default 8080 | quote }}
   FCREPO_REST_PATH: {{ .Values.fcrepo.restPath | default "rest" }}
+  {{- if .Values.fits.enabled }}
+  FITS_SERVLET_URL: http://{{ template "hyrax.fits.host" . }}:{{ .Values.fits.servicePort | default 8080 }}{{ .Values.fits.subPath | default "/fits" }}
+  {{- else if .Values.externalFits.enabled }}
+  FITS_SERVLET_URL: {{ .Values.externalFits.url }}
+  {{- end }}
   REDIS_PROVIDER: SIDEKIQ_REDIS_URL
   {{- if .Values.minio.enabled }}
   MINIO_ENDPOINT: {{ template "hyrax.minio.fullname" . }}

diff --git a/chart/hyrax/values.yaml b/chart/hyrax/values.yaml
@@ -228,6 +228,17 @@ fcrepo:
       repository: bitnami/postgresql
       tag: 12.3.0
 
+# configuration for an external/existing fits service;
+#   ignored if `fits.enabled` is true
+externalFits:
+  enabled: false
+  url: "" # existing FITS servlet endpoint
+
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
+
 memcached:
   enabled: false
 

diff --git a/docker-compose-koppie.yml b/docker-compose-koppie.yml
@@ -115,6 +115,13 @@ services:
     networks:
       - koppie
 
+  fits:
+    image: harvardlts/fitsservlet_container:1.5.0
+    ports:
+      - 8080
+    networks:
+      - koppie
+
   memcached:
     image: bitnami/memcached
     ports:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -122,6 +122,13 @@ services:
     networks:
       - hyrax
 
+  fits:
+    image: harvardlts/fitsservlet_container:1.5.0
+    ports:
+      - 8080
+    networks:
+      - hyrax
+
   memcached:
     image: bitnami/memcached
     ports:

diff --git a/lib/hyrax/configuration.rb b/lib/hyrax/configuration.rb
@@ -518,6 +518,12 @@ def characterization_proxy
       @characterization_proxy ||= :original_file
     end
 
+    ##
+    # Options to pass to the characterization service
+    # @!attribute [rw] characterization_options
+    #  @return [Hash] of options like {ch12n_tool: :fits_servlet}
+    attr_accessor :characterization_options
+
     # Attributes for the lock manager which ensures a single process/thread is mutating a ore:Aggregation at once.
     # @!attribute [w] lock_retry_count
     #   How many times to retry to acquire the lock before raising UnableToAcquireLockError