[i858] - Enables chunked uploads for improved file upload reliability…

… and efficiency Issue: - scientist-softserv/adventist_knapsack#858 Chunked uploads allow large files to be uploaded in smaller parts, addressing common issues with file size limits, network interruptions, and server resource management. This change supports smoother, more reliable uploads for large files in Bulkrax.
samvera · Oct 25, 2024 · 3701aad · 3701aad
1 parent 92d6042
commit 3701aad
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 14 deletions.
diff --git a/app/assets/javascripts/bulkrax/bulkrax.js b/app/assets/javascripts/bulkrax/bulkrax.js
@@ -19,4 +19,5 @@ $(document).on('turbolinks:load ready', function() {
     })
     return true
   })
+  $('#fileupload-bulkrax').hyraxUploader({maxNumberOfFiles: 1});
 });
diff --git a/app/controllers/bulkrax/importers_controller.rb b/app/controllers/bulkrax/importers_controller.rb
@@ -83,6 +83,7 @@ def create
       if api_request?
         return return_json_response unless valid_create_params?
       end
+      uploads = Hyrax::UploadedFile.find(params[:uploaded_files]) if params[:uploaded_files].present?
       file = file_param
       cloud_files = cloud_params
 
@@ -93,7 +94,7 @@ def create
       # on a new import otherwise it only gets updated during the update path
       @importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import'
       if @importer.save
-        files_for_import(file, cloud_files)
+        files_for_import(file, cloud_files, uploads)
         if params[:commit] == 'Create and Import'
           Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id)
           render_request('Importer was successfully created and import has been queued.')
@@ -120,15 +121,15 @@ def update
       if api_request?
         return return_json_response unless valid_update_params?
       end
-
+      uploads = Hyrax::UploadedFile.find(params[:uploaded_files]) if params[:uploaded_files].present?
       file = file_param
       cloud_files = cloud_params
 
       # Skipped during a continue
       field_mapping_params if params[:importer][:parser_fields].present?
 
       if @importer.update(importer_params)
-        files_for_import(file, cloud_files) unless file.nil? && cloud_files.nil?
+        files_for_import(file, cloud_files, uploads)
         # do not perform the import
         unless params[:commit] == 'Update Importer'
           set_files_parser_fields
@@ -218,8 +219,9 @@ def export_errors
 
     private
 
-    def files_for_import(file, cloud_files)
-      return if file.blank? && cloud_files.blank?
+    def files_for_import(file, cloud_files, uploads)
+      return if file.blank? && cloud_files.blank? && uploads.blank?
+
       @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present?
       if cloud_files.present?
         @importer[:parser_fields]['cloud_file_paths'] = cloud_files
@@ -229,8 +231,16 @@ def files_for_import(file, cloud_files)
         target = @importer.parser.retrieve_cloud_files(cloud_files, @importer)
         @importer[:parser_fields]['import_file_path'] = target if target.present?
       end
+
+      if uploads.present?
+        uploads.each do |upload|
+          @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(upload.file.file)
+        end
+      end
+
       @importer.save
     end
+
 
     # Use callbacks to share common setup or constraints between actions.
     def set_importer

diff --git a/app/views/bulkrax/importers/_csv_fields.html.erb b/app/views/bulkrax/importers/_csv_fields.html.erb
@@ -1,5 +1,4 @@
 <div class='csv_fields'>
-
   <%= fi.input :visibility,
     label: 'Default Visibility',
     collection: [
@@ -11,7 +10,6 @@
     input_html: { class: 'form-control' },
     hint: 'If your CSV includes the visibility field, it will override the default setting.'
   %>
-
   <% if defined?(::Hyrax) %>
     <% rights_statements = Hyrax.config.rights_statement_service_class.new %>
     <%= fi.input :rights_statement,
@@ -24,26 +22,63 @@
           %>
     <%= fi.input :override_rights_statement, as: :boolean, hint: 'If checked, always use the selected rights statment. If unchecked, use rights or rights_statement from the record and only use the provided value if dc:rights is blank.', input_html: { checked: (importer.parser_fields['override_rights_statement'] == "1") } %>
   <% end %>
-  <h4>Add CSV File to Import:</h4>
+  <h4>Add CSV or ZIP File to Import:</h4>
   <%# accept a single file upload; data files and bags will need to be added another way %>
-
   <% file_style_list = ['Upload a File', 'Specify a Path on the Server'] %>
   <% file_style_list << 'Existing Entries' unless importer.new_record? %>
   <%= fi.input :file_style, collection: file_style_list, as: :radio_buttons, label: false %>
+
   <div id='file_upload'>
-    <%= fi.input 'file', as: :file, input_html: { accept: 'text/csv,application/zip,application/gzip' } %><br />
+    <div id="fileupload-bulkrax">
+      <noscript><input type="hidden" name="redirect" value="<%= main_app.root_path %>" /></noscript>
+      <table role="presentation" class="table table-striped"><tbody class="files"></tbody></table>
+      <div class="fileupload-buttonbar">
+        <div class="row">
+          <div class="col-12">
+            <div class="fileinput-button" id="add-files">
+              <input id="addfiles" type="file" style="display:none;" name="files[]" accept="text/csv,application/zip,application/gzip" />
+              <button type="button" class="btn btn-success" onclick="document.getElementById('addfiles').click();">
+                <span class="fa fa-plus"></span>
+                <span>Add Files</span>
+              </button>
+            </div>
+            <button type="reset" id="file-upload-cancel-btn" class="btn btn-warning cancel">
+              <span class="fa fa-ban"></span>
+              <span>Cancel Upload</span>
+            </button>
+            <span class="fileupload-process"></span>
+          </div>
+        </div>
+        <div class="row">
+          <div class="col-12">
+            <div class="fileupload-progress fade">
+              <div class="progress" role="progressbar" aria-valuemin="0" aria-valuemax="100">
+                <div class="progress-bar progress-bar-striped progress-bar-animated bg-success" style="width:0%;"></div>
+              </div>
+              <div class="progress-extended">&nbsp;</div>
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="dropzone">
+        Drop files here to upload
+      </div>
+    </div>
+    <%= render 'hyrax/uploads/js_templates' %>
   </div>
+
   <div id='file_path'>
     <%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
   </div>
+
   <div id='existing_options'>
     <%= fi.collection_check_boxes :entry_statuses, [['Failed'], ['Pending'], ['Skipped'], ['Deleted'], ['Complete']], :first, :first %>
   </div>
-
+  
   <% if defined?(::Hyrax) && Hyrax.config.browse_everything? %>
-	  <h4>Add Files to Import:</h4>
-	  <p>Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.</p>
+    <h4>Add Files to Import:</h4>
+    <p>Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.</p>
     <%= render 'browse_everything', form: form %>
   <% end %>
   <br />
-</div>
+</div>