From 3701aad7776c5a9d1ac3319e29627f461719c89c Mon Sep 17 00:00:00 2001 From: Shana Moore Date: Fri, 25 Oct 2024 12:22:05 -0700 Subject: [PATCH] [i858] - Enables chunked uploads for improved file upload reliability and efficiency Issue: - https://github.com/scientist-softserv/adventist_knapsack/issues/858 Chunked uploads allow large files to be uploaded in smaller parts, addressing common issues with file size limits, network interruptions, and server resource management. This change supports smoother, more reliable uploads for large files in Bulkrax. --- app/assets/javascripts/bulkrax/bulkrax.js | 1 + .../bulkrax/importers_controller.rb | 20 +++++-- .../bulkrax/importers/_csv_fields.html.erb | 53 +++++++++++++++---- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/app/assets/javascripts/bulkrax/bulkrax.js b/app/assets/javascripts/bulkrax/bulkrax.js index 006ce820..7720c2a4 100644 --- a/app/assets/javascripts/bulkrax/bulkrax.js +++ b/app/assets/javascripts/bulkrax/bulkrax.js @@ -19,4 +19,5 @@ $(document).on('turbolinks:load ready', function() { }) return true }) + $('#fileupload-bulkrax').hyraxUploader({maxNumberOfFiles: 1}); }); diff --git a/app/controllers/bulkrax/importers_controller.rb b/app/controllers/bulkrax/importers_controller.rb index 2fe5c470..d6d1355b 100644 --- a/app/controllers/bulkrax/importers_controller.rb +++ b/app/controllers/bulkrax/importers_controller.rb @@ -83,6 +83,7 @@ def create if api_request? return return_json_response unless valid_create_params? end + uploads = Hyrax::UploadedFile.find(params[:uploaded_files]) if params[:uploaded_files].present? file = file_param cloud_files = cloud_params @@ -93,7 +94,7 @@ def create # on a new import otherwise it only gets updated during the update path @importer.parser_fields['update_files'] = true if params[:commit] == 'Create and Import' if @importer.save - files_for_import(file, cloud_files) + files_for_import(file, cloud_files, uploads) if params[:commit] == 'Create and Import' Bulkrax::ImporterJob.send(@importer.parser.perform_method, @importer.id) render_request('Importer was successfully created and import has been queued.') @@ -120,7 +121,7 @@ def update if api_request? return return_json_response unless valid_update_params? end - + uploads = Hyrax::UploadedFile.find(params[:uploaded_files]) if params[:uploaded_files].present? file = file_param cloud_files = cloud_params @@ -128,7 +129,7 @@ def update field_mapping_params if params[:importer][:parser_fields].present? if @importer.update(importer_params) - files_for_import(file, cloud_files) unless file.nil? && cloud_files.nil? + files_for_import(file, cloud_files, uploads) # do not perform the import unless params[:commit] == 'Update Importer' set_files_parser_fields @@ -218,8 +219,9 @@ def export_errors private - def files_for_import(file, cloud_files) - return if file.blank? && cloud_files.blank? + def files_for_import(file, cloud_files, uploads) + return if file.blank? && cloud_files.blank? && uploads.blank? + @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(file) if file.present? if cloud_files.present? @importer[:parser_fields]['cloud_file_paths'] = cloud_files @@ -229,8 +231,16 @@ def files_for_import(file, cloud_files) target = @importer.parser.retrieve_cloud_files(cloud_files, @importer) @importer[:parser_fields]['import_file_path'] = target if target.present? end + + if uploads.present? + uploads.each do |upload| + @importer[:parser_fields]['import_file_path'] = @importer.parser.write_import_file(upload.file.file) + end + end + @importer.save end + # Use callbacks to share common setup or constraints between actions. def set_importer diff --git a/app/views/bulkrax/importers/_csv_fields.html.erb b/app/views/bulkrax/importers/_csv_fields.html.erb index fba5ce0e..31f3719f 100644 --- a/app/views/bulkrax/importers/_csv_fields.html.erb +++ b/app/views/bulkrax/importers/_csv_fields.html.erb @@ -1,5 +1,4 @@
- <%= fi.input :visibility, label: 'Default Visibility', collection: [ @@ -11,7 +10,6 @@ input_html: { class: 'form-control' }, hint: 'If your CSV includes the visibility field, it will override the default setting.' %> - <% if defined?(::Hyrax) %> <% rights_statements = Hyrax.config.rights_statement_service_class.new %> <%= fi.input :rights_statement, @@ -24,26 +22,63 @@ %> <%= fi.input :override_rights_statement, as: :boolean, hint: 'If checked, always use the selected rights statment. If unchecked, use rights or rights_statement from the record and only use the provided value if dc:rights is blank.', input_html: { checked: (importer.parser_fields['override_rights_statement'] == "1") } %> <% end %> -

Add CSV File to Import:

+

Add CSV or ZIP File to Import:

<%# accept a single file upload; data files and bags will need to be added another way %> - <% file_style_list = ['Upload a File', 'Specify a Path on the Server'] %> <% file_style_list << 'Existing Entries' unless importer.new_record? %> <%= fi.input :file_style, collection: file_style_list, as: :radio_buttons, label: false %> +
- <%= fi.input 'file', as: :file, input_html: { accept: 'text/csv,application/zip,application/gzip' } %>
+
+ + +
+
+
+
+ + +
+ + +
+
+
+
+
+
+
+
+
 
+
+
+
+
+
+ Drop files here to upload +
+
+ <%= render 'hyrax/uploads/js_templates' %>
+
<%= fi.input :import_file_path, as: :string, input_html: { value: importer.parser_fields['import_file_path'] } %>
+
<%= fi.collection_check_boxes :entry_statuses, [['Failed'], ['Pending'], ['Skipped'], ['Deleted'], ['Complete']], :first, :first %>
- + <% if defined?(::Hyrax) && Hyrax.config.browse_everything? %> -

Add Files to Import:

-

Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.

+

Add Files to Import:

+

Choose files to upload. The filenames must be unique, and the filenames must be referenced in a column called 'file' in the accompanying CSV file.

<%= render 'browse_everything', form: form %> <% end %>
-
+ \ No newline at end of file