Skip to content

Commit

Permalink
[i858] - Enables chunked uploads for improved file upload reliability (…
Browse files Browse the repository at this point in the history 
…#982)

* [i858] - Enables chunked uploads for improved file upload reliability and efficiency

Issue:
- notch8/adventist_knapsack#858

Chunked uploads allow large files to be uploaded in smaller parts, addressing common issues with file size limits, network interruptions, and server resource management.

This change supports smoother, more reliable uploads for large files in Bulkrax.

* Adds Hyrax file uploader to bagit, and xml partial

* ♻️ Move common code into a shared partial

* 🎁 Adds custom js validations to add/remove required attr

* Update importers_controller_spec.rb

* remove upperbound for Hyrax version

* bump versions to make bundling on arm processors work again

* ✅ adds Hyrax::UploadedFile context to importers controller spec

* ♻️ Move Bulkrax uploader logic to importers.js and ensure conditional initialization

- Moved Bulkrax file upload logic from bulkrax.js to importers.js to consolidate related logic
- Added conditional check to initialize uploader only if `hyraxUploader` is defined as a function
- Ensured toggle of 'required' attribute based on uploaded file presence for import form consistency

* re add deleted comments

---------

Co-authored-by: Rob Kaufman <rob@notch8.com>
  • Loading branch information
@ShanaLMoore @orangewolf
ShanaLMoore and orangewolf authored Oct 30, 2024
1 parent f4fb52d commit 77528bf
Show file tree
Hide file tree
Showing 11 changed files with 272 additions and 127 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ spec/test_app/log/*.log
spec/test_app/tmp/
tmp/**
*~undo-tree~
/vendor
4 changes: 4 additions & 0 deletions Gemfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ group :development, :test do
gem 'sqlite3', '~> 1.4'
end

group :test do
gem 'rails-controller-testing'
end

group :lint do
gem 'bixby'
gem 'rubocop-factory_bot', require: false
Expand Down
16 changes: 9 additions & 7 deletions app/assets/javascripts/bulkrax/bulkrax.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,20 @@ $(document).on('turbolinks:load ready', function() {
$('button#err_toggle').click(function() {
$('#error_trace').toggle();
});

$('button#fm_toggle').click(function() {
$('#field_mapping').toggle();
});

$('#bulkraxItemModal').on('show.bs.modal', function (event) {
var button = $(event.relatedTarget) // Button that triggered the modal
var recipient = button.data('entry-id') // Extract info from data-* attributes
var button = $(event.relatedTarget); // Button that triggered the modal
var recipient = button.data('entry-id'); // Extract info from data-* attributes
// If necessary, you could initiate an AJAX request here (and then do the updating in a callback).
// Update the modal's content. We'll use jQuery here, but you could use a data binding library or other methods instead.
var modal = $(this)
var modal = $(this);
modal.find('a').each(function() {
this.href = this.href.replace(/\d+\?/, recipient + '?')
})
return true
})
this.href = this.href.replace(/\d+\?/, recipient + '?');
});
return true;
});
});
233 changes: 134 additions & 99 deletions app/assets/javascripts/bulkrax/importers.js.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,205 +2,240 @@
// All this logic will automatically be available in application.js.

function prepBulkrax(event) {
var refresh_button = $('.refresh-set-source')
var base_url = $('#importer_parser_fields_base_url')
var initial_base_url = base_url.val()
var file_path_value = $('#importer_parser_fields_import_file_path').val()
handleFileToggle(file_path_value)
var refresh_button = $('.refresh-set-source');
var base_url = $('#importer_parser_fields_base_url');
var initial_base_url = base_url.val();
var file_path_value = $('#importer_parser_fields_import_file_path').val();
handleFileToggle(file_path_value);

// Initialize the uploader only if hyraxUploader is defined
if (typeof $.fn.hyraxUploader === 'function') {
// Initialize the uploader
$('.fileupload-bulkrax').hyraxUploader({ maxNumberOfFiles: 1 });

// Function to toggle 'required' attribute based on uploaded files
function toggleRequiredAttribute() {
const fileInput = $('#addfiles');
const uploadedFilesTable = $('.fileupload-bulkrax tbody.files');

if (uploadedFilesTable.find('tr.template-download').length > 0) {
// Remove 'required' if there are uploaded files
fileInput.removeAttr('required');
} else {
// Add 'required' if no uploaded files
fileInput.attr('required', 'required');
}
}

// Check the required attribute when a file is added or removed
$('#addfiles').on('change', function() {
toggleRequiredAttribute();
});

// Also check when an upload completes or is canceled
$('.fileupload-bulkrax').on('fileuploadcompleted fileuploaddestroyed', function() {
toggleRequiredAttribute();
});

// Ensure 'required' is only added if there are no files on form reset
$('#file-upload-cancel-btn').on('click', function() {
$('#addfiles').attr('required', 'required');
$('#addfiles').val(''); // Clear file input to ensure 'required' behavior resets
});

// Initial check in case files are already uploaded
toggleRequiredAttribute();
}

// handle refreshing/loading of external sets via button click
$('body').on('click', '.refresh-set-source', function(e) {
e.preventDefault()
e.preventDefault();

external_set_select = $("#importer_parser_fields_set")
handleSourceLoad(refresh_button, base_url, external_set_select)
})
external_set_select = $("#importer_parser_fields_set");
handleSourceLoad(refresh_button, base_url, external_set_select);
});

// handle refreshing/loading of external sets via blur event for the base_url field
$('body').on('blur', '#importer_parser_fields_base_url', function(e) {
e.preventDefault()
e.preventDefault();

// retrieve the latest base_url
base_url = $('#importer_parser_fields_base_url')
base_url = $('#importer_parser_fields_base_url');

// ensure we don't make another query if the value is the same -- this can be forced by clicking the refresh button
if (initial_base_url != base_url.val()) {
external_set_select = $("#importer_parser_fields_set")
handleSourceLoad(refresh_button, base_url, external_set_select)
initial_base_url = base_url.val()
external_set_select = $("#importer_parser_fields_set");
handleSourceLoad(refresh_button, base_url, external_set_select);
initial_base_url = base_url.val();
}
})
});

// hide and show correct parser fields depending on klass setting
$('body').on('change', '#importer_parser_klass', function(e) {
handleParserKlass()
})
handleParserKlass()
handleParserKlass();
});
handleParserKlass();

// observer for cloud files being added
var form = document.getElementById('new_importer');
if (form == null) {
var form = document.getElementsByClassName('edit_importer')[0];
form = document.getElementsByClassName('edit_importer')[0];
}

// only setup the observer on the new and edit importer pages
if (form != null) {
var config = { childList: true, attributes: true };
var callback = function(mutationsList) {
for(var mutation of mutationsList) {

if (mutation.type == 'childList') {
browseButton = document.getElementById('browse');
var exp = /selected_files\[[0-9]*\]\[url\]/
var exp = /selected_files\[[0-9]*\]\[url\]/;
for (var node of mutation.addedNodes) {
if (node.attributes != undefined) {
var name = node.attributes.name.value
var name = node.attributes.name.value;
if (exp.test(name)) {
browseButton.innerHTML = 'Cloud Files Added';
browseButton.style.backgroundColor = 'green';
browseButton.after(document.createElement("br"), node.value.toString())
browseButton.after(document.createElement("br"), node.value.toString());
}
}
}
}
}
};
var observer = new MutationObserver (callback);
observer.observe (form, config);
var observer = new MutationObserver(callback);
observer.observe(form, config);
}
}

function handleFileToggle(file_path) {
if (file_path === undefined || file_path.length === 0) {
$('#file_path').hide()
$('#file_upload').hide()
$('#cloud').hide()
$('#existing_options').hide()
$('#file_path input').attr('required', null)
$('#file_upload input').attr('required', null)
$('#file_path').hide();
$('#file_upload').hide();
$('#cloud').hide();
$('#existing_options').hide();
$('#file_path input').attr('required', null);
$('#file_upload input').attr('required', null);
} else {
$('#file_path').show()
$('#file_upload').hide()
$('#cloud').hide()
$('#existing_options').hide()
$('#file_path input').attr('required', 'required')
$('#file_upload input').attr('required', null)
$('#importer_parser_fields_file_style_specify_a_path_on_the_server').attr('checked', true)
$('#file_path').show();
$('#file_upload').hide();
$('#cloud').hide();
$('#existing_options').hide();
$('#file_path input').attr('required', 'required');
$('#file_upload input').attr('required', null);
$('#importer_parser_fields_file_style_specify_a_path_on_the_server').attr('checked', true);
}

$('#importer_parser_fields_file_style_upload_a_file').click(function(e){
$('#file_path').hide()
$('#file_upload').show()
$('#cloud').hide()
$('#existing_options').hide()
$('#file_path input').attr('required', null)
$('#file_upload input').attr('required', 'required')
})
$('#file_path').hide();
$('#file_upload').show();
$('#cloud').hide();
$('#existing_options').hide();
$('#file_path input').attr('required', null);
$('#file_upload input').attr('required', 'required');
});
$('#importer_parser_fields_file_style_specify_a_path_on_the_server').click(function(e){
$('#file_path').show()
$('#file_upload').hide()
$('#cloud').hide()
$('#existing_options').hide()
$('#file_path input').attr('required', 'required')
$('#file_upload input').attr('required', null)
})
$('#file_path').show();
$('#file_upload').hide();
$('#cloud').hide();
$('#existing_options').hide();
$('#file_path input').attr('required', 'required');
$('#file_upload input').attr('required', null);
});
$('#importer_parser_fields_file_style_add_cloud_file').click(function(e){
$('#file_path').hide()
$('#file_upload').hide()
$('#cloud').show()
$('#existing_options').hide()
$('#file_path input').attr('required', null)
$('#file_upload input').attr('required', null)
})
$('#file_path').hide();
$('#file_upload').hide();
$('#cloud').show();
$('#existing_options').hide();
$('#file_path input').attr('required', null);
$('#file_upload input').attr('required', null);
});
$('#importer_parser_fields_file_style_existing_entries').click(function(e){
$('#file_path').hide()
$('#file_upload').hide()
$('#cloud').hide()
$('#existing_options').show()
$('#file_path input').attr('required', null)
$('#file_upload input').attr('required', null)
})

$('#file_path').hide();
$('#file_upload').hide();
$('#cloud').hide();
$('#existing_options').show();
$('#file_path input').attr('required', null);
$('#file_upload input').attr('required', null);
});
}

function handleParserKlass() {
<% Bulkrax.parsers.map{ |p| p[:partial]}.uniq.each do |partial| %>
if($('.<%= partial %>').length > 0) {
window.<%= partial %> = $('.<%= partial %>').detach()
window.<%= partial %> = $('.<%= partial %>').detach();
}
<% end %>

var parser_klass = $("#importer_parser_klass option:selected")
var parser_klass = $("#importer_parser_klass option:selected");
if(parser_klass.length > 0 && parser_klass.data('partial') && parser_klass.data('partial').length > 0) {
$('.parser_fields').append(window[parser_klass.data('partial')])
$('.parser_fields').append(window[parser_klass.data('partial')]);
}

handleBrowseEverything()
var file_path_value = $('#importer_parser_fields_import_file_path').val()
handleFileToggle(file_path_value)
handleBrowseEverything();
var file_path_value = $('#importer_parser_fields_import_file_path').val();
handleFileToggle(file_path_value);
}

function handleBrowseEverything(){
var button = $("button[data-toggle='browse-everything']")
var button = $("button[data-toggle='browse-everything']");
if(button.length == 0) { return; }
button.browseEverything({
route: button.data('route'),
target: button.data('target')
}).done(function(data) {
var evt = { isDefaultPrevented: function() { return false; } };
$('.ev-browser.show').removeClass('show')
$('.ev-browser.show').removeClass('show');
if($('#fileupload').length > 0) {
var files = $.map(data, function(d) { return { name: d.file_name, size: d.file_size, id: d.url } });
var files = $.map(data, function(d) { return { name: d.file_name, size: d.file_size, id: d.url }; });
$.blueimp.fileupload.prototype.options.done.call($('#fileupload').fileupload(), evt, { result: { files: files }});
}
return true
// User has submitted files; data contains an array of URLs and their options
return true;
}).cancel(function() {
$('.ev-browser.show').removeClass('show')
// User cancelled the browse operation
$('.ev-browser.show').removeClass('show');
}).fail(function(status, error, text) {
$('.ev-browser.show').removeClass('show')
// URL retrieval experienced a technical failure
$('.ev-browser.show').removeClass('show');
});
}

function handleSourceLoad(refresh_button, base_url, external_set_select) {
if (base_url.val() == "") { // ignore empty base_url value
return
return;
}

var initial_button_text = refresh_button.html()
var initial_button_text = refresh_button.html();

refresh_button.html('Refreshing...')
refresh_button.attr('disabled', true)
refresh_button.html('Refreshing...');
refresh_button.attr('disabled', true);

$.post('/importers/external_sets', {
base_url: base_url.val(),
}, function(res) {
if (!res.error) {
genExternalSetOptions(external_set_select, res.sets) // sets is [[name, spec]...]
genExternalSetOptions(external_set_select, res.sets);
} else {
setError(external_set_select, res.error)
setError(external_set_select, res.error);
}

refresh_button.html(initial_button_text)
refresh_button.attr('disabled', false)
})
refresh_button.html(initial_button_text);
refresh_button.attr('disabled', false);
});
}

function genExternalSetOptions(selector, sets) {
out = '<option value="">- Select One -</option>'
out = '<option value="">- Select One -</option>';

out += sets.map(function(set) {
return '<option value="'+set[1]+'">'+set[0]+'</option>'
})
return '<option value="'+set[1]+'">'+set[0]+'</option>';
});

selector.html(out)
selector.attr('disabled', false)
selector.html(out);
selector.attr('disabled', false);
}

function setError(selector, error) {
selector.html('<option value="none">Error - Please enter Base URL and try again</option>')
selector.attr('disabled', true)
selector.html('<option value="none">Error - Please enter Base URL and try again</option>');
selector.attr('disabled', true);
}

$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax})
$(document).on({'ready': prepBulkrax, 'turbolinks:load': prepBulkrax});
Loading

0 comments on commit 77528bf

Please sign in to comment.