From 4bad9e63a7806a154fa74aa349bd0da937bfbda5 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Fri, 24 Mar 2023 14:17:02 -0400 Subject: [PATCH 001/396] Handle invalid URI errors --- app/controllers/application_controller.rb | 23 ++++++++++++++++--- .../master_files_controller_spec.rb | 18 ++++++++++++--- .../media_objects_controller_spec.rb | 18 ++++++++++++--- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 7077cd790f..5c9965d956 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -21,6 +21,7 @@ class ApplicationController < ActionController::Base include Hydra::Controller::ControllerBehavior # To deal with a load order issue breaking persona impersonate include Samvera::Persona::BecomesBehavior + include Identifier layout 'avalon' # Prevent CSRF attacks by raising an exception. @@ -186,6 +187,22 @@ def current_ability end end + rescue_from URI::InvalidURIError do |exception| + # Strip all non-alphanumeric characters from passed in NOID + noid_id = params[:id]&.gsub(/[^A-Za-z0-9]/, '') + + # If cleaned NOID is valid, redo the request. Otherwise generate an error page. + if noid_service.valid?(noid_id) + redirect_to(request.parameters.merge(id: noid_id)) + else + if request.format == :json + render json: {errors: ["#{params[:id]} not found"]}, status: 404 + else + render '/errors/unknown_pid', status: 404 + end + end + end + def configure_permitted_parameters devise_parameter_sanitizer.permit(:sign_up) do |user_params| user_params.permit(:username, :email, :password, :password_confirmation) diff --git a/spec/controllers/master_files_controller_spec.rb b/spec/controllers/master_files_controller_spec.rb index 13d5c63001..f51ce554d3 100644 --- a/spec/controllers/master_files_controller_spec.rb +++ b/spec/controllers/master_files_controller_spec.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -238,6 +238,18 @@ class << file expect(get(:show, params: { id: fedora3_pid })).to redirect_to(master_file_url(master_file.id)) end end + + context 'misformed NOID' do + # URI::InvalidURIError handling in ApplicationController + it 'should redirect to the requested master file' do + get 'show', params: { id: "#{master_file.id}] " } + expect(response).to redirect_to(master_file_path(id: master_file.id)) + end + it 'should redirect to homepage if invalid' do + get 'show', params: { id: "nonvalid noid]" } + expect(response).to render_template("errors/unknown_pid") + end + end end describe "#embed" do diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb index 92cfa97e7b..05fa1a8641 100644 --- a/spec/controllers/media_objects_controller_spec.rb +++ b/spec/controllers/media_objects_controller_spec.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -1239,6 +1239,18 @@ end end end + + context 'misformed NOID' do + # URI::InvalidURIError handling in ApplicationController + it 'should redirect to the requested media object if valid' do + get 'show', params: { id: "#{media_object.id}] !-()" } + expect(response).to redirect_to(media_object_path(id: media_object.id)) + end + it 'should redirect to homepage if invalid' do + get 'show', params: { id: "nonvalid noid]" } + expect(response).to render_template("errors/unknown_pid") + end + end end describe "#destroy" do From 68599114fcce5a560bb43c59f59d77c4c2328b43 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Wed, 29 Mar 2023 16:28:56 -0400 Subject: [PATCH 002/396] Refactor noid validation into module --- .../admin/collections_controller.rb | 7 ++-- app/controllers/application_controller.rb | 17 -------- app/controllers/concerns/noid_validator.rb | 39 +++++++++++++++++++ app/controllers/master_files_controller.rb | 7 ++-- app/controllers/media_objects_controller.rb | 9 +++-- .../admin_collections_controller_spec.rb | 20 ++++++++-- .../master_files_controller_spec.rb | 4 +- .../media_objects_controller_spec.rb | 6 +-- 8 files changed, 73 insertions(+), 36 deletions(-) create mode 100644 app/controllers/concerns/noid_validator.rb diff --git a/app/controllers/admin/collections_controller.rb b/app/controllers/admin/collections_controller.rb index d62870d2a5..dc8bf92529 100644 --- a/app/controllers/admin/collections_controller.rb +++ b/app/controllers/admin/collections_controller.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -13,6 +13,7 @@ # --- END LICENSE_HEADER BLOCK --- class Admin::CollectionsController < ApplicationController + include NoidValidator include Rails::Pagination before_action :authenticate_user! diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 5c9965d956..25dd22dbb1 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -21,7 +21,6 @@ class ApplicationController < ActionController::Base include Hydra::Controller::ControllerBehavior # To deal with a load order issue breaking persona impersonate include Samvera::Persona::BecomesBehavior - include Identifier layout 'avalon' # Prevent CSRF attacks by raising an exception. @@ -187,22 +186,6 @@ def current_ability end end - rescue_from URI::InvalidURIError do |exception| - # Strip all non-alphanumeric characters from passed in NOID - noid_id = params[:id]&.gsub(/[^A-Za-z0-9]/, '') - - # If cleaned NOID is valid, redo the request. Otherwise generate an error page. - if noid_service.valid?(noid_id) - redirect_to(request.parameters.merge(id: noid_id)) - else - if request.format == :json - render json: {errors: ["#{params[:id]} not found"]}, status: 404 - else - render '/errors/unknown_pid', status: 404 - end - end - end - def configure_permitted_parameters devise_parameter_sanitizer.permit(:sign_up) do |user_params| user_params.permit(:username, :email, :password, :password_confirmation) diff --git a/app/controllers/concerns/noid_validator.rb b/app/controllers/concerns/noid_validator.rb new file mode 100644 index 0000000000..0427e14311 --- /dev/null +++ b/app/controllers/concerns/noid_validator.rb @@ -0,0 +1,39 @@ +# Copyright 2011-2023, The Trustees of Indiana University and Northwestern +# University. Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# --- END LICENSE_HEADER BLOCK --- + +module NoidValidator + extend ActiveSupport::Concern + + include Identifier + + included do + before_action :validate_noid, only: [:show] + end + + private + + def validate_noid + return if noid_service.valid?(params[:id]) + + # Strip all non-alphanumeric characters from passed in NOID + noid_id = params[:id]&.gsub(/[^A-Za-z0-9]/, '') + + # If cleaned NOID is valid, redo the request. Otherwise raise error. + if noid_service.valid?(noid_id) + redirect_to(request.parameters.merge(id: noid_id)) + else + raise ActiveFedora::ObjectNotFoundError + end + end +end diff --git a/app/controllers/master_files_controller.rb b/app/controllers/master_files_controller.rb index d16f661c7f..cbb9756a4a 100644 --- a/app/controllers/master_files_controller.rb +++ b/app/controllers/master_files_controller.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -18,6 +18,7 @@ class MasterFilesController < ApplicationController # include Avalon::Controller::ControllerBehavior + include NoidValidator before_action :authenticate_user!, :only => [:create] before_action :set_masterfile, except: [:create, :oembed] diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb index 04bfc4669f..4e7de92727 100644 --- a/app/controllers/media_objects_controller.rb +++ b/app/controllers/media_objects_controller.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -20,6 +20,7 @@ class MediaObjectsController < ApplicationController include Avalon::Workflow::WorkflowControllerBehavior include Avalon::Controller::ControllerBehavior include ConditionalPartials + include NoidValidator include SecurityHelper before_action :authenticate_user!, except: [:show, :set_session_quality, :show_stream_details, :manifest] @@ -180,7 +181,7 @@ def update_media_object render json: { errors: ["Collection not found for #{api_params[:collection_id]}"] }, status: 422 return end - + @media_object.collection = collection end diff --git a/spec/controllers/admin_collections_controller_spec.rb b/spec/controllers/admin_collections_controller_spec.rb index af9b659c6f..1b828da96e 100644 --- a/spec/controllers/admin_collections_controller_spec.rb +++ b/spec/controllers/admin_collections_controller_spec.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -258,6 +258,18 @@ expect(JSON.parse(response.body)["errors"].first.class).to eq String end end + + context 'misformed NOID' do + it 'should redirect to the requested collection' do + get 'show', params: { id: "#{collection.id}] " } + expect(response).to redirect_to(admin_collection_path(id: collection.id)) + end + it 'should redirect to unknown_pid page if invalid' do + get 'show', params: { id: "nonvalid noid]" } + expect(response).to render_template("errors/unknown_pid") + expect(response.response_code).to eq(404) + end + end end describe "#items" do @@ -277,7 +289,7 @@ context "with structure" do let!(:mf_1) { FactoryBot.create(:master_file, :with_structure, media_object: collection.media_objects[0]) } let!(:mf_2) { FactoryBot.create(:master_file, :with_structure, media_object: collection.media_objects[1]) } - + it "should not return structure by default" do get 'items', params: { id: collection.id, format: 'json' } expect(JSON.parse(response.body)[collection.media_objects[0].id]["files"][0]["structure"]).to be_blank diff --git a/spec/controllers/master_files_controller_spec.rb b/spec/controllers/master_files_controller_spec.rb index f51ce554d3..8a8556d829 100644 --- a/spec/controllers/master_files_controller_spec.rb +++ b/spec/controllers/master_files_controller_spec.rb @@ -240,14 +240,14 @@ class << file end context 'misformed NOID' do - # URI::InvalidURIError handling in ApplicationController it 'should redirect to the requested master file' do get 'show', params: { id: "#{master_file.id}] " } expect(response).to redirect_to(master_file_path(id: master_file.id)) end - it 'should redirect to homepage if invalid' do + it 'should redirect to unknown_pid page if invalid' do get 'show', params: { id: "nonvalid noid]" } expect(response).to render_template("errors/unknown_pid") + expect(response.response_code).to eq(404) end end end diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb index 05fa1a8641..19445fae9a 100644 --- a/spec/controllers/media_objects_controller_spec.rb +++ b/spec/controllers/media_objects_controller_spec.rb @@ -784,8 +784,8 @@ end it "should return an error if the PID does not exist" do - expect(MediaObject).to receive(:find).with('no-such-object') { raise ActiveFedora::ObjectNotFoundError } get :show, params: { id: 'no-such-object' } + expect(response).to render_template("errors/unknown_pid") expect(response.response_code).to eq(404) end @@ -1241,14 +1241,14 @@ end context 'misformed NOID' do - # URI::InvalidURIError handling in ApplicationController it 'should redirect to the requested media object if valid' do get 'show', params: { id: "#{media_object.id}] !-()" } expect(response).to redirect_to(media_object_path(id: media_object.id)) end - it 'should redirect to homepage if invalid' do + it 'should redirect to unknown_pid page if invalid' do get 'show', params: { id: "nonvalid noid]" } expect(response).to render_template("errors/unknown_pid") + expect(response.response_code).to eq(404) end end end From 7e41773c108a355e2287c708cf1c315838495f5b Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Wed, 29 Mar 2023 17:15:52 -0400 Subject: [PATCH 003/396] Fixes for codeclimate --- app/controllers/concerns/noid_validator.rb | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/app/controllers/concerns/noid_validator.rb b/app/controllers/concerns/noid_validator.rb index 0427e14311..4cb632b601 100644 --- a/app/controllers/concerns/noid_validator.rb +++ b/app/controllers/concerns/noid_validator.rb @@ -23,17 +23,14 @@ module NoidValidator private - def validate_noid - return if noid_service.valid?(params[:id]) + def validate_noid + return if noid_service.valid?(params[:id]) - # Strip all non-alphanumeric characters from passed in NOID - noid_id = params[:id]&.gsub(/[^A-Za-z0-9]/, '') + # Strip all non-alphanumeric characters from passed in NOID + noid_id = params[:id]&.gsub(/[^A-Za-z0-9]/, '') - # If cleaned NOID is valid, redo the request. Otherwise raise error. - if noid_service.valid?(noid_id) - redirect_to(request.parameters.merge(id: noid_id)) - else - raise ActiveFedora::ObjectNotFoundError + # If cleaned NOID is valid, redo the request. Otherwise raise error. + raise ActiveFedora::ObjectNotFoundError if !noid_service.valid?(noid_id) + redirect_to(request.parameters.merge(:only_path => true, id: noid_id)) end - end end From 5d5e80a96a9bfbb5848139bcdc82114cc1112a82 Mon Sep 17 00:00:00 2001 From: Mason Ballengee <68433277+masaball@users.noreply.github.com> Date: Thu, 30 Mar 2023 09:18:19 -0400 Subject: [PATCH 004/396] Update noid_validator.rb --- app/controllers/concerns/noid_validator.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/concerns/noid_validator.rb b/app/controllers/concerns/noid_validator.rb index 4cb632b601..53d822cc14 100644 --- a/app/controllers/concerns/noid_validator.rb +++ b/app/controllers/concerns/noid_validator.rb @@ -29,8 +29,8 @@ def validate_noid # Strip all non-alphanumeric characters from passed in NOID noid_id = params[:id]&.gsub(/[^A-Za-z0-9]/, '') - # If cleaned NOID is valid, redo the request. Otherwise raise error. - raise ActiveFedora::ObjectNotFoundError if !noid_service.valid?(noid_id) - redirect_to(request.parameters.merge(:only_path => true, id: noid_id)) + # If cleaned NOID is not valid raise error, otherwise redirect request. + raise ActiveFedora::ObjectNotFoundError unless noid_service.valid?(noid_id) + redirect_to(request.parameters.merge(id: noid_id)) end end From e79de74bb0c333b62a94ecd9965bbcac81188808 Mon Sep 17 00:00:00 2001 From: dananji Date: Tue, 4 Apr 2023 13:34:19 -0700 Subject: [PATCH 005/396] Fix overlapping captions in player in iPhone --- .../assets/javascripts/mediaelement/mediaelement-and-player.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js b/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js index 2ad33e56fc..4096ed43b6 100644 --- a/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js +++ b/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js @@ -2777,7 +2777,7 @@ Object.assign(_player2.default.prototype, { } }, displayCaptions: function displayCaptions() { - if (this.tracks === undefined) { + if (this.tracks === undefined || _mejs2.default.Features.isiPhone) { return; } From 7c2bf97ad1005c059906f3643b0958c791217dd6 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Wed, 5 Apr 2023 11:32:29 -0400 Subject: [PATCH 006/396] Cascade user delete to db tables with user.id foreign key --- app/models/user.rb | 8 +++++--- spec/models/user_spec.rb | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 6fa11c9b99..e1a0573f37 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -14,6 +14,8 @@ class User < ActiveRecord::Base has_many :checkouts, dependent: :destroy + has_many :playlists, dependent: :destroy + has_many :timelines, dependent: :destroy attr_writer :login # Connects this user object to Hydra behaviors. include Hydra::User diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index a89677bf6a..0e33afbe12 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -146,6 +146,16 @@ returned_checkout = FactoryBot.create(:checkout, user_id: user.id, return_time: DateTime.current - 1.day) expect { user.destroy }.to change { Checkout.all.count }.from(2).to(0) end + it 'removes playlists for user' do + user = FactoryBot.create(:public) + playlist = FactoryBot.create(:playlist, user_id: user.id) + expect { user.destroy }.to change { Playlist.exists? playlist.id }.from( true ).to( false ) + end + it 'removes timelines for user' do + user = FactoryBot.create(:public) + timeline = FactoryBot.create(:timeline, user_id: user.id) + expect { user.destroy }.to change { Timeline.exists? timeline.id }.from( true ).to( false ) + end end describe '#timeline_tags' do From a2f5d36525062b83548527a45af36c4d0e1ec5f7 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Wed, 5 Apr 2023 12:01:42 -0400 Subject: [PATCH 007/396] Edit user delete confirmation message --- app/controllers/samvera/persona/users_controller.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/samvera/persona/users_controller.rb b/app/controllers/samvera/persona/users_controller.rb index 3c890ad90a..10bb2b211c 100644 --- a/app/controllers/samvera/persona/users_controller.rb +++ b/app/controllers/samvera/persona/users_controller.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -87,7 +87,7 @@ def paged_index view_context.link_to('Edit', main_app.edit_persona_user_path(presenter)) end become_button = view_context.link_to('Become', main_app.impersonate_persona_user_path(presenter), method: :post) - delete_button = view_context.link_to('Delete', main_app.persona_user_path(presenter), method: :delete, class: 'btn btn-danger btn-sm action-delete', data: { confirm: "Are you sure you wish to delete the user '#{presenter.email}'? This action is irreversible." }) + delete_button = view_context.link_to('Delete', main_app.persona_user_path(presenter), method: :delete, class: 'btn btn-danger btn-sm action-delete', data: { confirm: "Are you sure you wish to delete the user '#{presenter.email}'? This action will also delete all playlists and timelines belonging to '#{presenter.email}'. This action is irreversible." }) formatted_roles = format_roles(presenter.groups) sign_in = last_sign_in(presenter) [ From 87d852ce97756d6528a95a7ca20e20cac13ae7eb Mon Sep 17 00:00:00 2001 From: dananji Date: Tue, 11 Apr 2023 15:00:12 -0400 Subject: [PATCH 008/396] Fix flashing error message for mouse-events while scrubbing --- .../mediaelement/mediaelement-and-player.js | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js b/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js index 4096ed43b6..c691764260 100644 --- a/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js +++ b/vendor/assets/javascripts/mediaelement/mediaelement-and-player.js @@ -1698,6 +1698,7 @@ Object.assign(_player2.default.prototype, { buildprogress: function buildprogress(player, controls, layers, media) { var lastKeyPressTime = 0, + lastMouseDownTime = 0, mouseIsDown = false, startedPaused = false; @@ -1900,7 +1901,7 @@ Object.assign(_player2.default.prototype, { t.timefloat.style.display = 'block'; } }, - updateSlider = function updateSlider() { + updateSlider = function updateSlider() { var seconds = t.getCurrentTime(), timeSliderText = _i18n2.default.t('mejs.time-slider'), time = (0, _time.secondsToTimeCode)(seconds, player.options.alwaysShowHours, player.options.showTimecodeFrameCount, player.options.framesPerSecond, player.options.secondsDecimalLength, player.options.timeFormat), @@ -1923,12 +1924,12 @@ Object.assign(_player2.default.prototype, { t.slider.removeAttribute('aria-valuetext'); } }, - restartPlayer = function restartPlayer() { + restartPlayer = function restartPlayer() { if (new Date() - lastKeyPressTime >= 1000) { t.play(); } }, - handleMouseup = function handleMouseup() { + handleMouseup = function handleMouseup(e) { if (mouseIsDown && t.getCurrentTime() !== null && t.newTime.toFixed(4) !== t.getCurrentTime().toFixed(4)) { t.setCurrentTime(t.newTime, true); t.setCurrentRailHandle(t.newTime); @@ -1936,19 +1937,15 @@ Object.assign(_player2.default.prototype, { } if (t.forcedHandlePause) { t.slider.focus(); - setTimeout(function () { - let playPromise = t.play(); - if(playPromise !== undefined) { - playPromise.then(_ => { - // Show play UI, and enable controls - t.play(); - t.enableControls(); - }) - .catch(error => { }) + setTimeout(() => { + if (new Date() - lastMouseDownTime >= 1000) { + t.play(); + t.forcedHandlePause = false; } - }, 150) + }, 1100); } - t.forcedHandlePause = false; + e.preventDefault(); + e.stopPropagation(); }; t.slider.addEventListener('focus', function () { @@ -2052,7 +2049,6 @@ Object.assign(_player2.default.prototype, { for (var i = 0, total = events.length; i < total; i++) { t.slider.addEventListener(events[i], function (e) { - t.forcedHandlePause = false; if (t.getDuration() !== Infinity) { if (e.which === 1 || e.which === 0) { if (!t.paused) { @@ -2061,6 +2057,7 @@ Object.assign(_player2.default.prototype, { } mouseIsDown = true; + lastMouseDownTime = new Date(); handleMouseMove(e); var endEvents = ['mouseup', 'touchend']; @@ -2072,8 +2069,8 @@ Object.assign(_player2.default.prototype, { } }); } - t.globalBind('mouseup.dur touchend.dur', function () { - handleMouseup(); + t.globalBind('mouseup.dur touchend.dur', function (e) { + handleMouseup(e); mouseIsDown = false; if (t.timefloat) { t.timefloat.style.display = 'none'; From a9447488b51471e33c717713fafdace1680c9916 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Tue, 4 Apr 2023 13:54:36 -0400 Subject: [PATCH 009/396] Add metadata check before publishing/unpublishing and update publish/unpublish error message. Also updates the error message generated by attempting to delete a collection with invalid media objects to provide the IDs of the culprit media objects. --- .../admin/collections_controller.rb | 4 +- app/controllers/media_objects_controller.rb | 10 +++-- .../admin_collections_controller_spec.rb | 33 ++++++++++++++ .../media_objects_controller_spec.rb | 44 +++++++++++++++++-- 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/app/controllers/admin/collections_controller.rb b/app/controllers/admin/collections_controller.rb index dc8bf92529..3fc57b5aab 100644 --- a/app/controllers/admin/collections_controller.rb +++ b/app/controllers/admin/collections_controller.rb @@ -210,7 +210,9 @@ def destroy target_path = admin_collection_path(@target_collection) @source_collection.reload else - flash[:error] = "Collection contains invalid media objects that cannot be moved. Please address these issues before attempting to delete #{@source_collection.name}." + mos = @source_collection.media_objects.to_a + invalid_mos = mos.collect { |mo| "
  • #{mo.id}
    • " unless mo.valid? }.compact + flash[:error] = "Collection contains invalid media objects that cannot be moved. Please address these issues before attempting to delete #{@source_collection.name}.
      #{invalid_mos.join('')}
    ".html_safe redirect_to admin_collection_path(@source_collection) and return end end diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb index 4e7de92727..7c74cc3e93 100644 --- a/app/controllers/media_objects_controller.rb +++ b/app/controllers/media_objects_controller.rb @@ -418,8 +418,12 @@ def update_status Array(params[:id]).each do |id| media_object = MediaObject.find(id) if cannot? :update, media_object - errors += ["#{media_object.title} (#{id}) (permission denied)."] + errors += ["#{media_object&.title} (#{id}) (permission denied)."] else + unless (media_object.title.present? && media_object.date_issued.present?) + errors += ["Validation failed: Title field is required., Date issued field is required."] + break + end begin case status when 'publish' @@ -432,7 +436,7 @@ def update_status media_object.publish!(nil) success_count += 1 else - errors += ["#{media_object.title} (#{id}) (permission denied)."] + errors += ["#{media_object&.title} (#{id}) (permission denied)."] end end rescue ActiveFedora::RecordInvalid => e @@ -441,7 +445,7 @@ def update_status end end message = "#{success_count} #{'media object'.pluralize(success_count)} successfully #{status}ed." if success_count.positive? - message = "Unable to publish #{'item'.pluralize(errors.count)}: #{ errors.join('
    ') }" if errors.count > 0 + message = "Unable to #{status} #{'item'.pluralize(errors.count)}: #{ errors.join('
    ') }" if errors.count > 0 redirect_back(fallback_location: root_path, flash: {notice: message.html_safe}) end diff --git a/spec/controllers/admin_collections_controller_spec.rb b/spec/controllers/admin_collections_controller_spec.rb index 1b828da96e..5d08ff673b 100644 --- a/spec/controllers/admin_collections_controller_spec.rb +++ b/spec/controllers/admin_collections_controller_spec.rb @@ -593,6 +593,39 @@ end end + describe "#destroy" do + let!(:collection) { FactoryBot.create(:collection, items: 1) } + let!(:target) { FactoryBot.create(:collection) } + before do + login_user collection.managers.first + end + + it "moves media objects to target collection" do + media_object = collection.media_objects[0] + expect { delete :destroy, params: { id: collection.id, target_collection_id: target.id } }.to change { target.media_objects.count }.by(1) + expect(target.media_objects).to include(media_object) + end + + it "deletes the collection" do + expect { delete :destroy, params: { id: collection.id, target_collection_id: target.id } }.to change { Admin::Collection.count }.by(-1) + expect(Admin::Collection.all).not_to include(collection) + end + + it "redirects to target collection" do + delete :destroy, params: { id: collection.id, target_collection_id: target.id } + expect(response).to redirect_to(admin_collection_path(target)) + end + + context "collection with invalid media object" do + it "does not delete collection" do + allow_any_instance_of(MediaObject).to receive(:valid?).and_return(false) + expect { delete :destroy, params: { id: collection.id, target_collection_id: target.id } }.not_to change { Admin::Collection.count } + expect(response).to redirect_to(admin_collection_path(collection)) + expect(flash[:error]).to eq("Collection contains invalid media objects that cannot be moved. Please address these issues before attempting to delete #{collection.name}.
    • #{collection.media_objects[0].id}
    ") + end + end + end + describe '#attach_poster' do let(:collection) { FactoryBot.create(:collection) } diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb index 19445fae9a..526eb44be4 100644 --- a/spec/controllers/media_objects_controller_spec.rb +++ b/spec/controllers/media_objects_controller_spec.rb @@ -1383,6 +1383,18 @@ media_object.reload expect(media_object).not_to be_published end + + it "item is invalid and no last_completed_step" do + media_object = FactoryBot.create(:media_object, collection: collection) + media_object.title = nil + media_object.date_issued = nil + media_object.workflow.last_completed_step = '' + media_object.save!(validate: false) + get 'update_status', params: { id: media_object.id, status: 'publish' } + expect(flash[:notice]).to eq("Unable to publish item: Validation failed: Title field is required., Date issued field is required.") + media_object.reload + expect(media_object).not_to be_published + end end end @@ -1394,9 +1406,35 @@ expect(media_object).not_to be_published end - it "should fail when id doesn't exist" do - get 'update_status', params: { id: 'this-id-is-fake', status: 'unpublish' } - expect(response.code).to eq '404' + context "should fail when" do + it "id doesn't exist" do + get 'update_status', params: { id: 'this-id-is-fake', status: 'unpublish' } + expect(response.code).to eq '404' + end + + it "item is invalid" do + media_object = FactoryBot.create(:published_media_object, collection: collection) + media_object.title = nil + media_object.date_issued = nil + media_object.workflow.last_completed_step = 'file-upload' + media_object.save!(validate: false) + get 'update_status', params: { id: media_object.id, status: 'unpublish' } + expect(flash[:notice]).to eq("Unable to unpublish item: Validation failed: Title field is required., Date issued field is required.") + media_object.reload + expect(media_object).to be_published + end + + it "item is invalid and no last_completed_step" do + media_object = FactoryBot.create(:published_media_object, collection: collection) + media_object.title = nil + media_object.date_issued = nil + media_object.workflow.last_completed_step = '' + media_object.save!(validate: false) + get 'update_status', params: { id: media_object.id, status: 'unpublish' } + expect(flash[:notice]).to eq("Unable to unpublish item: Validation failed: Title field is required., Date issued field is required.") + media_object.reload + expect(media_object).to be_published + end end it "should unpublish multiple items" do From 8dc1b1318cc3d3763fb54a979989f1884a582667 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Tue, 4 Apr 2023 15:55:36 -0400 Subject: [PATCH 010/396] Fix for codeclimate --- app/controllers/media_objects_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb index 7c74cc3e93..8b848fe3b2 100644 --- a/app/controllers/media_objects_controller.rb +++ b/app/controllers/media_objects_controller.rb @@ -420,7 +420,7 @@ def update_status if cannot? :update, media_object errors += ["#{media_object&.title} (#{id}) (permission denied)."] else - unless (media_object.title.present? && media_object.date_issued.present?) + unless media_object.title.present? && media_object.date_issued.present? errors += ["Validation failed: Title field is required., Date issued field is required."] break end From 68ada1a4438a29dda438a63b4a3d11bb1c66eb45 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 11 Apr 2023 10:42:58 -0400 Subject: [PATCH 011/396] Refactor #publish! to allow skipping of validations Co-authored-by: Mason Ballengee --- app/models/media_object.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/media_object.rb b/app/models/media_object.rb index 8edc45bd48..30f8aaba31 100644 --- a/app/models/media_object.rb +++ b/app/models/media_object.rb @@ -156,9 +156,13 @@ def collection= co # Sets the publication status. To unpublish an object set it to nil or # omit the status which will default to unpublished. This makes the act # of publishing _explicit_ instead of an accidental side effect. - def publish!(user_key) + def publish!(user_key, validate: false) self.avalon_publisher = user_key.blank? ? nil : user_key - save! + if validate + save(validate: false) || raise "Save failed" + else + save! + end end def finished_processing? From 6cd178201c0899ad20d0610c49c68309e6c1c7eb Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Mon, 10 Apr 2023 09:50:53 -0400 Subject: [PATCH 012/396] Edit error message for publish/unpublish --- app/controllers/media_objects_controller.rb | 4 ++-- spec/controllers/media_objects_controller_spec.rb | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb index 8b848fe3b2..5f923cf807 100644 --- a/app/controllers/media_objects_controller.rb +++ b/app/controllers/media_objects_controller.rb @@ -421,8 +421,8 @@ def update_status errors += ["#{media_object&.title} (#{id}) (permission denied)."] else unless media_object.title.present? && media_object.date_issued.present? - errors += ["Validation failed: Title field is required., Date issued field is required."] - break + errors += ["#{media_object&.title} (#{id}) (missing required fields)"] + next end begin case status diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb index 526eb44be4..4fdf9a0e6f 100644 --- a/spec/controllers/media_objects_controller_spec.rb +++ b/spec/controllers/media_objects_controller_spec.rb @@ -1379,7 +1379,7 @@ media_object.workflow.last_completed_step = 'file-upload' media_object.save!(validate: false) get 'update_status', params: { id: media_object.id, status: 'publish' } - expect(flash[:notice]).to eq("Unable to publish item: Validation failed: Title field is required., Date issued field is required.") + expect(flash[:notice]).to eq("Unable to publish item: #{media_object&.title} (#{media_object.id}) (missing required fields)") media_object.reload expect(media_object).not_to be_published end @@ -1391,7 +1391,7 @@ media_object.workflow.last_completed_step = '' media_object.save!(validate: false) get 'update_status', params: { id: media_object.id, status: 'publish' } - expect(flash[:notice]).to eq("Unable to publish item: Validation failed: Title field is required., Date issued field is required.") + expect(flash[:notice]).to eq("Unable to publish item: #{media_object&.title} (#{media_object.id}) (missing required fields)") media_object.reload expect(media_object).not_to be_published end @@ -1419,7 +1419,7 @@ media_object.workflow.last_completed_step = 'file-upload' media_object.save!(validate: false) get 'update_status', params: { id: media_object.id, status: 'unpublish' } - expect(flash[:notice]).to eq("Unable to unpublish item: Validation failed: Title field is required., Date issued field is required.") + expect(flash[:notice]).to eq("Unable to unpublish item: #{media_object&.title} (#{media_object.id}) (missing required fields)") media_object.reload expect(media_object).to be_published end @@ -1431,7 +1431,7 @@ media_object.workflow.last_completed_step = '' media_object.save!(validate: false) get 'update_status', params: { id: media_object.id, status: 'unpublish' } - expect(flash[:notice]).to eq("Unable to unpublish item: Validation failed: Title field is required., Date issued field is required.") + expect(flash[:notice]).to eq("Unable to unpublish item: #{media_object&.title} (#{media_object.id}) (missing required fields)") media_object.reload expect(media_object).to be_published end From 36fb447924faaf2e93329bee02e206478ce6a4d8 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Tue, 11 Apr 2023 15:40:52 -0400 Subject: [PATCH 013/396] Refactor MediaObject controller for new #publish! validation logic Co-authored-by: Chris Colvard Also updates bulk_action_job UpdateStatus and adjusts tests for new behavior. --- app/controllers/media_objects_controller.rb | 10 ++--- app/jobs/bulk_action_jobs.rb | 8 ++-- app/models/media_object.rb | 10 ++--- .../media_objects_controller_spec.rb | 45 +++++++++---------- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb index 5f923cf807..2a21e68cca 100644 --- a/app/controllers/media_objects_controller.rb +++ b/app/controllers/media_objects_controller.rb @@ -420,14 +420,14 @@ def update_status if cannot? :update, media_object errors += ["#{media_object&.title} (#{id}) (permission denied)."] else - unless media_object.title.present? && media_object.date_issued.present? - errors += ["#{media_object&.title} (#{id}) (missing required fields)"] - next - end begin case status when 'publish' - media_object.publish!(user_key) + unless media_object.title.present? && media_object.date_issued.present? + errors += ["#{media_object&.title} (#{id}) (missing required fields)"] + next + end + media_object.publish!(user_key, validate: true) # additional save to set permalink media_object.save( validate: false ) success_count += 1 diff --git a/app/jobs/bulk_action_jobs.rb b/app/jobs/bulk_action_jobs.rb index 9067f20698..4c6ac18605 100644 --- a/app/jobs/bulk_action_jobs.rb +++ b/app/jobs/bulk_action_jobs.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -109,7 +109,7 @@ def perform(documents, user_key, params) media_object = MediaObject.find(id) case status when 'publish' - media_object.publish!(user_key) + media_object.publish!(user_key, validate: true) # additional save to set permalink if media_object.save successes += [media_object] diff --git a/app/models/media_object.rb b/app/models/media_object.rb index 30f8aaba31..e1b7277aa5 100644 --- a/app/models/media_object.rb +++ b/app/models/media_object.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -159,9 +159,9 @@ def collection= co def publish!(user_key, validate: false) self.avalon_publisher = user_key.blank? ? nil : user_key if validate - save(validate: false) || raise "Save failed" - else save! + else + raise "Save failed" unless save(validate: false) end end diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb index 4fdf9a0e6f..aad72c013f 100644 --- a/spec/controllers/media_objects_controller_spec.rb +++ b/spec/controllers/media_objects_controller_spec.rb @@ -1406,35 +1406,32 @@ expect(media_object).not_to be_published end + it 'unpublishes invalid items' do + media_object = FactoryBot.create(:published_media_object, collection: collection) + media_object.title = nil + media_object.date_issued = nil + media_object.save!(validate: false) + get 'update_status', params: { id: media_object.id, status: 'unpublish' } + media_object.reload + expect(media_object).not_to be_published + end + + it 'unpublishes invalid items and no last completed step' do + media_object = FactoryBot.create(:published_media_object, collection: collection) + media_object.title = nil + media_object.date_issued = nil + media_object.workflow.last_completed_step = '' + media_object.save!(validate: false) + get 'update_status', params: { id: media_object.id, status: 'unpublish' } + media_object.reload + expect(media_object).not_to be_published + end + context "should fail when" do it "id doesn't exist" do get 'update_status', params: { id: 'this-id-is-fake', status: 'unpublish' } expect(response.code).to eq '404' end - - it "item is invalid" do - media_object = FactoryBot.create(:published_media_object, collection: collection) - media_object.title = nil - media_object.date_issued = nil - media_object.workflow.last_completed_step = 'file-upload' - media_object.save!(validate: false) - get 'update_status', params: { id: media_object.id, status: 'unpublish' } - expect(flash[:notice]).to eq("Unable to unpublish item: #{media_object&.title} (#{media_object.id}) (missing required fields)") - media_object.reload - expect(media_object).to be_published - end - - it "item is invalid and no last_completed_step" do - media_object = FactoryBot.create(:published_media_object, collection: collection) - media_object.title = nil - media_object.date_issued = nil - media_object.workflow.last_completed_step = '' - media_object.save!(validate: false) - get 'update_status', params: { id: media_object.id, status: 'unpublish' } - expect(flash[:notice]).to eq("Unable to unpublish item: #{media_object&.title} (#{media_object.id}) (missing required fields)") - media_object.reload - expect(media_object).to be_published - end end it "should unpublish multiple items" do From 1da718bf5383b207927bb3620989909f4deb087a Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Tue, 11 Apr 2023 15:51:40 -0400 Subject: [PATCH 014/396] Revert changes to admin collections controller and tests --- .../admin/collections_controller.rb | 4 +-- .../admin_collections_controller_spec.rb | 33 ------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/app/controllers/admin/collections_controller.rb b/app/controllers/admin/collections_controller.rb index 3fc57b5aab..dc8bf92529 100644 --- a/app/controllers/admin/collections_controller.rb +++ b/app/controllers/admin/collections_controller.rb @@ -210,9 +210,7 @@ def destroy target_path = admin_collection_path(@target_collection) @source_collection.reload else - mos = @source_collection.media_objects.to_a - invalid_mos = mos.collect { |mo| "
  • #{mo.id}
    • " unless mo.valid? }.compact - flash[:error] = "Collection contains invalid media objects that cannot be moved. Please address these issues before attempting to delete #{@source_collection.name}.
      #{invalid_mos.join('')}
    ".html_safe + flash[:error] = "Collection contains invalid media objects that cannot be moved. Please address these issues before attempting to delete #{@source_collection.name}." redirect_to admin_collection_path(@source_collection) and return end end diff --git a/spec/controllers/admin_collections_controller_spec.rb b/spec/controllers/admin_collections_controller_spec.rb index 5d08ff673b..1b828da96e 100644 --- a/spec/controllers/admin_collections_controller_spec.rb +++ b/spec/controllers/admin_collections_controller_spec.rb @@ -593,39 +593,6 @@ end end - describe "#destroy" do - let!(:collection) { FactoryBot.create(:collection, items: 1) } - let!(:target) { FactoryBot.create(:collection) } - before do - login_user collection.managers.first - end - - it "moves media objects to target collection" do - media_object = collection.media_objects[0] - expect { delete :destroy, params: { id: collection.id, target_collection_id: target.id } }.to change { target.media_objects.count }.by(1) - expect(target.media_objects).to include(media_object) - end - - it "deletes the collection" do - expect { delete :destroy, params: { id: collection.id, target_collection_id: target.id } }.to change { Admin::Collection.count }.by(-1) - expect(Admin::Collection.all).not_to include(collection) - end - - it "redirects to target collection" do - delete :destroy, params: { id: collection.id, target_collection_id: target.id } - expect(response).to redirect_to(admin_collection_path(target)) - end - - context "collection with invalid media object" do - it "does not delete collection" do - allow_any_instance_of(MediaObject).to receive(:valid?).and_return(false) - expect { delete :destroy, params: { id: collection.id, target_collection_id: target.id } }.not_to change { Admin::Collection.count } - expect(response).to redirect_to(admin_collection_path(collection)) - expect(flash[:error]).to eq("Collection contains invalid media objects that cannot be moved. Please address these issues before attempting to delete #{collection.name}.
    • #{collection.media_objects[0].id}
    ") - end - end - end - describe '#attach_poster' do let(:collection) { FactoryBot.create(:collection) } From a2e541981e33bdcbb2624b18c986e2672a4b1682 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Fri, 14 Apr 2023 11:00:25 -0400 Subject: [PATCH 015/396] Change publish! default to validate: true --- app/controllers/media_objects_controller.rb | 4 ++-- app/jobs/bulk_action_jobs.rb | 4 ++-- app/models/media_object.rb | 2 +- spec/models/media_object_spec.rb | 20 +++++++++++++++++--- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb index 2a21e68cca..7249094022 100644 --- a/app/controllers/media_objects_controller.rb +++ b/app/controllers/media_objects_controller.rb @@ -427,13 +427,13 @@ def update_status errors += ["#{media_object&.title} (#{id}) (missing required fields)"] next end - media_object.publish!(user_key, validate: true) + media_object.publish!(user_key) # additional save to set permalink media_object.save( validate: false ) success_count += 1 when 'unpublish' if can? :unpublish, media_object - media_object.publish!(nil) + media_object.publish!(nil, validate: false) success_count += 1 else errors += ["#{media_object&.title} (#{id}) (permission denied)."] diff --git a/app/jobs/bulk_action_jobs.rb b/app/jobs/bulk_action_jobs.rb index 4c6ac18605..8b8c36160f 100644 --- a/app/jobs/bulk_action_jobs.rb +++ b/app/jobs/bulk_action_jobs.rb @@ -109,7 +109,7 @@ def perform(documents, user_key, params) media_object = MediaObject.find(id) case status when 'publish' - media_object.publish!(user_key, validate: true) + media_object.publish!(user_key) # additional save to set permalink if media_object.save successes += [media_object] @@ -117,7 +117,7 @@ def perform(documents, user_key, params) errors += [media_object] end when 'unpublish' - if media_object.publish!(nil) + if media_object.publish!(nil, validate: false) successes += [media_object] else errors += [media_object] diff --git a/app/models/media_object.rb b/app/models/media_object.rb index e1b7277aa5..527abd67de 100644 --- a/app/models/media_object.rb +++ b/app/models/media_object.rb @@ -156,7 +156,7 @@ def collection= co # Sets the publication status. To unpublish an object set it to nil or # omit the status which will default to unpublished. This makes the act # of publishing _explicit_ instead of an accidental side effect. - def publish!(user_key, validate: false) + def publish!(user_key, validate: true) self.avalon_publisher = user_key.blank? ? nil : user_key if validate save! diff --git a/spec/models/media_object_spec.rb b/spec/models/media_object_spec.rb index 238b1b32b5..dbb8f5b0dc 100644 --- a/spec/models/media_object_spec.rb +++ b/spec/models/media_object_spec.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -557,6 +557,20 @@ media_object.publish!(nil) expect(media_object.to_solr["workflow_published_sim"]).to eq('Unpublished') end + context 'validate: false' do + it 'publishes' do + media_object.publish!('adam@adam.com', validate: false) + expect(media_object.to_solr["workflow_published_sim"]).to eq('Published') + end + it 'unpublishes' do + media_object.publish!(nil, validate: false) + expect(media_object.to_solr["workflow_published_sim"]).to eq('Unpublished') + end + it 'raises runtime error if save fails' do + allow_any_instance_of(MediaObject).to receive(:save).and_return(false) + expect { media_object.publish!(nil, validate: false) }.to raise_error(RuntimeError) + end + end end end From 4b2ebeeb95a56948c735e7c4e1d7fa09dafeb682 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Tue, 4 Apr 2023 14:52:00 -0400 Subject: [PATCH 016/396] Allow configuration of recaptcha v3 or v2_checkbox --- app/controllers/comments_controller.rb | 13 ++++++++++++- app/views/comments/index.html.erb | 14 ++++++++++---- config/settings.yml | 7 +++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/app/controllers/comments_controller.rb b/app/controllers/comments_controller.rb index 3967d7ce41..e3e29753c7 100644 --- a/app/controllers/comments_controller.rb +++ b/app/controllers/comments_controller.rb @@ -29,7 +29,7 @@ def create @comment.subject = params[:comment][:subject] @comment.comment = params[:comment][:comment] - if @comment.valid? && (Settings.recaptcha.blank? || verify_recaptcha(model: @comment)) + if @comment.valid? && recaptcha_valid? begin CommentsMailer.contact_email(@comment.to_h).deliver_later rescue Errno::ECONNRESET => e @@ -47,4 +47,15 @@ def create def set_subjects @subjects = Comment::SUBJECTS end + + def recaptcha_valid? + return true unless Settings.recaptcha.site_key.present? + options = case Settings.recaptcha.type + when "v2_checkbox" + { model: @comment } + when "v3" + { action: Settings.recaptcha.v3.action, minimum_score: Settings.recaptcha.v3.minimum_score } + end + verify_recaptcha(options) + end end diff --git a/app/views/comments/index.html.erb b/app/views/comments/index.html.erb index f0ed402d87..90991478d1 100644 --- a/app/views/comments/index.html.erb +++ b/app/views/comments/index.html.erb @@ -29,10 +29,16 @@ Unless required by applicable law or agreed to in writing, software distributed <%= comment.email_field :email_confirmation, label: 'Confirm email address' %> <%= comment.select :subject, options_for_select(@subjects), { :prompt => 'Select one' }, { :control_class => 'form-control' } %> <%= comment.text_area :comment, rows: 10 %> - <% if Settings.recaptcha.present?%> -
    - <%= recaptcha_tags %> -
    + <% if Settings.recaptcha.site_key.present?%> + <% if Settings.recaptcha.type == "v2_checkbox" %> +
    + <%= recaptcha_tags %> +
    + <% elsif Settings.recaptcha.type == "v3" %> +
    + <%= recaptcha_v3(action: Settings.recaptcha.v3.action) %> +
    + <% end %> <% end %> <%= comment.primary "Submit comments" %> diff --git a/config/settings.yml b/config/settings.yml index 7bcda7e282..dfee409912 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -104,3 +104,10 @@ caption_default: # Language should be 2 or 3 letter ISO 639 codes language: 'en' name: 'English' +recaptcha: + site_key: # Setting a site_key will enable recaptcha on the comments form + secret_key: # Required along with site_key + type: "v2_checkbox" # or "v3" + v3: + action: "comment" + minimum_score: 0.5 From 431d495b5a9d10384a9ca80f3f653f27b7668d18 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Wed, 19 Apr 2023 16:52:31 -0400 Subject: [PATCH 017/396] Test comments controller including recaptcha validation --- spec/controllers/comments_controller_spec.rb | 75 ++++++++++++++++++++ spec/factories/comments.rb | 22 ++++++ 2 files changed, 97 insertions(+) create mode 100644 spec/controllers/comments_controller_spec.rb create mode 100644 spec/factories/comments.rb diff --git a/spec/controllers/comments_controller_spec.rb b/spec/controllers/comments_controller_spec.rb new file mode 100644 index 0000000000..8430b09eda --- /dev/null +++ b/spec/controllers/comments_controller_spec.rb @@ -0,0 +1,75 @@ +# Copyright 2011-2023, The Trustees of Indiana University and Northwestern +# University. Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# --- END LICENSE_HEADER BLOCK --- + +require 'rails_helper' + +describe CommentsController do + render_views + + let(:comment) { FactoryBot.build(:comment) } + let(:comment_parameters) { comment.to_h.merge({ email_confirmation: comment.email }) } + + describe '#create' do + it 'sends a comment email' do + post :create, params: { comment: comment_parameters } + expect(response.status).to be 200 + end + + context 'missing parameters' do + it 'shows error message' do + post :create, params: { comment: { name: "Test" } } + expect(response).to render_template(:index) + end + end + + context 'recaptcha enabled' do + before do + allow(Settings.recaptcha).to receive(:site_key).and_return("site_key") + allow(Settings.recaptcha).to receive(:secret_key).and_return("secret_key") + allow(Settings.recaptcha).to receive(:type).and_return(recaptcha_type) + allow(Recaptcha.configuration).to receive(:site_key!).and_return("site_key") + allow(Recaptcha.configuration).to receive(:secret_key!).and_return("secret_key") + end + + context 'recaptcha v2' do + let(:recaptcha_type) { "v2_checkbox" } + + it 'sends a comment email' do + allow(controller).to receive(:verify_recaptcha) + post :create, params: { comment: comment_parameters } + expect(response.status).to be 200 + expect(controller).to have_received(:verify_recaptcha).with({ model: assigns(:comment) }) + end + end + + context 'recaptcha v3' do + let(:recaptcha_type) { "v3" } + let(:action) { "comment" } + let(:minimum_score) { 0.7 } + + before do + allow(Settings.recaptcha.v3).to receive(:action).and_return(action) + allow(Settings.recaptcha.v3).to receive(:minimum_score).and_return(minimum_score) + end + + it 'sends a comment email' do + allow(controller).to receive(:verify_recaptcha) + post :create, params: { comment: comment_parameters } + expect(response.status).to be 200 + expect(controller).to have_received(:verify_recaptcha).with({ action: action, minimum_score: minimum_score }) + end + end + end + end +end diff --git a/spec/factories/comments.rb b/spec/factories/comments.rb new file mode 100644 index 0000000000..eec25d4046 --- /dev/null +++ b/spec/factories/comments.rb @@ -0,0 +1,22 @@ +# Copyright 2011-2023, The Trustees of Indiana University and Northwestern +# University. Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# --- END LICENSE_HEADER BLOCK --- + +FactoryBot.define do + factory :comment do + name { Faker::Name.name } + email { Faker::Internet.email } + subject { "General feedback" } + comment { Faker::Lorem.sentence } + end +end From 06077d384ec6e880217c5a53af381089789b21c9 Mon Sep 17 00:00:00 2001 From: dananji Date: Tue, 18 Apr 2023 16:54:03 -0400 Subject: [PATCH 018/396] Set default playlist option to first playlist --- app/views/media_objects/_mejs4_add_to_playlist.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/media_objects/_mejs4_add_to_playlist.html.erb b/app/views/media_objects/_mejs4_add_to_playlist.html.erb index f9d975056f..45da7449bd 100644 --- a/app/views/media_objects/_mejs4_add_to_playlist.html.erb +++ b/app/views/media_objects/_mejs4_add_to_playlist.html.erb @@ -44,7 +44,7 @@ This view file goes with the custom 'Add To Playlist' MediaElement 4 plugin.
    <%= label_tag(:playlist_id, "Playlist") %> - <%= collection_select(:post, :playlist_id, @add_playlist_item_playlists, :id, :title, {}, {class: "form-control form-model", style: 'width:100%;'}) %> + <%= collection_select(:post, :playlist_id, @add_playlist_item_playlists, :id, :title, { :selected => @add_playlist_item_playlists.first.id }, {class: "form-control form-model", style: 'width:100%;'}) %>
    From daa2f3953008506f11a77e3bdd6a56d2bb9b5225 Mon Sep 17 00:00:00 2001 From: dananji Date: Wed, 19 Apr 2023 12:18:37 -0400 Subject: [PATCH 019/396] Remove 'media_element_add_to_playlist' gem, add it's JS and SVG into Avalon --- Gemfile | 1 - Gemfile.lock | 9 -- app/assets/images/add_to_playlist_icon.svg | 18 +++ .../javascripts/add-playlist-option.js.coffee | 105 ++++++++++++++++++ .../_mejs4_add_to_playlist.html.erb | 1 - 5 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 app/assets/images/add_to_playlist_icon.svg create mode 100644 app/assets/javascripts/add-playlist-option.js.coffee diff --git a/Gemfile b/Gemfile index a6733c68bb..d5fbfc9c9b 100644 --- a/Gemfile +++ b/Gemfile @@ -78,7 +78,6 @@ gem 'active_encode', '~> 1.0', '>= 1.1.3' gem 'audio_waveform-ruby', '~> 1.0.7', require: 'audio_waveform' gem 'browse-everything', git: "https://github.com/avalonmediasystem/browse-everything.git", branch: 'v1.2-avalon' gem 'fastimage' -gem 'media_element_add_to_playlist', git: 'https://github.com/avalonmediasystem/media-element-add-to-playlist.git', tag: 'avalon-r6.5' gem 'mediainfo', git: "https://github.com/avalonmediasystem/mediainfo.git", tag: 'v0.7.1-avalon' gem 'rest-client', '~> 2.0' gem 'roo' diff --git a/Gemfile.lock b/Gemfile.lock index 271d12d1db..c5d22d6cdf 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -38,14 +38,6 @@ GIT signet (~> 0.8) typhoeus -GIT - remote: https://github.com/avalonmediasystem/media-element-add-to-playlist.git - revision: dfde0084f6c17ca16fa675086eb025440f99a02d - tag: avalon-r6.5 - specs: - media_element_add_to_playlist (0.0.1) - rails (> 4.0) - GIT remote: https://github.com/avalonmediasystem/mediainfo.git revision: bea9479d33328c6b483ee19c008730f939d98266 @@ -1060,7 +1052,6 @@ DEPENDENCIES lograge mail (> 2.8.0.1) marc - media_element_add_to_playlist! mediainfo! mysql2 net-ldap diff --git a/app/assets/images/add_to_playlist_icon.svg b/app/assets/images/add_to_playlist_icon.svg new file mode 100644 index 0000000000..3fb01d60de --- /dev/null +++ b/app/assets/images/add_to_playlist_icon.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + diff --git a/app/assets/javascripts/add-playlist-option.js.coffee b/app/assets/javascripts/add-playlist-option.js.coffee new file mode 100644 index 0000000000..61d2a4deb8 --- /dev/null +++ b/app/assets/javascripts/add-playlist-option.js.coffee @@ -0,0 +1,105 @@ +addnew = 'Add new playlist' +select_element = $('#post_playlist_id') +add_success = false + +select_element.prepend(new Option(addnew)) + +getSearchTerm = () -> + return $('span.select2-search--dropdown input').val(); + +matchWithNew = (params, data) -> + term = params.term || '' + term = term.toLowerCase() + text = data.text.toLowerCase() + if (text.includes(addnew.toLowerCase()) || text.includes(term)) + return data + return null + +sortWithNew = (data) -> + return data.sort((a, b) -> + if (b.text.trim() == addnew) + return 1 + if (a.text < b.text || a.text.trim() == addnew) + return -1 + if (a.text > b.text) + return 1 + return 0) + +formatAddNew = (data) -> + term = getSearchTerm() || '' + if (data.text == addnew ) + if (term != '') + term = addnew + ' "' + term + '"' + else + term = addnew + return ' + '+term+' + ' + else + start = data.text.toLowerCase().indexOf(term.toLowerCase()) + if (start != -1) + end = start+term.length-1 + return data.text.substring(0,start) + '' + data.text.substring(start, end+1) + '' + data.text.substring(end+1) + return data.text + +showNewPlaylistModal = (playlistName) -> + #set to defaults first + $('#new_playlist_submit').val('Create') + $('#new_playlist_submit').prop("disabled", false) + $('#playlist_title').val(playlistName) + $('#playlist_comment').val("") + $('#playlist_visibility_private').prop('checked', true) + #remove any possible old errors + $('#title_error').remove() + $('#playlist_title').parent().removeClass('has-error') + add_success = false + #finally show + $('#add-playlist-modal').modal('show') + return true + +$('#add-playlist-modal').on('hidden.bs.modal', () -> + if (!add_success) + op = select_element.children()[1] + if (op) + newval = op.value + else + newval = -1 + select_element.val(newval).trigger('change.select2') +) + +select_element.select2({ + templateResult: formatAddNew + escapeMarkup: + (markup) -> return markup + matcher: matchWithNew + sorter: sortWithNew + }) + .on('select2:selecting', + (evt) -> + choice = evt.params.args.data.text + if (choice.indexOf(addnew) != -1) + showNewPlaylistModal(getSearchTerm()) + ) + +$('#playlist_form').submit( + () -> + if($('#playlist_title').val()) + $('#new_playlist_submit').val('Saving...') + $('#new_playlist_submit').prop("disabled", true) + return true + if($('#title_error').length == 0) + $('#playlist_title').after('
    Name is required
    ') + $('#playlist_title').parent().addClass('has-error') + return false +) + +$("#playlist_form").bind('ajax:success', + (event) -> + [data, status, xhr] = event.detail + $('#add-playlist-modal').modal('hide') + if (data.errors) + console.log(data.errors.title[0]) + else + add_success = true + select_element.append(new Option(data.title, data.id.toString(), true, true)).trigger('change') + ) diff --git a/app/views/media_objects/_mejs4_add_to_playlist.html.erb b/app/views/media_objects/_mejs4_add_to_playlist.html.erb index 45da7449bd..b54f983753 100644 --- a/app/views/media_objects/_mejs4_add_to_playlist.html.erb +++ b/app/views/media_objects/_mejs4_add_to_playlist.html.erb @@ -20,7 +20,6 @@ This view file goes with the custom 'Add To Playlist' MediaElement 4 plugin. <% content_for :page_scripts do %> <%= javascript_include_tag 'select2.min' %> <%= stylesheet_link_tag 'select2.min' %> - <%= javascript_include_tag 'add-playlist-option' %> <% end %> <% # Get users playlists %> From 518224c379751cdfe0859028ce0db9a11bf8a1df Mon Sep 17 00:00:00 2001 From: dananji Date: Fri, 21 Apr 2023 09:45:33 -0400 Subject: [PATCH 020/396] Fix code climate issues --- .../javascripts/add-playlist-option.js.coffee | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/app/assets/javascripts/add-playlist-option.js.coffee b/app/assets/javascripts/add-playlist-option.js.coffee index 61d2a4deb8..7c2cfc3fb2 100644 --- a/app/assets/javascripts/add-playlist-option.js.coffee +++ b/app/assets/javascripts/add-playlist-option.js.coffee @@ -5,7 +5,7 @@ add_success = false select_element.prepend(new Option(addnew)) getSearchTerm = () -> - return $('span.select2-search--dropdown input').val(); + return $('span.select2-search--dropdown input').val() matchWithNew = (params, data) -> term = params.term || '' @@ -58,13 +58,13 @@ showNewPlaylistModal = (playlistName) -> return true $('#add-playlist-modal').on('hidden.bs.modal', () -> - if (!add_success) - op = select_element.children()[1] - if (op) - newval = op.value - else - newval = -1 - select_element.val(newval).trigger('change.select2') + if (!add_success) + op = select_element.children()[1] + if (op) + newval = op.value + else + newval = -1 + select_element.val(newval).trigger('change.select2') ) select_element.select2({ @@ -88,7 +88,8 @@ $('#playlist_form').submit( $('#new_playlist_submit').prop("disabled", true) return true if($('#title_error').length == 0) - $('#playlist_title').after('
    Name is required
    ') + $('#playlist_title') + .after('
    Name is required
    ') $('#playlist_title').parent().addClass('has-error') return false ) @@ -101,5 +102,7 @@ $("#playlist_form").bind('ajax:success', console.log(data.errors.title[0]) else add_success = true - select_element.append(new Option(data.title, data.id.toString(), true, true)).trigger('change') + select_element + .append(new Option(data.title, data.id.toString(), true, true)) + .trigger('change') ) From 214eaa9f2ec55e2a91161591a9bf0c89365b3c95 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Fri, 28 Apr 2023 15:59:31 -0400 Subject: [PATCH 021/396] Mark transcripts as machine generated --- .../supplemental_files_controller.rb | 38 ++++++++++++-- app/models/supplemental_file.rb | 12 +++-- .../_supplemental_files_list.html.erb | 8 +++ spec/models/supplemental_file_spec.rb | 8 +-- .../supplemental_files_controller_examples.rb | 49 ++++++++++++++++--- 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/app/controllers/supplemental_files_controller.rb b/app/controllers/supplemental_files_controller.rb index a8ccf15e89..daf0601910 100644 --- a/app/controllers/supplemental_files_controller.rb +++ b/app/controllers/supplemental_files_controller.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -73,14 +73,14 @@ def show end end - # Update the label of the supplemental file + # Update the label and tags of the supplemental file def update raise Avalon::NotFound, "Cannot update the supplemental file: #{params[:id]} not found" unless SupplementalFile.exists? params[:id].to_s @supplemental_file = SupplementalFile.find(params[:id]) raise Avalon::NotFound, "Cannot update the supplemental file: #{@supplemental_file.id} not found" unless @object.supplemental_files.any? { |f| f.id == @supplemental_file.id } raise Avalon::BadRequest, "Updating file contents not allowed" if supplemental_file_params[:file].present? - @supplemental_file.label = supplemental_file_params[:label] + edit_file_information raise Avalon::SaveError, @supplemental_file.errors.full_messages unless @supplemental_file.save flash[:success] = "Supplemental file successfully updated." @@ -136,6 +136,34 @@ def edit_structure_path edit_media_object_path(media_object_id, step: 'file-upload') end + def edit_file_information + ident = "machine_generated_#{params[:id].to_s}".to_sym + + if params[ident] && !@supplemental_file.machine_generated? + @supplemental_file.tags += ['machine_generated'] + edit_file_label supplemental_file_params[:label] + elsif !params[ident] && @supplemental_file.machine_generated? + @supplemental_file.tags -= ['machine_generated'] + label = supplemental_file_params[:label].gsub(" (machine generated)", '') + @supplemental_file.label = label + elsif params[ident] && @supplemental_file.machine_generated? + edit_file_label supplemental_file_params[:label] + else + @supplemental_file.label = supplemental_file_params[:label] + end + end + + def edit_file_label label + label = if label.exclude?('(machine generated)') && File.extname(label).present? + label.gsub(File.extname(label), " (machine generated)#{File.extname(label)}") + elsif label.exclude?('(machine generated)') && File.extname(label).blank? + label + (" (machine generated)") + else + label + end + @supplemental_file.label = label + end + def object_supplemental_file_path if @object.is_a? MasterFile master_file_supplemental_file_path(id: @supplemental_file.id, master_file_id: @object.id) diff --git a/app/models/supplemental_file.rb b/app/models/supplemental_file.rb index 178d3afe77..1ae6421477 100644 --- a/app/models/supplemental_file.rb +++ b/app/models/supplemental_file.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -16,7 +16,7 @@ class SupplementalFile < ApplicationRecord has_one_attached :file # TODO: the empty tag should represent a generic supplemental file - validates :tags, array_inclusion: ['transcript', 'caption', '', nil] + validates :tags, array_inclusion: ['transcript', 'caption', 'machine_generated', '', nil] serialize :tags, Array @@ -24,4 +24,8 @@ def attach_file(new_file) file.attach(new_file) self.label = file.filename.to_s if label.blank? end + + def machine_generated? + self.tags.include?('machine_generated') + end end diff --git a/app/views/media_objects/_supplemental_files_list.html.erb b/app/views/media_objects/_supplemental_files_list.html.erb index 33b9e36981..ed9fb91335 100644 --- a/app/views/media_objects/_supplemental_files_list.html.erb +++ b/app/views/media_objects/_supplemental_files_list.html.erb @@ -36,6 +36,14 @@ Unless required by applicable law or agreed to in writing, software distributed Cancel <% end %> + <% if tag == 'transcript' %> + + <%= label_tag "machine_generated_#{file.id}" do %> + <%= check_box_tag "machine_generated_#{file.id}", '1', file.machine_generated? %> + Machine Generated + <% end %> + + <% end %> <% end %> diff --git a/spec/models/supplemental_file_spec.rb b/spec/models/supplemental_file_spec.rb index e2683f2e62..8403699c75 100644 --- a/spec/models/supplemental_file_spec.rb +++ b/spec/models/supplemental_file_spec.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -20,7 +20,7 @@ end context "with valid tags" do - let(:tags) { ["transcript", "caption"] } + let(:tags) { ["transcript", "caption", "machine_generated"] } it "can store tags" do subject.tags = tags diff --git a/spec/support/supplemental_files_controller_examples.rb b/spec/support/supplemental_files_controller_examples.rb index 9118620b6b..4df886d4bf 100644 --- a/spec/support/supplemental_files_controller_examples.rb +++ b/spec/support/supplemental_files_controller_examples.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -37,11 +37,11 @@ {} } - let(:valid_update_attributes) { + let(:valid_update_attributes) { { label: 'new label' } } - let(:invalid_update_attributes) { + let(:invalid_update_attributes) { { file: uploaded_file } } @@ -96,7 +96,7 @@ expect(response).to redirect_to Rails.application.routes.url_helpers.rails_blob_path(supplemental_file.file, disposition: "attachment") end end - + describe "POST #create" do let(:media_object) { FactoryBot.create(:media_object) } let(:master_file) { FactoryBot.create(:master_file, media_object_id: media_object.id) } @@ -245,6 +245,39 @@ end end + context "machine generated transcript" do + let(:machine_param) { "machine_generated_#{supplemental_file.id}".to_sym } + context "missing machine_generated tag" do + let(:supplemental_file) { FactoryBot.create(:supplemental_file, :with_transcript_file, :with_transcript_tag, label: 'label') } + it "adds machine generated note to label and tags" do + expect { + put :update, params: { class_id => object.id, id: supplemental_file.id, machine_param => 1, supplemental_file: valid_update_attributes, format: :html }, session: valid_session + }.to change { master_file.reload.supplemental_files.first.label }.from('label').to('new label (machine generated)') + .and change { master_file.reload.supplemental_files.first.tags }.from(['transcript']).to(['transcript', 'machine_generated']) + end + end + context "with machine_generated tag" do + let(:supplemental_file) { FactoryBot.create(:supplemental_file, :with_transcript_file, tags: ['transcript', 'machine_generated'], label: 'label (machine generated)') } + let(:valid_update_attributes) { { label: 'label (machine generated)' } } + it "does not add more instances of machine generated note" do + expect { + put :update, params: { class_id => object.id, id: supplemental_file.id, machine_param => 1, supplemental_file: valid_update_attributes, format: :html }, session: valid_session + }.to not_change { master_file.reload.supplemental_files.first.label }.from('label (machine generated)') + .and not_change { master_file.reload.supplemental_files.first.tags }.from(['transcript', 'machine_generated']) + end + end + context "removing machine_generated designation" do + let(:supplemental_file) { FactoryBot.create(:supplemental_file, :with_transcript_file, tags: ['transcript', 'machine_generated'], label: 'label (machine generated)') } + let(:valid_update_attributes) { { label: 'label (machine generated)' } } + it "removes machine generated note from label and tags" do + expect { + put :update, params: { class_id => object.id, id: supplemental_file.id, supplemental_file: valid_update_attributes, format: :html }, session: valid_session + }.to change { master_file.reload.supplemental_files.first.label }.from('label (machine generated)').to('label') + .and change { master_file.reload.supplemental_files.first.tags }.from(['transcript', 'machine_generated']).to(['transcript']) + end + end + end + context "with invalid params" do it "returns a 400" do put :update, params: { class_id => object.id, id: supplemental_file.id, supplemental_file: invalid_update_attributes, format: :html }, session: valid_session @@ -287,7 +320,7 @@ end end - context 'html request' do + context 'html request' do context "with valid params" do it "deletes the SupplementalFile" do expect { @@ -298,7 +331,7 @@ # TODO: expect file contents to have been removed end end - + context "with missing id" do it "returns a 404" do delete :destroy, params: { class_id => object.id, id: 'missing-id', format: :html }, session: valid_session From 2b9510d6bd675c94dd6bb819ae1aa7620ac320b3 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Tue, 2 May 2023 09:07:24 -0400 Subject: [PATCH 022/396] Fixes for codeclimate --- app/controllers/supplemental_files_controller.rb | 6 +++--- app/models/supplemental_file.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/supplemental_files_controller.rb b/app/controllers/supplemental_files_controller.rb index daf0601910..0035bdd90f 100644 --- a/app/controllers/supplemental_files_controller.rb +++ b/app/controllers/supplemental_files_controller.rb @@ -137,7 +137,7 @@ def edit_structure_path end def edit_file_information - ident = "machine_generated_#{params[:id].to_s}".to_sym + ident = "machine_generated_#{params[:id]}".to_sym if params[ident] && !@supplemental_file.machine_generated? @supplemental_file.tags += ['machine_generated'] @@ -153,11 +153,11 @@ def edit_file_information end end - def edit_file_label label + def edit_file_label(label) label = if label.exclude?('(machine generated)') && File.extname(label).present? label.gsub(File.extname(label), " (machine generated)#{File.extname(label)}") elsif label.exclude?('(machine generated)') && File.extname(label).blank? - label + (" (machine generated)") + label + " (machine generated)" else label end diff --git a/app/models/supplemental_file.rb b/app/models/supplemental_file.rb index 1ae6421477..2b09f0e139 100644 --- a/app/models/supplemental_file.rb +++ b/app/models/supplemental_file.rb @@ -26,6 +26,6 @@ def attach_file(new_file) end def machine_generated? - self.tags.include?('machine_generated') + tags.include?('machine_generated') end end From 87e3a687c8900706c6b96b9a966c7bb8dd260878 Mon Sep 17 00:00:00 2001 From: Chris Colvard Date: Fri, 28 Apr 2023 17:10:15 -0400 Subject: [PATCH 023/396] Include metadata into IIIF manifest the same way as it is rendered on the item view page --- app/models/iiif_manifest_presenter.rb | 158 +++++++++++++++----- spec/factories/media_objects.rb | 3 +- spec/models/iiif_manifest_presenter_spec.rb | 17 +++ 3 files changed, 141 insertions(+), 37 deletions(-) diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb index a7bbd2a888..004de336bb 100644 --- a/app/models/iiif_manifest_presenter.rb +++ b/app/models/iiif_manifest_presenter.rb @@ -1,11 +1,11 @@ # Copyright 2011-2023, The Trustees of Indiana University and Northwestern # University. Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -# +# # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the @@ -13,6 +13,11 @@ # --- END LICENSE_HEADER BLOCK --- class IiifManifestPresenter + # Should 
     tags be allowed?
+  # They aren't listed in the IIIF spec but we use them in our normal view page
+  IIIF_ALLOWED_TAGS = ['a', 'b', 'br', 'i', 'img', 'p', 'pre', 'small', 'span', 'sub', 'sup'].freeze
+  IIIF_ALLOWED_ATTRIBUTES = ['href', 'src', 'alt'].freeze
+
   attr_reader :media_object, :master_files
 
   def initialize(media_object:, master_files:)
@@ -42,11 +47,7 @@ def to_s
   end
 
   def manifest_metadata
-    @manifest_metadata ||= iiif_metadata_fields.collect do |f|
-      value = media_object.send(f)
-      next if value.blank?
-      { 'label' => f.to_s.titleize, 'value' => Array(value) }
-    end.compact
+    @manifest_metadata ||= iiif_metadata_fields.compact
   end
 
   def thumbnail
@@ -75,36 +76,121 @@ def homepage
 
   private
 
-    def iiif_metadata_fields
-      # TODO: refine and order this list of fields
-      [:title, :creator, :date_created, :date_issued, :note, :contributor,
-       :publisher, :subject, :genre, :geographic_subject, :temporal_subject, :topical_subject, :rights_statement]
+  def sanitize(value)
+    Rails::Html::Sanitizer.safe_list_sanitizer.new.sanitize(value, tags: IIIF_ALLOWED_TAGS, attributes: IIIF_ALLOWED_ATTRIBUTES)
+  end
+
+  # Following methods adapted from ApplicationHelper and MediaObjectHelper
+  def metadata_field(label, value, default = nil)
+    sanitized_values = Array(value).collect { |v| sanitize(v.to_s.strip) }.delete_if(&:empty?)
+    return nil if sanitized_values.empty? && default.nil?
+    sanitized_values = Array(default) if sanitized_values.empty?
+    label = label.pluralize(sanitized_values.size)
+    { 'label' => label, 'value' => sanitized_values.join('; ') }
+  end
+
+  def combined_display_date(media_object)
+    result = media_object.date_issued
+    result += " (Creation date: #{media_object.date_created})" if media_object.date_created.present?
+    result
+  end
+
+  def display_other_identifiers(media_object)
+    # bibliographic_id has form [:type,"value"], other_identifier has form [[:type,"value],[:type,"value"],...]
+    ids = media_object.bibliographic_id.present? ? [media_object.bibliographic_id] : []
+    ids += Array(media_object.other_identifier)
+    ids.uniq.collect { |i| "#{ModsDocument::IDENTIFIER_TYPES[i[:source]]}: #{i[:id]}" }
+  end
+
+  def note_fields(media_object)
+    fields = []
+    note_types = ModsDocument::NOTE_TYPES.clone
+    note_types['table of contents'] = 'Contents'
+    note_types['general'] = 'Notes'
+    sorted_note_types = note_types.keys.sort
+    sorted_note_types.prepend(sorted_note_types.delete('general'))
+    sorted_note_types.each do |note_type|
+      notes = note_type == 'table of contents' ? media_object.table_of_contents : gather_notes_of_type(media_object, note_type)
+      fields << metadata_field(note_types[note_type], notes.collect { |note| "
    #{note}
    " }.join)
     end
+    fields
+  end
+
+  def gather_notes_of_type(media_object, type)
+    media_object.note.present? ? media_object.note.select { |n| n[:type] == type }.collect { |n| n[:note] } : []
+  end
+
+  def display_collection(media_object)
+    "#{media_object.collection.name}"
+  end
+
+  def display_unit(media_object)
+    "#{media_object.collection.unit}"
+  end
+
+  def display_language(media_object)
+    media_object.language.collect { |l| l[:text] }.uniq
+  end
+
+  def display_related_item(media_object)
+    media_object.related_item_url.collect { |r| "#{r[:label]}" }
+  end
+
+  def display_rights_statement(media_object)
+    return nil unless media_object.rights_statement.present?
+    label = ModsDocument::RIGHTS_STATEMENTS[media_object.rights_statement]
+    return nil unless label.present?
+    "#{label}"
+  end
 
-    def image_for(document)
-      master_file_id = document["section_id_ssim"].try :first
-
-      video_count = document["avalon_resource_type_ssim"].count{|m| m.start_with?('moving image'.titleize) } rescue 0
-      audio_count = document["avalon_resource_type_ssim"].count{|m| m.start_with?('sound recording'.titleize) } rescue 0
-
-      if master_file_id
-        if video_count > 0
-         Rails.application.routes.url_helpers.thumbnail_master_file_url(master_file_id)
-        elsif audio_count > 0
-          ActionController::Base.helpers.asset_url('audio_icon.png')
-        else
-          nil
-        end
-      else
-        if video_count > 0 && audio_count > 0
-          ActionController::Base.helpers.asset_url('hybrid_icon.png')
-        elsif video_count > audio_count
-          ActionController::Base.helpers.asset_url('video_icon.png')
-        elsif audio_count > video_count
-          ActionController::Base.helpers.asset_url('audio_icon.png')
-        else
-          nil
-        end
+  def display_summary(media_object)
+    return nil unless media_object.abstract.present?
+    "
    #{media_object.abstract}
    "
+  end
+
+  def iiif_metadata_fields
+    fields = [
+      metadata_field('Title', media_object.title),
+      metadata_field('Date', combined_display_date(media_object), 'Not provided'),
+      metadata_field('Main contributor', media_object.creator),
+      metadata_field('Summary', display_summary(media_object)),
+      metadata_field('Contributor', media_object.contributor),
+      metadata_field('Publisher', media_object.publisher),
+      metadata_field('Genre', media_object.genre),
+      metadata_field('Subject', media_object.subject),
+      metadata_field('Time period', media_object.temporal_subject),
+      metadata_field('Location', media_object.geographic_subject),
+      metadata_field('Collection', display_collection(media_object)),
+      metadata_field('Unit', display_unit(media_object)),
+      metadata_field('Language', display_language(media_object)),
+      metadata_field('Rights Statement', display_rights_statement(media_object)),
+      metadata_field('Terms of Use', media_object.terms_of_use),
+      metadata_field('Physical Description', media_object.physical_description),
+      metadata_field('Related Item', display_related_item(media_object))
+    ]
+    fields += note_fields(media_object)
+    fields += [metadata_field('Other Identifier', display_other_identifiers(media_object))]
+    fields
+  end
+
+  def image_for(document)
+    master_file_id = document["section_id_ssim"].try :first
+
+    video_count = document["avalon_resource_type_ssim"]&.count { |m| m.start_with?('moving image'.titleize) } || 0
+    audio_count = document["avalon_resource_type_ssim"]&.count { |m| m.start_with?('sound recording'.titleize) } || 0
+
+    if master_file_id
+      if video_count > 0
+        Rails.application.routes.url_helpers.thumbnail_master_file_url(master_file_id)
+      elsif audio_count > 0
+        ActionController::Base.helpers.asset_url('audio_icon.png')
       end
+    elsif video_count > 0 && audio_count > 0
+      ActionController::Base.helpers.asset_url('hybrid_icon.png')
+    elsif video_count > audio_count
+      ActionController::Base.helpers.asset_url('video_icon.png')
+    elsif audio_count > video_count
+      ActionController::Base.helpers.asset_url('audio_icon.png')
     end
+  end
 end
diff --git a/spec/factories/media_objects.rb b/spec/factories/media_objects.rb
index 1a1a2dcb76..70e318bb1b 100644
--- a/spec/factories/media_objects.rb
+++ b/spec/factories/media_objects.rb
@@ -40,13 +40,14 @@
         geographic_subject { [Faker::Address.country] }
         physical_description { [Faker::Lorem.word] }
         table_of_contents { [Faker::Lorem.paragraph] }
-        note { [{ note: Faker::Lorem.paragraph, type: 'general' }] }
+        note { [{ note: Faker::Lorem.paragraph, type: 'general' }, { note: Faker::Lorem.paragraph, type: 'local' }] }
         other_identifier { [{ id: Faker::Lorem.word, source: 'local' }] }
         language { ['eng'] }
         related_item_url { [{ url: Faker::Internet.url, label: Faker::Lorem.sentence }]}
         bibliographic_id { { id: Faker::Lorem.word, source: 'local' } }
         comment { ['MO comment'] }
         rights_statement { ['http://rightsstatements.org/vocab/InC-EDU/1.0/'] }
+        terms_of_use { [ 'Terms of Use: Be kind. Rewind.' ] }
         # after(:create) do |mo|
         #   mo.update_datastream(:descMetadata, {
         #     note: {note[Faker::Lorem.paragraph],
diff --git a/spec/models/iiif_manifest_presenter_spec.rb b/spec/models/iiif_manifest_presenter_spec.rb
index cdced8a966..8f412f019d 100644
--- a/spec/models/iiif_manifest_presenter_spec.rb
+++ b/spec/models/iiif_manifest_presenter_spec.rb
@@ -29,4 +29,21 @@
       expect(subject[:label]).to include("none" => ["View in Repository"])
     end
   end
+
+  context 'metadata' do
+    let(:media_object) { FactoryBot.build(:fully_searchable_media_object) }
+    subject do
+      metadata_hash = {}
+      presenter.manifest_metadata.map { |e| metadata_hash[e['label']] = e['value'] }
+      metadata_hash
+    end
+
+    it 'provides metadata' do
+      ['Title', 'Date', 'Main contributor', 'Summary', 'Contributor', 'Publisher', 'Genre', 'Subject', 'Time period',
+       'Location', 'Collection', 'Unit', 'Language', 'Rights Statement', 'Terms of Use', 'Physical Description',
+       'Related Item', 'Notes', 'Contents', 'Local Note', 'Other Identifiers'].each do |field|
+        expect(subject).to include(field)
+      end
+    end
+  end
 end

From 29ef48197986e90092d0587a98c8751a63b39400 Mon Sep 17 00:00:00 2001
From: dananji 
Date: Tue, 2 May 2023 14:13:37 -0400
Subject: [PATCH 024/396] New SME build

---
 yarn.lock | 51 ++++++++++++++++++++++-----------------------------
 1 file changed, 22 insertions(+), 29 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 0e2c6a24cc..72761be0ef 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -894,9 +894,9 @@
     "@babel/plugin-transform-react-pure-annotations" "^7.14.5"
 
 "@babel/runtime@^7.1.2", "@babel/runtime@^7.15.4", "@babel/runtime@^7.2.0", "@babel/runtime@^7.3.1", "@babel/runtime@^7.4.4", "@babel/runtime@^7.8.3", "@babel/runtime@^7.9.2":
-  version "7.21.0"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.21.0.tgz#5b55c9d394e5fcf304909a8b00c07dc217b56673"
-  integrity sha512-xwII0//EObnq89Ji5AKYQaRYiW/nZ3llSv29d49IuxPhKbtJoLP+9QUUZ4nVragQVtaVGeZrpB+ZtG/Pdy/POw==
+  version "7.21.5"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.21.5.tgz#8492dddda9644ae3bda3b45eabe87382caee7200"
+  integrity sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==
   dependencies:
     regenerator-runtime "^0.13.11"
 
@@ -3869,9 +3869,9 @@ hex-color-regex@^1.1.0:
   integrity sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==
 
 hls.js@^1.1.2:
-  version "1.3.4"
-  resolved "https://registry.yarnpkg.com/hls.js/-/hls.js-1.3.4.tgz#8212a3f95c3321f64a586f20e67876f3a9d09488"
-  integrity sha512-iFEwVqtEDk6sKotcTwtJ5OMo/nuDTk9PrpB8FI2J2WYf8EriTVfR4FaK0aNyYtwbYeRSWCXJKlz23xeREdlNYg==
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/hls.js/-/hls.js-1.4.0.tgz#e73af3e7c8d4310a2bcc92d5ade362dfb781fef7"
+  integrity sha512-VEjg7Rx5FlE9TB3MIn0HPgq3J+vR7EoQnjaqMCk/ISEaCOSZlAFh4g867f1QkSxZiq9kHeUZo+iH16X7VS3jKA==
 
 "hls.js@https://github.com/avalonmediasystem/hls.js#stricter_ts_probing":
   version "0.13.1"
@@ -4148,11 +4148,6 @@ ini@^1.3.4, ini@^1.3.5:
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c"
   integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==
 
-inline-worker@~1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/inline-worker/-/inline-worker-1.1.0.tgz#55e96f54915a642b00872a2daa6fe832b424c98d"
-  integrity sha512-2nlxBGg5Uoop6IRco9wMzlAz62690ylokrUDg3IaCi9bJaq0HykbWlRtRXgvSKiBNmCBGXnOCFBkIFjaSGJocA==
-
 internal-slot@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/internal-slot/-/internal-slot-1.0.3.tgz#7347e307deeea2faac2ac6205d4bc7d34967f59c"
@@ -4653,10 +4648,10 @@ klona@^2.0.4:
   resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.4.tgz#7bb1e3affb0cb8624547ef7e8f6708ea2e39dfc0"
   integrity sha512-ZRbnvdg/NxqzC7L9Uyqzf4psi1OM4Cuc+sJAkQPjO6XkQIJTNbfK2Rsmbw8fx1p2mkZdp2FZYo2+LwXYY/uwIA==
 
-konva@~7.2.5:
-  version "7.2.5"
-  resolved "https://registry.yarnpkg.com/konva/-/konva-7.2.5.tgz#9b4ac3a353e6be66e3e69123bf2a0cbc61efeb26"
-  integrity sha512-yk/li8rUF+09QNlOdkwbEId+QvfATMe/aMGVouWW1oFoUVTYWHsQuIAE6lWy11DK8mLJEJijkNAXC5K+NVlMew==
+konva@^9.0.0:
+  version "9.0.1"
+  resolved "https://registry.yarnpkg.com/konva/-/konva-9.0.1.tgz#92b4171beaa94273b108bd87b08c4d3e51a0da22"
+  integrity sha512-wzpkprJ8idE42TDF9Lu9RNjVVYNXrj0apvTK3pujdHQhX1iNV+MUquSxYN8HqjYSG95QQ51jhFzRLWhnhf44Mw==
 
 last-call-webpack-plugin@^3.0.0:
   version "3.0.0"
@@ -5624,14 +5619,12 @@ pbkdf2@^3.0.3:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
-peaks.js@^0.26.0:
-  version "0.26.0"
-  resolved "https://registry.yarnpkg.com/peaks.js/-/peaks.js-0.26.0.tgz#0ff00369b19a74522b0fe313f6b967b7c05aa1c1"
-  integrity sha512-kUsVDa+ISqK8V5B3dy4aXgQ1AbtPa7AIEHfJsOpSLIqgkrq1d3EOukbzrMiyNnODeh9qRaGEgHq40KlQEQK90Q==
+peaks.js@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/peaks.js/-/peaks.js-2.1.0.tgz#d494c1752e13c0d9a1064d5ec5f408640eec032b"
+  integrity sha512-oWsqrHckPv9w1yXIciX81kVXPZSFifGhETmR1bLFbV7KlqPRCYvLVaTpHc6Nl+Ogps0BMRvCnrQLEYCWBKA0uA==
   dependencies:
     eventemitter3 "~4.0.7"
-    konva "~7.2.5"
-    waveform-data "~4.1.0"
 
 picomatch@^2.0.4, picomatch@^2.2.1:
   version "2.3.0"
@@ -6604,7 +6597,7 @@ react-redux@^7.2.6:
 
 "react-structural-metadata-editor@https://github.com/avalonmediasystem/react-structural-metadata-editor":
   version "1.1.0"
-  resolved "https://github.com/avalonmediasystem/react-structural-metadata-editor#c29229daf1b2efe398f55cb8dfc0c603bdd6441d"
+  resolved "https://github.com/avalonmediasystem/react-structural-metadata-editor#50dbd6d38fde3ccdaf174c2048a7c4f7e3fcbe36"
   dependencies:
     "@babel/runtime" "^7.4.4"
     "@fortawesome/fontawesome-svg-core" "^1.2.4"
@@ -6614,9 +6607,10 @@ react-redux@^7.2.6:
     "@material-ui/icons" "4.2.1"
     axios "^0.21.2"
     hls.js "^1.1.2"
+    konva "^9.0.0"
     lodash "4.17.21"
     manifesto.js "^4.2.14"
-    peaks.js "^0.26.0"
+    peaks.js "^2.1.0"
     prop-types "^15.6.2"
     react-bootstrap "1.6.4"
     react-dnd "^7.0.2"
@@ -6627,6 +6621,7 @@ react-redux@^7.2.6:
     redux-thunk "^2.3.0"
     rxjs "^6.4.0"
     uuid "^3.3.2"
+    waveform-data "^4.3.0"
 
 react-transition-group@2.5.3:
   version "2.5.3"
@@ -8005,12 +8000,10 @@ watchpack@^1.7.4:
     chokidar "^3.4.1"
     watchpack-chokidar2 "^2.0.1"
 
-waveform-data@~4.1.0:
-  version "4.1.1"
-  resolved "https://registry.yarnpkg.com/waveform-data/-/waveform-data-4.1.1.tgz#641e6ae6c190529f9df79176009075618f236946"
-  integrity sha512-OAKQPY0j4e/cn7jv/zHFw8bzL4i93MCGlyfc42LDQxeT6pnvzxynzbNm5TCopPViKAan5zs1/8WrAsC4VxJjlg==
-  dependencies:
-    inline-worker "~1.1.0"
+waveform-data@^4.3.0:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/waveform-data/-/waveform-data-4.3.0.tgz#03fcafa522ec1302f48583cc0c9ffd20737d499d"
+  integrity sha512-gddZgTV1dM8Mfaqn+mCJjwTAc7O/GbX2LgOrijRCHEGC60ZVdd/qTzrgEcCE5MfBoBPgwQ85huy1icSf5IZY5Q==
 
 wbuf@^1.1.0, wbuf@^1.7.3:
   version "1.7.3"

From 8997c9d677a7e657c026bf54008abdc77cfb29a9 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 3 May 2023 11:42:12 -0400
Subject: [PATCH 025/396] Remove auth_service from public iiif manifests

---
 app/models/iiif_canvas_presenter.rb       | 38 ++++++++++++++---------
 spec/models/iiif_canvas_presenter_spec.rb | 20 ++++++++----
 2 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 9325d24626..f36adceefc 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -55,13 +55,7 @@ def video_content
 
     def video_display_content(quality)
       IIIFManifest::V3::DisplayContent.new(Rails.application.routes.url_helpers.hls_manifest_master_file_url(master_file.id, quality: quality),
-                                           label: quality,
-                                           width: master_file.width.to_i,
-                                           height: master_file.height.to_i,
-                                           duration: stream_info[:duration],
-                                           type: 'Video',
-                                           format: 'application/x-mpegURL',
-                                           auth_service: auth_service(quality))
+                                           manifest_attributes(quality, 'Video'))
     end
 
     def audio_content
@@ -70,11 +64,7 @@ def audio_content
 
     def audio_display_content(quality)
       IIIFManifest::V3::DisplayContent.new(Rails.application.routes.url_helpers.hls_manifest_master_file_url(master_file.id, quality: quality),
-                                           label: quality,
-                                           duration: stream_info[:duration],
-                                           type: 'Sound',
-                                           format: 'application/x-mpegURL',
-                                           auth_service: auth_service(quality))
+                                           manifest_attributes(quality, 'Sound'))
     end
 
     def stream_urls
@@ -134,6 +124,24 @@ def parse_hour_min_sec(s)
       smh[0] + (60 * smh[1]) + (3600 * smh[2])
     end
 
+    def manifest_attributes(quality, media_type)
+      media_hash = {
+                      label: quality,
+                      width: master_file.width.to_i,
+                      height: master_file.height.to_i,
+                      duration: stream_info[:duration],
+                      type: media_type,
+                      format: 'application/x-mpegURL'
+                    }.compact
+
+      if master_file.media_object.visibility == 'public'
+        media_hash
+      else
+        media_hash.merge!({ auth_service: auth_service(quality) })
+      end
+    end
+
+
     # Note that the method returns empty Nokogiri Document instead of nil when structure_tesim doesn't exist or is empty.
     def structure_ng_xml
       # TODO: The XML parser should handle invalid XML files, for ex, if a non-leaf node has no valid "Div" or "Span" children,
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index f12563be45..270e56c185 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -37,6 +37,14 @@
       logout_service = subject[:service].find { |s| s[:@type] == "AuthLogoutService1"}
       expect(logout_service[:@id]).to eq Rails.application.routes.url_helpers.destroy_user_session_url
     end
+
+    context 'when public media object' do
+      let(:media_object) { FactoryBot.build(:fully_searchable_media_object) }
+
+      it "does not provide an auth service" do
+        expect(presenter.display_content.first.auth_service).to be_nil
+      end
+    end
   end
 
   describe '#display_content' do
@@ -81,9 +89,9 @@
       let(:structure_xml) { "" }
 
       it 'autogenerates a basic range' do
-	expect(subject.label.to_s).to eq "{\"none\"=>[\"#{master_file.embed_title}\"]}"
-	expect(subject.items.size).to eq 1
-	expect(subject.items.first).to be_a IiifCanvasPresenter
+      	expect(subject.label.to_s).to eq "{\"none\"=>[\"#{master_file.embed_title}\"]}"
+      	expect(subject.items.size).to eq 1
+      	expect(subject.items.first).to be_a IiifCanvasPresenter
         expect(subject.items.first.media_fragment).to eq 't=0,'
       end
     end

From 4bdae673d77435583bdc6172627c538d49ac2a94 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 4 May 2023 10:58:38 -0400
Subject: [PATCH 026/396] Fixes for codeclimate

---
 app/models/iiif_canvas_presenter.rb       | 17 ++++++++---------
 spec/models/iiif_canvas_presenter_spec.rb |  4 ++--
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index f36adceefc..4d6d76ee56 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -126,22 +126,21 @@ def parse_hour_min_sec(s)
 
     def manifest_attributes(quality, media_type)
       media_hash = {
-                      label: quality,
-                      width: master_file.width.to_i,
-                      height: master_file.height.to_i,
-                      duration: stream_info[:duration],
-                      type: media_type,
-                      format: 'application/x-mpegURL'
-                    }.compact
+                     label: quality,
+                     width: master_file.width.to_i,
+                     height: master_file.height.to_i,
+                     duration: stream_info[:duration],
+                     type: media_type,
+                     format: 'application/x-mpegURL'
+                   }.compact
 
       if master_file.media_object.visibility == 'public'
         media_hash
       else
-        media_hash.merge!({ auth_service: auth_service(quality) })
+        media_hash.merge!(auth_service: auth_service(quality))
       end
     end
 
-
     # Note that the method returns empty Nokogiri Document instead of nil when structure_tesim doesn't exist or is empty.
     def structure_ng_xml
       # TODO: The XML parser should handle invalid XML files, for ex, if a non-leaf node has no valid "Div" or "Span" children,
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 270e56c185..ce8f85c34f 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -15,7 +15,7 @@
 require 'rails_helper'
 
 describe IiifCanvasPresenter do
-  let(:media_object) { FactoryBot.build(:media_object) }
+  let(:media_object) { FactoryBot.build(:media_object, visibility: 'private') }
   let(:derivative) { FactoryBot.build(:derivative) }
   let(:master_file) { FactoryBot.build(:master_file, media_object: media_object, derivatives: [derivative]) }
   let(:stream_info) { master_file.stream_details }
@@ -39,7 +39,7 @@
     end
 
     context 'when public media object' do
-      let(:media_object) { FactoryBot.build(:fully_searchable_media_object) }
+      let(:media_object) { FactoryBot.build(:media_object, visibility: 'public') }
 
       it "does not provide an auth service" do
         expect(presenter.display_content.first.auth_service).to be_nil

From 2409ade3ab74cb731504008fae92276070200fe0 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Tue, 9 May 2023 13:54:33 -0400
Subject: [PATCH 027/396] Ensure that manifest_attribute hash is converted to
 kwargs

---
 app/models/iiif_canvas_presenter.rb | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 4d6d76ee56..4401f8aea6 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -55,7 +55,7 @@ def video_content
 
     def video_display_content(quality)
       IIIFManifest::V3::DisplayContent.new(Rails.application.routes.url_helpers.hls_manifest_master_file_url(master_file.id, quality: quality),
-                                           manifest_attributes(quality, 'Video'))
+                                           **manifest_attributes(quality, 'Video'))
     end
 
     def audio_content
@@ -64,7 +64,7 @@ def audio_content
 
     def audio_display_content(quality)
       IIIFManifest::V3::DisplayContent.new(Rails.application.routes.url_helpers.hls_manifest_master_file_url(master_file.id, quality: quality),
-                                           manifest_attributes(quality, 'Sound'))
+                                           **manifest_attributes(quality, 'Sound'))
     end
 
     def stream_urls
@@ -126,13 +126,13 @@ def parse_hour_min_sec(s)
 
     def manifest_attributes(quality, media_type)
       media_hash = {
-                     label: quality,
-                     width: master_file.width.to_i,
-                     height: master_file.height.to_i,
-                     duration: stream_info[:duration],
-                     type: media_type,
-                     format: 'application/x-mpegURL'
-                   }.compact
+        label: quality,
+        width: master_file.width.to_i,
+        height: master_file.height.to_i,
+        duration: stream_info[:duration],
+        type: media_type,
+        format: 'application/x-mpegURL'
+      }.compact
 
       if master_file.media_object.visibility == 'public'
         media_hash

From a8b9d0cae0d8cd3273123e1a71a538b4694f4a10 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 10 May 2023 12:52:18 -0400
Subject: [PATCH 028/396] Bump active_encode and add ffmpeg cleanup rake task

---
 Gemfile               |  2 +-
 Gemfile.lock          |  4 ++--
 lib/tasks/avalon.rake | 18 +++++++++++++++---
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/Gemfile b/Gemfile
index d5fbfc9c9b..f9123f4cbc 100644
--- a/Gemfile
+++ b/Gemfile
@@ -74,7 +74,7 @@ gem 'omniauth-lti', git: "https://github.com/avalonmediasystem/omniauth-lti.git"
 gem "omniauth-saml", "~> 2.0"
 
 # Media Access & Transcoding
-gem 'active_encode', '~> 1.0', '>= 1.1.3'
+gem 'active_encode', '~> 1.2'
 gem 'audio_waveform-ruby', '~> 1.0.7', require: 'audio_waveform'
 gem 'browse-everything', git: "https://github.com/avalonmediasystem/browse-everything.git", branch: 'v1.2-avalon'
 gem 'fastimage'
diff --git a/Gemfile.lock b/Gemfile.lock
index c5d22d6cdf..7feea026ab 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -135,7 +135,7 @@ GEM
     active_elastic_job (3.2.0)
       aws-sdk-sqs (~> 1)
       rails (>= 5.2.6, < 7.1)
-    active_encode (1.1.3)
+    active_encode (1.2.0)
       addressable (~> 2.8)
       rails
     active_fedora-datastreams (0.5.0)
@@ -985,7 +985,7 @@ DEPENDENCIES
   active-fedora (~> 14.0, >= 14.0.1)
   active_annotations (~> 0.4)
   active_elastic_job
-  active_encode (~> 1.0, >= 1.1.3)
+  active_encode (~> 1.2)
   active_fedora-datastreams (~> 0.5)
   activejob-traffic_control
   activejob-uniqueness
diff --git a/lib/tasks/avalon.rake b/lib/tasks/avalon.rake
index 53269d430b..651bd5b8a8 100644
--- a/lib/tasks/avalon.rake
+++ b/lib/tasks/avalon.rake
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -43,6 +43,18 @@ namespace :avalon do
     CleanupSessionJob.perform_now
   end
 
+  desc 'clean out old ffmpeg and pass_through encode files'
+  task local_encode_cleanup: :environment do
+    options = {
+      older_than: ENV['older_than'], # Default is 2.weeks
+      no_outputs: ENV['no_outputs'].to_a, # Default is ['input_metadata', 'duration_input_metadata', 'error.log', 'exit_status.code', 'progress', 'completed', 'pid', 'output_metadata-*']
+      outputs: ENV['outputs'], # Default is false
+      all: ENV['all'] # Default is false
+    }.compact
+
+    ActiveEncode::EngineAdapters::FfmpegAdapter.remove_old_files!(options)
+  end
+
   namespace :services do
     services = ["jetty", "felix", "delayed_job"]
     desc "Start Avalon's dependent services"

From 210f1e233683ffd44d85e90b4a91c2b431bf350d Mon Sep 17 00:00:00 2001
From: Mason Ballengee <68433277+masaball@users.noreply.github.com>
Date: Wed, 10 May 2023 14:26:20 -0400
Subject: [PATCH 029/396] Update lib/tasks/avalon.rake

Co-authored-by: Chris Colvard 
---
 lib/tasks/avalon.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tasks/avalon.rake b/lib/tasks/avalon.rake
index 651bd5b8a8..baf37b26c1 100644
--- a/lib/tasks/avalon.rake
+++ b/lib/tasks/avalon.rake
@@ -47,7 +47,7 @@ namespace :avalon do
   task local_encode_cleanup: :environment do
     options = {
       older_than: ENV['older_than'], # Default is 2.weeks
-      no_outputs: ENV['no_outputs'].to_a, # Default is ['input_metadata', 'duration_input_metadata', 'error.log', 'exit_status.code', 'progress', 'completed', 'pid', 'output_metadata-*']
+      no_outputs: ENV['no_outputs']&.to_a, # Default is ['input_metadata', 'duration_input_metadata', 'error.log', 'exit_status.code', 'progress', 'completed', 'pid', 'output_metadata-*']
       outputs: ENV['outputs'], # Default is false
       all: ENV['all'] # Default is false
     }.compact

From f550f049d0f802e42c5609053b63de53d02a73e6 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 10 May 2023 14:54:58 -0400
Subject: [PATCH 030/396] Remove pre tags

---
 app/models/iiif_manifest_presenter.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 004de336bb..2a4f9d6890 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -15,7 +15,7 @@
 class IiifManifestPresenter
   # Should 
     tags be allowed?
   # They aren't listed in the IIIF spec but we use them in our normal view page
-  IIIF_ALLOWED_TAGS = ['a', 'b', 'br', 'i', 'img', 'p', 'pre', 'small', 'span', 'sub', 'sup'].freeze
+  IIIF_ALLOWED_TAGS = ['a', 'b', 'br', 'i', 'img', 'p', 'small', 'span', 'sub', 'sup'].freeze
   IIIF_ALLOWED_ATTRIBUTES = ['href', 'src', 'alt'].freeze
 
   attr_reader :media_object, :master_files
@@ -111,7 +111,7 @@ def note_fields(media_object)
     sorted_note_types.prepend(sorted_note_types.delete('general'))
     sorted_note_types.each do |note_type|
       notes = note_type == 'table of contents' ? media_object.table_of_contents : gather_notes_of_type(media_object, note_type)
-      fields << metadata_field(note_types[note_type], notes.collect { |note| "
    #{note}
    " }.join)
+      fields << metadata_field(note_types[note_type], notes.join)
     end
     fields
   end
@@ -145,7 +145,7 @@ def display_rights_statement(media_object)
 
   def display_summary(media_object)
     return nil unless media_object.abstract.present?
-    "
    #{media_object.abstract}
    "
+    media_object.abstract
   end
 
   def iiif_metadata_fields

From e5f4c88d72538b69aaf1e31c7c8570c947f3f321 Mon Sep 17 00:00:00 2001
From: Jon Cameron 
Date: Fri, 5 May 2023 13:19:19 -0400
Subject: [PATCH 031/396] Change Table of Contents Label

---
 app/models/iiif_manifest_presenter.rb       | 2 +-
 spec/models/iiif_manifest_presenter_spec.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 2a4f9d6890..5ab1755641 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -105,7 +105,7 @@ def display_other_identifiers(media_object)
   def note_fields(media_object)
     fields = []
     note_types = ModsDocument::NOTE_TYPES.clone
-    note_types['table of contents'] = 'Contents'
+    note_types['table of contents'] = 'Table of Contents'
     note_types['general'] = 'Notes'
     sorted_note_types = note_types.keys.sort
     sorted_note_types.prepend(sorted_note_types.delete('general'))
diff --git a/spec/models/iiif_manifest_presenter_spec.rb b/spec/models/iiif_manifest_presenter_spec.rb
index 8f412f019d..2ac55c00ee 100644
--- a/spec/models/iiif_manifest_presenter_spec.rb
+++ b/spec/models/iiif_manifest_presenter_spec.rb
@@ -41,7 +41,7 @@
     it 'provides metadata' do
       ['Title', 'Date', 'Main contributor', 'Summary', 'Contributor', 'Publisher', 'Genre', 'Subject', 'Time period',
        'Location', 'Collection', 'Unit', 'Language', 'Rights Statement', 'Terms of Use', 'Physical Description',
-       'Related Item', 'Notes', 'Contents', 'Local Note', 'Other Identifiers'].each do |field|
+       'Related Item', 'Notes', 'Table of Contents', 'Local Note', 'Other Identifiers'].each do |field|
         expect(subject).to include(field)
       end
     end

From 269cdc10bddceb7fb4ff71039de4abbca261b500 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 10 May 2023 15:30:05 -0400
Subject: [PATCH 032/396] Serialize multiple values as arrays

---
 app/models/iiif_manifest_presenter.rb       | 4 ++--
 spec/models/iiif_manifest_presenter_spec.rb | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 5ab1755641..85c8a6b559 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -86,7 +86,7 @@ def metadata_field(label, value, default = nil)
     return nil if sanitized_values.empty? && default.nil?
     sanitized_values = Array(default) if sanitized_values.empty?
     label = label.pluralize(sanitized_values.size)
-    { 'label' => label, 'value' => sanitized_values.join('; ') }
+    { 'label' => label, 'value' => sanitized_values }
   end
 
   def combined_display_date(media_object)
@@ -111,7 +111,7 @@ def note_fields(media_object)
     sorted_note_types.prepend(sorted_note_types.delete('general'))
     sorted_note_types.each do |note_type|
       notes = note_type == 'table of contents' ? media_object.table_of_contents : gather_notes_of_type(media_object, note_type)
-      fields << metadata_field(note_types[note_type], notes.join)
+      fields << metadata_field(note_types[note_type], notes)
     end
     fields
   end
diff --git a/spec/models/iiif_manifest_presenter_spec.rb b/spec/models/iiif_manifest_presenter_spec.rb
index 2ac55c00ee..a41d767c96 100644
--- a/spec/models/iiif_manifest_presenter_spec.rb
+++ b/spec/models/iiif_manifest_presenter_spec.rb
@@ -45,5 +45,13 @@
         expect(subject).to include(field)
       end
     end
+
+    context 'multiple values' do
+      let(:media_object) { FactoryBot.build(:fully_searchable_media_object, note: [{ note: Faker::Lorem.paragraph, type: 'general' }, { note: Faker::Lorem.paragraph, type: 'general' }]) }
+
+      it 'serializes multiple values as an array' do
+        expect(subject['Notes']).to be_a Array
+      end
+    end
   end
 end

From ccde6e35eb8186fc2f6e99fc19ae463a4fd1870b Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 10 May 2023 16:43:46 -0400
Subject: [PATCH 033/396] Use permalink for homepage property if available

---
 app/models/iiif_manifest_presenter.rb       |  2 +-
 spec/models/iiif_manifest_presenter_spec.rb | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 004de336bb..f49a84d14c 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -66,7 +66,7 @@ def ranges
   def homepage
     [
       {
-        id: Rails.application.routes.url_helpers.media_object_url(media_object),
+        id: media_object.permalink || Rails.application.routes.url_helpers.media_object_url(media_object),
         type: "Text",
         label: { "none" => [I18n.t('iiif.manifest.homepageLabel')] },
         format: "text/html"
diff --git a/spec/models/iiif_manifest_presenter_spec.rb b/spec/models/iiif_manifest_presenter_spec.rb
index 8f412f019d..29b8494855 100644
--- a/spec/models/iiif_manifest_presenter_spec.rb
+++ b/spec/models/iiif_manifest_presenter_spec.rb
@@ -22,12 +22,20 @@
   context 'homepage' do
     subject { presenter.homepage.first }
 
-    it 'provices a homepage' do
+    it 'provides a homepage' do
       expect(subject[:id]).to eq Rails.application.routes.url_helpers.media_object_url(media_object)
       expect(subject[:type]).to eq "Text"
       expect(subject[:format]).to eq "text/html"
       expect(subject[:label]).to include("none" => ["View in Repository"])
     end
+
+    context 'with permalink' do
+      let(:media_object) { FactoryBot.build(:media_object, permalink: "https://purl.example.edu/permalink") }
+
+      it 'uses the permalink' do
+        expect(subject[:id]).to eq media_object.permalink
+      end
+    end
   end
 
   context 'metadata' do

From 4e6b8de814a9bc64fd277c130c2c9e908e932552 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 11 May 2023 10:21:37 -0400
Subject: [PATCH 034/396] Remove editing of DB label

---
 .../supplemental_files_controller.rb          | 19 +------------------
 .../supplemental_files_controller_examples.rb | 13 ++++---------
 2 files changed, 5 insertions(+), 27 deletions(-)

diff --git a/app/controllers/supplemental_files_controller.rb b/app/controllers/supplemental_files_controller.rb
index 0035bdd90f..722a30c971 100644
--- a/app/controllers/supplemental_files_controller.rb
+++ b/app/controllers/supplemental_files_controller.rb
@@ -141,27 +141,10 @@ def edit_file_information
 
       if params[ident] && !@supplemental_file.machine_generated?
         @supplemental_file.tags += ['machine_generated']
-        edit_file_label supplemental_file_params[:label]
       elsif !params[ident] && @supplemental_file.machine_generated?
         @supplemental_file.tags -= ['machine_generated']
-        label = supplemental_file_params[:label].gsub(" (machine generated)", '')
-        @supplemental_file.label = label
-      elsif params[ident] && @supplemental_file.machine_generated?
-        edit_file_label supplemental_file_params[:label]
-      else
-        @supplemental_file.label = supplemental_file_params[:label]
       end
-    end
-
-    def edit_file_label(label)
-      label = if label.exclude?('(machine generated)') && File.extname(label).present?
-                label.gsub(File.extname(label), " (machine generated)#{File.extname(label)}")
-              elsif label.exclude?('(machine generated)') && File.extname(label).blank?
-                label + " (machine generated)"
-              else
-                label
-              end
-      @supplemental_file.label = label
+      @supplemental_file.label = supplemental_file_params[:label]
     end
 
     def object_supplemental_file_path
diff --git a/spec/support/supplemental_files_controller_examples.rb b/spec/support/supplemental_files_controller_examples.rb
index 4df886d4bf..9a27fba8bf 100644
--- a/spec/support/supplemental_files_controller_examples.rb
+++ b/spec/support/supplemental_files_controller_examples.rb
@@ -249,31 +249,26 @@
         let(:machine_param) { "machine_generated_#{supplemental_file.id}".to_sym }
         context "missing machine_generated tag" do
           let(:supplemental_file) { FactoryBot.create(:supplemental_file, :with_transcript_file, :with_transcript_tag, label: 'label') }
-          it "adds machine generated note to label and tags" do
+          it "adds machine generated note to tags" do
             expect {
               put :update, params: { class_id => object.id, id: supplemental_file.id, machine_param => 1, supplemental_file: valid_update_attributes, format: :html }, session: valid_session
-            }.to change { master_file.reload.supplemental_files.first.label }.from('label').to('new label (machine generated)')
-             .and change { master_file.reload.supplemental_files.first.tags }.from(['transcript']).to(['transcript', 'machine_generated'])
+            }.to change { master_file.reload.supplemental_files.first.tags }.from(['transcript']).to(['transcript', 'machine_generated'])
           end
         end
         context "with machine_generated tag" do
           let(:supplemental_file) { FactoryBot.create(:supplemental_file, :with_transcript_file, tags: ['transcript', 'machine_generated'], label: 'label (machine generated)') }
-          let(:valid_update_attributes) { { label: 'label (machine generated)' } }
           it "does not add more instances of machine generated note" do
             expect {
               put :update, params: { class_id => object.id, id: supplemental_file.id, machine_param => 1, supplemental_file: valid_update_attributes, format: :html }, session: valid_session
-            }.to not_change { master_file.reload.supplemental_files.first.label }.from('label (machine generated)')
-             .and not_change { master_file.reload.supplemental_files.first.tags }.from(['transcript', 'machine_generated'])
+            }.to not_change { master_file.reload.supplemental_files.first.tags }.from(['transcript', 'machine_generated'])
           end
         end
         context "removing machine_generated designation" do
           let(:supplemental_file) { FactoryBot.create(:supplemental_file, :with_transcript_file, tags: ['transcript', 'machine_generated'], label: 'label (machine generated)') }
-          let(:valid_update_attributes) { { label: 'label (machine generated)' } }
           it "removes machine generated note from label and tags" do
             expect {
               put :update, params: { class_id => object.id, id: supplemental_file.id, supplemental_file: valid_update_attributes, format: :html }, session: valid_session
-            }.to change { master_file.reload.supplemental_files.first.label }.from('label (machine generated)').to('label')
-             .and change { master_file.reload.supplemental_files.first.tags }.from(['transcript', 'machine_generated']).to(['transcript'])
+            }.to change { master_file.reload.supplemental_files.first.tags }.from(['transcript', 'machine_generated']).to(['transcript'])
           end
         end
       end

From 24b9cec2b57eee6b43b4ffee55192685be31585a Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 12 May 2023 11:38:01 -0400
Subject: [PATCH 035/396] Rescue Ldp::Gone in ApplicationJob

---
 app/jobs/application_job.rb       |  9 ++++++---
 spec/jobs/application_job_spec.rb | 25 +++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 spec/jobs/application_job_spec.rb

diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
index 30c5061965..c4de02b8cc 100644
--- a/app/jobs/application_job.rb
+++ b/app/jobs/application_job.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -13,4 +13,7 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class ApplicationJob < ActiveJob::Base
+  rescue_from Ldp::Gone do |exception|
+    Rails.logger.warn exception.message
+  end
 end
diff --git a/spec/jobs/application_job_spec.rb b/spec/jobs/application_job_spec.rb
new file mode 100644
index 0000000000..751b6e8665
--- /dev/null
+++ b/spec/jobs/application_job_spec.rb
@@ -0,0 +1,25 @@
+# Copyright 2011-2023, The Trustees of Indiana University and Northwestern
+#   University.  Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed
+#   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+#   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+#   specific language governing permissions and limitations under the License.
+# ---  END LICENSE_HEADER BLOCK  ---
+
+require 'rails_helper'
+
+describe ApplicationJob do
+  describe 'exception handling' do
+    it 'rescues Ldp::Gone errors' do
+      allow_any_instance_of(described_class).to receive(:perform).and_raise(Ldp::Gone)
+      expect(Rails.logger).to receive(:warn).with("Ldp::Gone")
+      expect { described_class.perform_now }.to_not raise_error
+    end
+  end
+end

From 1df38a213588731668ea6aa1853cafd28a51a47b Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 12 May 2023 11:46:58 -0400
Subject: [PATCH 036/396] Change Rails logger level

---
 app/jobs/application_job.rb       | 2 +-
 spec/jobs/application_job_spec.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
index c4de02b8cc..49e2ab6b5b 100644
--- a/app/jobs/application_job.rb
+++ b/app/jobs/application_job.rb
@@ -14,6 +14,6 @@
 
 class ApplicationJob < ActiveJob::Base
   rescue_from Ldp::Gone do |exception|
-    Rails.logger.warn exception.message
+    Rails.logger.error exception.message
   end
 end
diff --git a/spec/jobs/application_job_spec.rb b/spec/jobs/application_job_spec.rb
index 751b6e8665..8ce5cd96e8 100644
--- a/spec/jobs/application_job_spec.rb
+++ b/spec/jobs/application_job_spec.rb
@@ -18,7 +18,7 @@
   describe 'exception handling' do
     it 'rescues Ldp::Gone errors' do
       allow_any_instance_of(described_class).to receive(:perform).and_raise(Ldp::Gone)
-      expect(Rails.logger).to receive(:warn).with("Ldp::Gone")
+      expect(Rails.logger).to receive(:error).with("Ldp::Gone")
       expect { described_class.perform_now }.to_not raise_error
     end
   end

From c0e68fde431f8c03aea1dba07351aab47edd8905 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 12 May 2023 14:32:56 -0400
Subject: [PATCH 037/396] Administrative Facet for visibility level

---
 app/controllers/catalog_controller.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb
index 5d07e6c0d0..19c1929f84 100644
--- a/app/controllers/catalog_controller.rb
+++ b/app/controllers/catalog_controller.rb
@@ -90,6 +90,11 @@ class CatalogController < ApplicationController
     # Hide these facets if not a Collection Manager
     config.add_facet_field 'workflow_published_sim', label: 'Published', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow"
     config.add_facet_field 'avalon_uploader_ssi', label: 'Created by', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow"
+    config.add_facet_field 'read_access_group_ssim', label: 'Item access', if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow", query: {
+      public: { label: "Public", fq: "has_model_ssim:MediaObject AND read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC}" },
+      restricted: { label: "Logged in", fq: "has_model_ssim:MediaObject AND read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED}" },
+      private: { label: "Private", fq: "has_model_ssim:MediaObject AND NOT read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC} AND NOT read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED}" }
+    }
     config.add_facet_field 'read_access_virtual_group_ssim', label: 'External Group', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow", helper_method: :vgroup_display
     config.add_facet_field 'date_digitized_sim', label: 'Date Digitized', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow"#, partial: 'blacklight/hierarchy/facet_hierarchy'
     config.add_facet_field 'date_ingested_sim', label: 'Date Ingested', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow"

From 3dfd9582598fc691b088d4d422c7207a2720dfd7 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 12 May 2023 14:52:19 -0400
Subject: [PATCH 038/396] Include backtrace in error log

---
 app/jobs/application_job.rb       | 2 +-
 spec/jobs/application_job_spec.rb | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
index 49e2ab6b5b..5cf5f8819c 100644
--- a/app/jobs/application_job.rb
+++ b/app/jobs/application_job.rb
@@ -14,6 +14,6 @@
 
 class ApplicationJob < ActiveJob::Base
   rescue_from Ldp::Gone do |exception|
-    Rails.logger.error exception.message
+    Rails.logger.error(exception.message + '\n' + exception.backtrace.join('\n'))
   end
 end
diff --git a/spec/jobs/application_job_spec.rb b/spec/jobs/application_job_spec.rb
index 8ce5cd96e8..ab6fda2256 100644
--- a/spec/jobs/application_job_spec.rb
+++ b/spec/jobs/application_job_spec.rb
@@ -18,7 +18,8 @@
   describe 'exception handling' do
     it 'rescues Ldp::Gone errors' do
       allow_any_instance_of(described_class).to receive(:perform).and_raise(Ldp::Gone)
-      expect(Rails.logger).to receive(:error).with("Ldp::Gone")
+      allow_any_instance_of(Exception).to receive(:backtrace).and_return(["Test trace"])
+      expect(Rails.logger).to receive(:error).with('Ldp::Gone\nTest trace')
       expect { described_class.perform_now }.to_not raise_error
     end
   end

From dfb2bb624e535ede8b653dbd0dc0b26ff6e23c2a Mon Sep 17 00:00:00 2001
From: dananji 
Date: Mon, 15 May 2023 11:23:39 -0400
Subject: [PATCH 039/396] Add placeholderCanvas to IIIF manifest

---
 Gemfile                                   |  2 +-
 Gemfile.lock                              | 12 +++++++++---
 app/models/iiif_canvas_presenter.rb       |  9 +++++++++
 spec/models/iiif_canvas_presenter_spec.rb | 17 +++++++++++++++++
 4 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/Gemfile b/Gemfile
index f9123f4cbc..7ecca6a435 100644
--- a/Gemfile
+++ b/Gemfile
@@ -53,7 +53,7 @@ gem 'avalon-about', git: 'https://github.com/avalonmediasystem/avalon-about.git'
 #gem 'bootstrap-sass', '< 3.4.1' # Pin to less than 3.4.1 due to change in behavior with popovers
 gem 'bootstrap-toggle-rails'
 gem 'bootstrap_form'
-gem 'iiif_manifest', '~> 1.3'
+gem 'iiif_manifest', git: 'https://github.com/samvera-labs/iiif_manifest.git', branch: 'main'
 gem 'rack-cors', require: 'rack/cors'
 gem 'rails_same_site_cookie'
 gem 'recaptcha', require: 'recaptcha/rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index 7feea026ab..ec47eba561 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -65,6 +65,14 @@ GIT
       ims-lti
       omniauth
 
+GIT
+  remote: https://github.com/samvera-labs/iiif_manifest.git
+  revision: 33db00d04abbb0c66a6947bf24f8d913c5319c4f
+  branch: main
+  specs:
+    iiif_manifest (1.3.1)
+      activesupport (>= 4)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -493,8 +501,6 @@ GEM
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     iconv (1.0.8)
-    iiif_manifest (1.3.1)
-      activesupport (>= 4)
     ims-lti (1.1.13)
       builder
       oauth (>= 0.4.5, < 0.6)
@@ -1041,7 +1047,7 @@ DEPENDENCIES
   hooks
   hydra-head (~> 12.0)
   iconv (~> 1.0.6)
-  iiif_manifest (~> 1.3)
+  iiif_manifest!
   ims-lti (~> 1.1.13)
   jbuilder (~> 2.0)
   jquery-datatables
diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 4401f8aea6..0620dbd10a 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -46,6 +46,15 @@ def sequence_rendering
     }]
   end
 
+  def placeholder_content
+    # height and width from /models/master_file/extract_still method
+    IIIFManifest::V3::DisplayContent.new( @master_file.poster_master_file_url(@master_file.id),
+                                          width: 1280,
+                                          height: 720,
+                                          type: 'Image',
+                                          format: 'image/jpeg')
+  end
+
   private
 
     def video_content
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index ce8f85c34f..2cdff6b88c 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -96,4 +96,21 @@
       end
     end
   end
+
+  describe '#placeholder_content' do
+    subject { presenter.placeholder_content }
+
+    it 'has format' do
+      expect(subject.format).to eq "image/jpeg"
+    end
+
+    it 'has type' do
+      expect(subject.type).to eq "Image"
+    end
+
+    it 'has height and width' do
+      expect(subject.width).to eq 1280
+      expect(subject.height).to eq 720
+    end
+  end
 end

From a9c0160f07fd8f1966baf1b4104b28c61d855050 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Mon, 15 May 2023 16:17:05 -0400
Subject: [PATCH 040/396] Ensure input is a uri and not a path when using
 multiple pre-transcoded derivatives

---
 app/models/master_file.rb             |  2 +-
 spec/models/master_file_spec.rb       | 39 +++++++++++++++++++++++++++
 spec/models/working_file_path_spec.rb |  4 +--
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index d76ae81e2e..e0cf0c9f52 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -263,7 +263,7 @@ def process_pass_through(file)
     input = nil
     # Options hash: { outputs: [{ label: 'low',  url: 'file:///derivatives/low.mp4' }, { label: 'high', url: 'file:///derivatives/high.mp4' }]}
     if file.is_a? Hash
-      input = file.sort_by { |f| QUALITY_ORDER[f[0]] }.last[1].path
+      input = FileLocator.new(file.sort_by { |f| QUALITY_ORDER[f[0]] }.last[1].to_path).uri.to_s
       options[:outputs] = file.collect { |quality, f| { label: quality.remove("quality-"), url: FileLocator.new(f.to_path).uri.to_s } }
     else
       #Build hash for single file skip transcoding
diff --git a/spec/models/master_file_spec.rb b/spec/models/master_file_spec.rb
index fd9b236467..f3e5c88208 100644
--- a/spec/models/master_file_spec.rb
+++ b/spec/models/master_file_spec.rb
@@ -131,6 +131,45 @@
         expect{ master_file.process }.to raise_error(RuntimeError)
       end
     end
+
+    context 'pass through' do
+      let(:master_file) { FactoryBot.create(:master_file, :not_processing, workflow_name: 'pass_through') }
+
+      context 'with multiple files' do
+	let(:low_file) { "spec/fixtures/videoshort.low.mp4" }
+	let(:medium_file) { "spec/fixtures/videoshort.medium.mp4" }
+	let(:high_file) { "spec/fixtures/videoshort.high.mp4" }
+        let(:outputs_hash) do
+          [
+            { label: 'low', url: FileLocator.new(low_file).uri.to_s },
+            { label: 'medium', url: FileLocator.new(medium_file).uri.to_s },
+            { label: 'high', url: FileLocator.new(high_file).uri.to_s }
+          ]
+        end
+        let(:files) do
+          {
+            "quality-low" => FileLocator.new(low_file).attachment,
+            "quality-medium" => FileLocator.new(medium_file).attachment,
+            "quality-high" => FileLocator.new(high_file).attachment
+          }
+        end
+
+        it 'creates an encode' do
+	  expect(master_file.encoder_class).to receive(:create).with("file://" + Rails.root.join(high_file).to_path, { outputs: outputs_hash, master_file_id: master_file.id, preset: master_file.workflow_name })
+	  master_file.process(files)
+        end
+      end
+
+      context 'with single file' do
+        let(:input_url) { FileLocator.new(master_file.file_location).uri.to_s }
+        let(:outputs_hash) { [{ label: 'high', url: input_url }] }
+
+        it 'creates an encode' do
+          expect(master_file.encoder_class).to receive(:create).with(input_url, { outputs: outputs_hash, master_file_id: master_file.id, preset: master_file.workflow_name })
+	  master_file.process
+        end
+      end
+    end
   end
 
   describe "delete" do
diff --git a/spec/models/working_file_path_spec.rb b/spec/models/working_file_path_spec.rb
index 096dc32862..815f5880ba 100644
--- a/spec/models/working_file_path_spec.rb
+++ b/spec/models/working_file_path_spec.rb
@@ -202,7 +202,7 @@
 
           input_path = FileLocator.new(working_file_path_high).uri.to_s
           expect(encoder_class).to have_received(:create).with(
-            input_path.remove("file://"), 
+            input_path, 
             master_file_id: master_file.id, 
             outputs: [{ label: "low", url: FileLocator.new(working_file_path_low).uri.to_s }, 
                       { label: "medium", url: FileLocator.new(working_file_path_medium).uri.to_s },
@@ -318,7 +318,7 @@
 
           input_path = Addressable::URI.convert_path(File.absolute_path(filename_high)).to_s
           expect(encoder_class).to have_received(:create).with(
-            File.absolute_path(filename_high),
+            FileLocator.new(filename_high).uri.to_s,
             master_file_id: master_file.id,
             outputs: [{ label: "low", url: Addressable::URI.convert_path(File.absolute_path(filename_low)).to_s },
                       { label: "medium", url: Addressable::URI.convert_path(File.absolute_path(filename_medium)).to_s },

From d732feb2b58cd1e58425457117060bc3f7e91ee1 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 10 May 2023 17:01:41 -0400
Subject: [PATCH 041/396] Avoid enqueueing many copies of the media object
 indexing job unnecessarily

---
 app/jobs/media_object_indexing_job.rb | 5 +++++
 spec/models/master_file_spec.rb       | 1 +
 2 files changed, 6 insertions(+)

diff --git a/app/jobs/media_object_indexing_job.rb b/app/jobs/media_object_indexing_job.rb
index 0de943a5e6..1fad07a407 100644
--- a/app/jobs/media_object_indexing_job.rb
+++ b/app/jobs/media_object_indexing_job.rb
@@ -15,6 +15,11 @@
 class MediaObjectIndexingJob < ApplicationJob
   queue_as :media_object_indexing
 
+  # Only enqueue one indexing job at a time but once one starts running allow others to be enqueued
+  # because the first step of the job is fetching the object from fedora at which point it might
+  # be stale and subsequent jobs might get a newer version of the media object.
+  unique :until_executing, on_conflict: :log
+
   def perform(id)
     mo = MediaObject.find(id)
     ActiveFedora::SolrService.add(mo.to_solr(include_child_fields: true), softCommit: true)
diff --git a/spec/models/master_file_spec.rb b/spec/models/master_file_spec.rb
index f3e5c88208..9cce147523 100644
--- a/spec/models/master_file_spec.rb
+++ b/spec/models/master_file_spec.rb
@@ -883,6 +883,7 @@
       # Force creation of master_file and then clear queue of byproduct jobs
       master_file
       ActiveJob::Base.queue_adapter.enqueued_jobs.clear
+      ActiveJob::Uniqueness.unlock!
     end
 
     it 'enqueues indexing of parent media object' do

From c6dd02ffd5868544eb6c8903bac409b0a25c9a0f Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 17 May 2023 13:49:58 -0400
Subject: [PATCH 042/396] Add supplemental files to iiif manifest and canvases

---
 .../iiif_supplemental_file_behavior.rb        | 37 +++++++++++++++++++
 .../concerns/supplemental_file_behavior.rb    | 12 +++++-
 app/models/iiif_canvas_presenter.rb           | 20 +++++++---
 app/models/iiif_manifest_presenter.rb         |  6 +++
 app/presenters/speedy_af/proxy/master_file.rb | 12 +++++-
 .../speedy_af/proxy/media_object.rb           | 14 +++++++
 app/views/media_objects/_item_view.html.erb   |  2 +-
 .../_supplemental_files_list.html.erb         |  2 +-
 spec/models/iiif_canvas_presenter_spec.rb     | 24 ++++++++++++
 spec/models/iiif_manifest_presenter_spec.rb   | 13 +++++++
 .../supplemental_file_shared_examples.rb      | 20 +++++++++-
 11 files changed, 148 insertions(+), 14 deletions(-)
 create mode 100644 app/models/concerns/iiif_supplemental_file_behavior.rb

diff --git a/app/models/concerns/iiif_supplemental_file_behavior.rb b/app/models/concerns/iiif_supplemental_file_behavior.rb
new file mode 100644
index 0000000000..187054aece
--- /dev/null
+++ b/app/models/concerns/iiif_supplemental_file_behavior.rb
@@ -0,0 +1,37 @@
+module IiifSupplementalFileBehavior
+  private
+
+  def supplemental_files_rendering(object)
+    object.supplemental_files(tag: nil).collect do |sf|
+      {
+        "@id" => object_supplemental_file_url(object, sf),
+        "type" => determine_rendering_type(sf.file.content_type),
+        "label" => { "en" => [sf.label] },
+        "format" => sf.file.content_type
+      }
+    end
+  end
+
+  def object_supplemental_file_url(object, supplemental_file)
+    if object.is_a? MasterFile
+      Rails.application.routes.url_helpers.master_file_supplemental_file_url(id: supplemental_file.id, master_file_id: object.id)
+    else
+      Rails.application.routes.url_helpers.media_object_supplemental_file_url(id: supplemental_file.id, media_object_id: object.id)
+    end
+  end
+
+  def determine_rendering_type(mime)
+    case mime
+    when 'application/pdf', 'application/msword', 'application/vnd.oasis.opendocument.text', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'text/html', 'text/plain', 'text/vtt'
+      'Text'
+    when 'image/bmp', 'image/gif', 'image/jpg', 'image/png', 'image/svg+xml', 'image/tiff', 'image/webp'
+      'Image'
+    when 'audio/aac', 'audio/midi', 'audio/mpeg', 'audio/ogg', 'audio/wav', 'audio/webm'
+      'Audio'
+    when 'video/mp4', 'video/mpeg', 'video/ogg', 'video/webm', 'video/x-msvideo'
+      'Video'
+    else
+      'Dataset'
+    end
+  end
+end
diff --git a/app/models/concerns/supplemental_file_behavior.rb b/app/models/concerns/supplemental_file_behavior.rb
index 4d9b6e71d0..d53a9f34da 100644
--- a/app/models/concerns/supplemental_file_behavior.rb
+++ b/app/models/concerns/supplemental_file_behavior.rb
@@ -27,9 +27,17 @@ module SupplementalFileBehavior
   # SupplementalFile = Struct.new(:id, :label, :absolute_path, keyword_init: true)
 
   # @return [SupplementalFile]
-  def supplemental_files
+  def supplemental_files(tag: '*')
     return [] if supplemental_files_json.blank?
-    JSON.parse(supplemental_files_json).collect { |file_gid| GlobalID::Locator.locate(file_gid) }
+    files = JSON.parse(supplemental_files_json).collect { |file_gid| GlobalID::Locator.locate(file_gid) }
+    case tag
+    when '*'
+      files
+    when nil
+      files.select { |file| file.tags.empty? }
+    else
+      files.select { |file| Array(tag).all? { |t| file.tags.include?(t) } }
+    end
   end
 
   # @param files [SupplementalFile]
diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 0620dbd10a..95bdf1d9a9 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -13,6 +13,8 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class IiifCanvasPresenter
+  include IiifSupplementalFileBehavior
+
   attr_reader :master_file, :stream_info
   attr_accessor :media_fragment
 
@@ -38,12 +40,7 @@ def display_content
   end
 
   def sequence_rendering
-    [{
-      "@id" => "#{@master_file.waveform_master_file_url(@master_file.id)}.json",
-      "type" => "Dataset",
-      "label" => { "en" => ["waveform.json"] },
-      "format" => "application/json"
-    }]
+    supplemental_files_rendering(master_file) + waveform_rendering
   end
 
   def placeholder_content
@@ -82,6 +79,17 @@ def stream_urls
       end
     end
 
+    def waveform_rendering
+      [
+        {
+          "@id" => "#{@master_file.waveform_master_file_url(@master_file.id)}.json",
+          "type" => "Dataset",
+          "label" => { "en" => ["waveform.json"] },
+          "format" => "application/json"
+        }
+      ]
+    end
+
     def simple_iiif_range(label = stream_info[:embed_title])
       # TODO: embed_title?
       IiifManifestRange.new(
diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 6f29ac8a96..8516c136c1 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -13,6 +13,8 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class IiifManifestPresenter
+  include IiifSupplementalFileBehavior
+
   # Should 
     tags be allowed?
   # They aren't listed in the IIIF spec but we use them in our normal view page
   IIIF_ALLOWED_TAGS = ['a', 'b', 'br', 'i', 'img', 'p', 'small', 'span', 'sub', 'sup'].freeze
@@ -74,6 +76,10 @@ def homepage
     ]
   end
 
+  def sequence_rendering
+    supplemental_files_rendering(media_object)
+  end
+
   private
 
   def sanitize(value)
diff --git a/app/presenters/speedy_af/proxy/master_file.rb b/app/presenters/speedy_af/proxy/master_file.rb
index 386336953d..56740e662d 100644
--- a/app/presenters/speedy_af/proxy/master_file.rb
+++ b/app/presenters/speedy_af/proxy/master_file.rb
@@ -39,8 +39,16 @@ def display_title
   end
 
   # @return [SupplementalFile]
-  def supplemental_files
+  def supplemental_files(tag: '*')
     return [] if supplemental_files_json.blank?
-    JSON.parse(supplemental_files_json).collect { |file_gid| GlobalID::Locator.locate(file_gid) }
+    files = JSON.parse(supplemental_files_json).collect { |file_gid| GlobalID::Locator.locate(file_gid) }
+    case tag
+    when '*'
+      files
+    when nil
+      files.select { |file| file.tags.empty? }
+    else
+      files.select { |file| Array(tag).all? { |t| file.tags.include?(t) } }
+    end
   end
 end
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index 614152a74b..ac8e540879 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -28,4 +28,18 @@ def model_name
   def to_param
     id
   end
+
+  # @return [SupplementalFile]
+  def supplemental_files(tag: '*')
+    return [] if supplemental_files_json.blank?
+    files = JSON.parse(supplemental_files_json).collect { |file_gid| GlobalID::Locator.locate(file_gid) }
+    case tag
+    when '*'
+      files
+    when nil
+      files.select { |file| file.tags.empty? }
+    else
+      files.select { |file| Array(tag).all? { |t| file.tags.include?(t) } }
+    end
+  end
 end
diff --git a/app/views/media_objects/_item_view.html.erb b/app/views/media_objects/_item_view.html.erb
index 9f15f70b3e..4345c0136a 100644
--- a/app/views/media_objects/_item_view.html.erb
+++ b/app/views/media_objects/_item_view.html.erb
@@ -54,7 +54,7 @@ Unless required by applicable law or agreed to in writing, software distributed
     
 
     <% if can_stream && @currentStream %>
-      <% supplemental_files = @masterFiles.map { |mf| {"id" => mf.id, "transcripts" => mf.supplemental_files.select { |sf| sf.tags.include?('transcript')} } }.flatten(1) %>
+      <% supplemental_files = @masterFiles.map { |mf| {"id" => mf.id, "transcripts" => mf.supplemental_files(tag: 'transcript') } }.flatten(1) %>
       <%= react_component("ReactIIIFTranscript", {base_url: request.protocol+request.host_with_port, transcripts: supplemental_files }) %>
     <% end %>
   
diff --git a/app/views/media_objects/_supplemental_files_list.html.erb b/app/views/media_objects/_supplemental_files_list.html.erb
index ed9fb91335..5ddbea3af0 100644
--- a/app/views/media_objects/_supplemental_files_list.html.erb
+++ b/app/views/media_objects/_supplemental_files_list.html.erb
@@ -14,7 +14,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 ---  END LICENSE_HEADER BLOCK  ---
 %>
           <% if section.supplemental_files.present? %>
-            <% files=tag.empty? ? supplemental_files.select { |sf| sf.tags.empty? } : supplemental_files.select { |sf| sf.tags.include?(tag) } %>
+            <% files=tag.empty? ? supplemental_files(tag: nil) : supplemental_files(tag: tag) %>
             
    
               <% files.each do |file| %>
                 
    
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 2cdff6b88c..adb90a0d4d 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -113,4 +113,28 @@
       expect(subject.height).to eq 720
     end
   end
+
+  describe '#sequence_rendering' do
+    let(:master_file) { FactoryBot.build(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
+    let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
+    let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, tags: ['caption', 'machine_generated']) }
+    let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
+    let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
+
+    subject { presenter.sequence_rendering }
+
+    it 'includes waveform' do
+      expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq true
+    end
+
+    it 'includes supplemental files' do
+      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq true
+    end
+
+    it 'does not include transcripts or captions' do
+      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{transcript_file.id}/ }).to eq false
+      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{caption_file.id}/ }).to eq false
+    end
+  end
 end
diff --git a/spec/models/iiif_manifest_presenter_spec.rb b/spec/models/iiif_manifest_presenter_spec.rb
index 7c20144a21..882ab2f18b 100644
--- a/spec/models/iiif_manifest_presenter_spec.rb
+++ b/spec/models/iiif_manifest_presenter_spec.rb
@@ -62,4 +62,17 @@
       end
     end
   end
+
+  describe '#sequence_rendering' do
+    let(:media_object) { FactoryBot.build(:fully_searchable_media_object, supplemental_files_json: supplemental_files_json) }
+    let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
+    let(:supplemental_files) { [supplemental_file] }
+    let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
+
+    subject { presenter.sequence_rendering }
+
+    it 'includes supplemental files' do
+      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq true
+    end
+  end
 end
diff --git a/spec/support/supplemental_file_shared_examples.rb b/spec/support/supplemental_file_shared_examples.rb
index 2eda80fb57..918ff3f70e 100644
--- a/spec/support/supplemental_file_shared_examples.rb
+++ b/spec/support/supplemental_file_shared_examples.rb
@@ -15,8 +15,10 @@
 RSpec.shared_examples 'an object that has supplemental files' do
   let(:object) { FactoryBot.build(described_class.model_name.singular.to_sym) }
   let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
-  let(:supplemental_files) { [supplemental_file] }
-  let(:supplemental_files_json) { [supplemental_file.to_global_id.to_s].to_json }
+  let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
+  let(:caption_file) { FactoryBot.create(:supplemental_file, tags: ['caption', 'machine_generated']) }
+  let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
+  let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
 
   context 'supplemental_files=' do
     it 'stores the supplemental files as a json string in a property' do
@@ -30,5 +32,19 @@
     it 'reifies the supplemental files from the stored json string' do
       expect(object.supplemental_files).to eq supplemental_files
     end
+
+    context 'filtering by tag' do
+      it 'returns only files with no tags' do
+        expect(object.supplemental_files(tag: nil)).to eq [supplemental_file]
+      end
+
+      it 'return only files with specified tag' do
+        expect(object.supplemental_files(tag: 'transcript')).to eq [transcript_file]
+      end
+
+      it 'returns only files with multiple specified tags' do
+        expect(object.supplemental_files(tag: ['caption', 'machine_generated'])).to eq [caption_file]
+      end
+    end
   end
 end

From 6ce7189a5c631f20eab795aa757644a15e055552 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 19 May 2023 11:48:40 -0400
Subject: [PATCH 043/396] Fix reference to supplemental_files in view

---
 app/views/media_objects/_supplemental_files_list.html.erb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/views/media_objects/_supplemental_files_list.html.erb b/app/views/media_objects/_supplemental_files_list.html.erb
index 5ddbea3af0..0be6dc8909 100644
--- a/app/views/media_objects/_supplemental_files_list.html.erb
+++ b/app/views/media_objects/_supplemental_files_list.html.erb
@@ -13,8 +13,8 @@ Unless required by applicable law or agreed to in writing, software distributed
   specific language governing permissions and limitations under the License.
 ---  END LICENSE_HEADER BLOCK  ---
 %>
-          <% if section.supplemental_files.present? %>
-            <% files=tag.empty? ? supplemental_files(tag: nil) : supplemental_files(tag: tag) %>
+          <% if section.supplemental_files_json.present? %>
+            <% files=tag.empty? ? section.supplemental_files(tag: nil) : section.supplemental_files(tag: tag) %>
             
    
               <% files.each do |file| %>
                 
    

From a6f3f11bd737b97212164948924e4538a56414f4 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 19 May 2023 13:23:48 -0400
Subject: [PATCH 044/396] Handle case where permalink is empty string

---
 app/models/iiif_manifest_presenter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 8516c136c1..7b648e745a 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -68,7 +68,7 @@ def ranges
   def homepage
     [
       {
-        id: media_object.permalink || Rails.application.routes.url_helpers.media_object_url(media_object),
+        id: media_object.permalink.presence || Rails.application.routes.url_helpers.media_object_url(media_object),
         type: "Text",
         label: { "none" => [I18n.t('iiif.manifest.homepageLabel')] },
         format: "text/html"

From eb19ece8eb5dce8aed5996b23c7066209765f9cc Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 19 May 2023 13:28:59 -0400
Subject: [PATCH 045/396] Use regexes to include more file types

---
 app/models/concerns/iiif_supplemental_file_behavior.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/models/concerns/iiif_supplemental_file_behavior.rb b/app/models/concerns/iiif_supplemental_file_behavior.rb
index 187054aece..e34d6e0b0e 100644
--- a/app/models/concerns/iiif_supplemental_file_behavior.rb
+++ b/app/models/concerns/iiif_supplemental_file_behavior.rb
@@ -24,11 +24,11 @@ def determine_rendering_type(mime)
     case mime
     when 'application/pdf', 'application/msword', 'application/vnd.oasis.opendocument.text', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'text/html', 'text/plain', 'text/vtt'
       'Text'
-    when 'image/bmp', 'image/gif', 'image/jpg', 'image/png', 'image/svg+xml', 'image/tiff', 'image/webp'
+    when /image\/.+/
       'Image'
-    when 'audio/aac', 'audio/midi', 'audio/mpeg', 'audio/ogg', 'audio/wav', 'audio/webm'
+    when /audio\/.+/
       'Audio'
-    when 'video/mp4', 'video/mpeg', 'video/ogg', 'video/webm', 'video/x-msvideo'
+    when /video\/.+/
       'Video'
     else
       'Dataset'

From 2f64947292c287dea5512ac250c626bbac2c2502 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 19 May 2023 14:03:50 -0400
Subject: [PATCH 046/396] Original names can be nil so use safe navigator

Found this when attempting to reindex in production.
---
 app/models/indexed_file.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/indexed_file.rb b/app/models/indexed_file.rb
index f6e3855801..8bc23bd106 100644
--- a/app/models/indexed_file.rb
+++ b/app/models/indexed_file.rb
@@ -17,6 +17,6 @@ class IndexedFile < ActiveFedora::File
 
   # Override
   def original_name
-    super.force_encoding("UTF-8")
+    super&.force_encoding("UTF-8")
   end
 end

From b7e40ee49e4d8fb1c5c6e05842503e12806dd5a9 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 19 May 2023 14:57:37 -0400
Subject: [PATCH 047/396] Update catalog_controller.rb

---
 app/controllers/catalog_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb
index 19c1929f84..47297f6f53 100644
--- a/app/controllers/catalog_controller.rb
+++ b/app/controllers/catalog_controller.rb
@@ -92,7 +92,7 @@ class CatalogController < ApplicationController
     config.add_facet_field 'avalon_uploader_ssi', label: 'Created by', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow"
     config.add_facet_field 'read_access_group_ssim', label: 'Item access', if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow", query: {
       public: { label: "Public", fq: "has_model_ssim:MediaObject AND read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC}" },
-      restricted: { label: "Logged in", fq: "has_model_ssim:MediaObject AND read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED}" },
+      restricted: { label: "Authenticated", fq: "has_model_ssim:MediaObject AND read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED}" },
       private: { label: "Private", fq: "has_model_ssim:MediaObject AND NOT read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC} AND NOT read_access_group_ssim:#{Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED}" }
     }
     config.add_facet_field 'read_access_virtual_group_ssim', label: 'External Group', limit: 5, if: Proc.new {|context, config, opts| Ability.new(context.current_user, context.user_session).can? :create, MediaObject}, group: "workflow", helper_method: :vgroup_display

From 57ad662bfd29776201c31c6378d48c01b2d74f3b Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 18 May 2023 16:22:55 -0400
Subject: [PATCH 048/396] Refactor supplemental file model and upload page

---
 app/models/supplemental_file.rb                 |  6 ++++--
 app/views/media_objects/_file_upload.html.erb   | 17 +----------------
 config/settings.yml                             |  4 ++--
 ...142233_add_language_to_supplemental_files.rb |  5 +++++
 4 files changed, 12 insertions(+), 20 deletions(-)
 create mode 100644 db/migrate/20230516142233_add_language_to_supplemental_files.rb

diff --git a/app/models/supplemental_file.rb b/app/models/supplemental_file.rb
index 2b09f0e139..3ee03c2d03 100644
--- a/app/models/supplemental_file.rb
+++ b/app/models/supplemental_file.rb
@@ -17,15 +17,17 @@ class SupplementalFile < ApplicationRecord
 
   # TODO: the empty tag should represent a generic supplemental file
   validates :tags, array_inclusion: ['transcript', 'caption', 'machine_generated', '', nil]
+  validates :language, inclusion: { in: LanguageTerm.map.keys }
 
   serialize :tags, Array
 
   def attach_file(new_file)
     file.attach(new_file)
     self.label = file.filename.to_s if label.blank?
+    self.language = tags.include?('caption') ? Settings.caption_default.language : 'eng'
   end
 
-  def machine_generated?
-    tags.include?('machine_generated')
+  def mime_type
+    self.file.content_type
   end
 end
diff --git a/app/views/media_objects/_file_upload.html.erb b/app/views/media_objects/_file_upload.html.erb
index 2cff27d691..b011eb0645 100644
--- a/app/views/media_objects/_file_upload.html.erb
+++ b/app/views/media_objects/_file_upload.html.erb
@@ -155,22 +155,7 @@ Unless required by applicable law or agreed to in writing, software distributed
                 <% end %>
                 
    
                    
-                   
    
-                    Captions
-                    <%= form_with model: section, :url => attach_captions_master_file_path(section.id), html: {method: "post", class: "caption-file-form" } do |form| %>
-                      <%= form.file_field :captions, class: "filedata" %>
-                      
-                      <% if section.captions.present? %>
-                        <%= link_to("Remove", delete_captions_master_file_path(section.id), title: 'Remove', method: :delete, class: "btn btn-danger btn-sm btn-confirmation") %>
-                      <% end %>
-                    <% end %>
-                    <% if section.captions.present? %>
-                      
    
-                        
    Uploaded file: <%= section.captions.original_name %>
    
-                      
    
-                    <% end %>
-                  
    
+                   <%= render partial: "supplemental_files_upload", locals: { section: section, label: 'Captions', tag: 'caption' } %>
                   
                   <%= render partial: "supplemental_files_upload", locals: { section: section, label: 'Transcripts', tag: 'transcript' } %>
                   
diff --git a/config/settings.yml b/config/settings.yml
index dfee409912..1b0fd6e7bd 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -101,8 +101,8 @@ controlled_digital_lending:
   collections_enabled: false
   default_lending_period: 'P14D' # ISO8601 duration format: P14D == 14.days, PT8H == 8.hours, etc.
 caption_default:
-  # Language should be 2 or 3 letter ISO 639 codes
-  language: 'en'
+  # Language should be 3 letter ISO 639-2 code
+  language: 'eng'
   name: 'English'
 recaptcha:
   site_key: # Setting a site_key will enable recaptcha on the comments form
diff --git a/db/migrate/20230516142233_add_language_to_supplemental_files.rb b/db/migrate/20230516142233_add_language_to_supplemental_files.rb
new file mode 100644
index 0000000000..39926d64b0
--- /dev/null
+++ b/db/migrate/20230516142233_add_language_to_supplemental_files.rb
@@ -0,0 +1,5 @@
+class AddLanguageToSupplementalFiles < ActiveRecord::Migration[7.0]
+  def change
+    add_column :supplemental_files, :language, :string
+  end
+end

From 1b49caee8b8d8b4134a22a8c04139b01b5db056e Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 18 May 2023 16:24:36 -0400
Subject: [PATCH 049/396] Update master files caption handling

---
 app/controllers/master_files_controller.rb  | 47 ---------------------
 app/models/concerns/master_file_behavior.rb | 33 ++++++++++++---
 app/models/concerns/master_file_intercom.rb |  9 ++--
 app/models/master_file.rb                   | 17 ++++----
 4 files changed, 40 insertions(+), 66 deletions(-)

diff --git a/app/controllers/master_files_controller.rb b/app/controllers/master_files_controller.rb
index cbb9756a4a..77d508654b 100644
--- a/app/controllers/master_files_controller.rb
+++ b/app/controllers/master_files_controller.rb
@@ -149,53 +149,6 @@ def attach_structure
     end
   end
 
-  def attach_captions
-    captions = nil
-    if flash.empty?
-      authorize! :edit, @master_file, message: "You do not have sufficient privileges to add files"
-      if params[:master_file].present? && params[:master_file][:captions].present?
-        captions_file = params[:master_file][:captions]
-        captions_ext = File.extname(captions_file.original_filename)
-        content_type = Mime::Type.lookup_by_extension(captions_ext.slice(1..-1)).to_s if captions_ext
-        if ["text/vtt", "text/srt"].include? content_type
-          captions = captions_file.open.read
-        else
-          flash[:error] = "Uploaded file is not a recognized captions file"
-        end
-      end
-      if captions.present?
-        @master_file.captions.content = captions.encode(Encoding.find('UTF-8'), invalid: :replace, undef: :replace, replace: '')
-        @master_file.captions.mime_type = content_type
-        @master_file.captions.original_name = params[:master_file][:captions].original_filename
-        flash[:success] = "Captions file succesfully added."
-      end
-      if flash[:error].blank?
-        unless @master_file.save
-          flash[:success] = nil
-          flash[:error] = "There was a problem storing the file"
-        end
-      end
-    end
-    respond_to do |format|
-      format.html { redirect_to edit_media_object_path(@master_file.media_object_id, step: 'file-upload') }
-      format.json { render json: {captions: captions, flash: flash} }
-    end
-  end
-
-  def delete_captions
-    authorize! :edit, @master_file, message: "You do not have sufficient privileges to remove files"
-
-    @master_file.captions.content = ''
-    @master_file.captions.original_name = ''
-
-    if @master_file.save
-      flash[:success] = "Captions file succesfully removed."
-    else
-      flash[:error] = "There was a problem removing captions file."
-    end
-    redirect_to edit_media_object_path(@master_file.media_object_id, step: 'file-upload')
-  end
-
   # Creates and Saves a File Asset to contain the the Uploaded file
   # If container_id is provided:
   # * the File Asset will use RELS-EXT to assert that it's a part of the specified container
diff --git a/app/models/concerns/master_file_behavior.rb b/app/models/concerns/master_file_behavior.rb
index a6c87fdde2..ef6f7a0676 100644
--- a/app/models/concerns/master_file_behavior.rb
+++ b/app/models/concerns/master_file_behavior.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -57,8 +57,14 @@ def stream_details
 
     poster_path = Rails.application.routes.url_helpers.poster_master_file_path(self)
     if has_captions?
-      captions_path = Rails.application.routes.url_helpers.captions_master_file_path(self)
-      captions_format = self.captions.mime_type
+      caption_paths = []
+      self.supplemental_file_captions.each { |c| caption_paths.append(build_caption_hash(c)) }
+
+      if self.captions
+        caption_paths.append(build_caption_hash(self.captions))
+      end
+
+      caption_paths
     end
 
     # Returns the hash
@@ -71,8 +77,7 @@ def stream_details
       stream_flash: flash,
       stream_hls: hls,
       cookie_auth: cookie_auth?,
-      captions_path: captions_path,
-      captions_format: captions_format,
+      caption_paths: caption_paths,
       duration: (duration.to_f / 1000),
       embed_title: embed_title
     })
@@ -141,4 +146,18 @@ def cookie_auth?
   def sort_streams array
     array.sort { |x, y| QUALITY_ORDER[x[:quality]] <=> QUALITY_ORDER[y[:quality]] }
   end
+
+  def build_caption_hash(caption)
+    path = if caption.kind_of?(IndexedFile)
+             Rails.application.routes.url_helpers.captions_master_file_path(self)
+           elsif caption.kind_of?(SupplementalFile)
+             Rails.application.routes.url_helpers.master_file_supplemental_file_path(master_file_id: self.id, id: caption.id)
+           end
+    {
+      path: path,
+      mime_type: caption.mime_type,
+      language: caption.kind_of?(SupplementalFile) ? caption.language : "en",
+      label: caption.kind_of?(SupplementalFile) ? caption.label : nil
+    }
+  end
 end
diff --git a/app/models/concerns/master_file_intercom.rb b/app/models/concerns/master_file_intercom.rb
index 28310bea8f..a0e261c019 100644
--- a/app/models/concerns/master_file_intercom.rb
+++ b/app/models/concerns/master_file_intercom.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -33,8 +33,9 @@ def to_ingest_api_hash(include_structure = true, remove_identifiers: false)
       file_checksum: file_checksum,
       file_format: file_format,
       other_identifier: (remove_identifiers ? [] : identifier.to_a),
-      captions: captions.content,
+      captions: captions,
       captions_type: caption_type,
+      supplemental_file_captions: supplemental_file_captions,
       comment: comment.to_a,
       display_aspect_ratio: display_aspect_ratio,
       original_frame_size: original_frame_size,
diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index e0cf0c9f52..2c44051307 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -477,6 +477,12 @@ def self.post_processing_move_filename(oldpath, options = {})
     end
   end
 
+  attr_writer :captions
+
+  def supplemental_file_captions
+    self.supplemental_files.select { |sf| sf.tags.include? 'caption' }
+  end
+
   def has_audio?
     # The MasterFile doesn't have an audio track unless the first derivative does
     # This is useful to skip unnecessary waveform generation
@@ -495,10 +501,6 @@ def has_captions?
     !captions.empty?
   end
 
-  def caption_type
-    has_captions? ? captions.mime_type : nil
-  end
-
   def has_waveform?
     !waveform.empty?
   end
@@ -519,7 +521,6 @@ def to_solr *args
       solr_doc['has_poster?_bs'] = has_poster?
       solr_doc['has_thumbnail?_bs'] = has_thumbnail?
       solr_doc['has_structuralMetadata?_bs'] = has_structuralMetadata?
-      solr_doc['caption_type_ss'] = caption_type
       solr_doc['identifier_ssim'] = identifier.map(&:downcase)
 
       solr_doc['percent_complete_ssi'] = percent_complete

From ab5a93a4a041b26ead157e97c1b92520ffa65833 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 18 May 2023 16:26:30 -0400
Subject: [PATCH 050/396] Update javascript and player to handle multiple
 captions

---
 .../media_player_wrapper/avalon_player_new.es6 | 18 +++++++++---------
 .../mejs4_helper_utility.es6                   |  6 ++++--
 .../modules/player/_video_element.html.erb     |  6 ++++--
 3 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6 b/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6
index 5615aa5a9e..7462fcf29c 100644
--- a/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6
+++ b/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6
@@ -265,10 +265,10 @@ class MEJSPlayer {
       }" type="application/x-mpegURL" data-quality="${source.quality}"/>`;
     });
 
-    if (this.currentStreamInfo.captions_path) {
-      markup += ``;
+    if (this.currentStreamInfo.caption_paths) {
+      for (c in this.currentStreamInfo.caption_paths) {
+        markup += ``;
+      }
     }
 
     this.node.innerHTML = markup;
@@ -301,7 +301,7 @@ class MEJSPlayer {
       // Set current source in absence of the quality selection
       let currentSource = this.currentStreamInfo.stream_hls
                         .filter(src => src.quality === this.player.options.defaultQuality)[0];
-     
+
       this.player.setSrc(currentSource.url);
     }
 
@@ -322,7 +322,7 @@ class MEJSPlayer {
    * @function reInitializeCaptions
    */
   reInitializeCaptions() {
-    if (this.currentStreamInfo.captions_path) {
+    if (this.currentStreamInfo.caption_paths) {
       // Place tracks button
       if(this.mejsUtility.isMobile()) {
         // after trackScrubber button in mobile devices
@@ -331,7 +331,7 @@ class MEJSPlayer {
         // after volume button in desktop devices
         this.player.featurePosition.tracks = this.player.featurePosition.volume + 1;
       }
-      
+
       this.player.buildtracks(this.player, null, this.player.layers, this.mediaElement);
       // Turn on captions
       this.toggleCaptions();
@@ -603,12 +603,12 @@ class MEJSPlayer {
    */
   initializePlayer() {
     let currentStreamInfo = this.currentStreamInfo;
-    // Set default quality value in localStorage  
+    // Set default quality value in localStorage
     this.localStorage.setItem('quality', this.defaultQuality);
     // Interval in seconds to jump forward and backward in media
     let jumpInterval = 5;
 
-    // Set default quality value in localStorage  
+    // Set default quality value in localStorage
     this.localStorage.setItem('quality', this.defaultQuality);
 
     // Mediaelement default root level configuration
diff --git a/app/assets/javascripts/media_player_wrapper/mejs4_helper_utility.es6 b/app/assets/javascripts/media_player_wrapper/mejs4_helper_utility.es6
index dfba22c634..e9f4e1c37c 100644
--- a/app/assets/javascripts/media_player_wrapper/mejs4_helper_utility.es6
+++ b/app/assets/javascripts/media_player_wrapper/mejs4_helper_utility.es6
@@ -46,8 +46,10 @@ class MEJSUtility {
       });
 
       // Add captions
-      if (currentStreamInfo.captions_path) {
-        markup += ``;
+      if (currentStreamInfo.caption_paths) {
+        for (c in currentStreamInfo.caption_paths) {
+          markup += ``;
+        }
       }
     }
     // Create 
    
                     <% end %>
                     

From 5f86c148fc4940b3157c0e083897de5c3c289636 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Mon, 22 May 2023 16:08:44 -0400
Subject: [PATCH 053/396] Use new supplemental file method and update tests

---
 app/models/master_file.rb           | 2 +-
 spec/factories/supplemental_file.rb | 4 ++++
 spec/models/master_file_spec.rb     | 7 ++++++-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index c66af37493..55846a87ba 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -480,7 +480,7 @@ def self.post_processing_move_filename(oldpath, options = {})
   attr_writer :supplemental_file_captions
 
   def supplemental_file_captions
-    self.supplemental_files.select { |sf| sf.tags.include? 'caption' }
+    self.supplemental_files(tag: 'caption')
   end
 
   def has_audio?
diff --git a/spec/factories/supplemental_file.rb b/spec/factories/supplemental_file.rb
index db619c1a60..4652799aac 100644
--- a/spec/factories/supplemental_file.rb
+++ b/spec/factories/supplemental_file.rb
@@ -28,5 +28,9 @@
     trait :with_transcript_tag do
       tags { ['transcript'] }
     end
+
+    trait :with_caption_tag do
+      tags { ['caption'] }
+    end
   end
 end
diff --git a/spec/models/master_file_spec.rb b/spec/models/master_file_spec.rb
index 631c075497..ec9b4a42f6 100644
--- a/spec/models/master_file_spec.rb
+++ b/spec/models/master_file_spec.rb
@@ -722,9 +722,14 @@
   end
 
   describe 'supplemental_file_captions' do
-    let(:master_file) { FactoryBot.create(:master_file) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_tag) }
+    let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
+    let(:master_file) { FactoryBot.create(:master_file, supplemental_files: [caption_file, transcript_file]) }
     it 'has a caption' do
+      expect(master_file.supplemental_file_captions).to_not be_empty
       expect(master_file.supplemental_file_captions).to all(be_kind_of(SupplementalFile))
+      expect(master_file.supplemental_file_captions).to_not include(transcript_file)
+      expect(master_file.supplemental_file_captions).to include(caption_file)
     end
   end
 

From 6ccc6ec41e19d6ade751f9e7dddc3f8ee09aba70 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Mon, 22 May 2023 16:23:05 -0400
Subject: [PATCH 054/396] Remove routes for MasterFIle Attach and Delete
 captions

---
 config/routes.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/config/routes.rb b/config/routes.rb
index 12e0482561..2b1f52b58c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -153,8 +153,6 @@
       post 'still',     :to => 'master_files#set_frame', :defaults => { :format => 'html' }
       get :embed
       post 'attach_structure'
-      post 'attach_captions'
-      delete 'captions', action: :delete_captions, as: 'delete_captions'
       get :captions
       get :waveform
       match ':quality.m3u8', to: 'master_files#hls_manifest', via: [:get], as: :hls_manifest

From 43315ed17a396f4b21fc4d0a1b9161850100b9c2 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Mon, 22 May 2023 16:36:21 -0400
Subject: [PATCH 055/396] Fixes for codeclimate

---
 .../supplemental_files_controller.rb          |  5 ++--
 app/models/concerns/master_file_behavior.rb   | 26 ++++++++++---------
 app/models/master_file.rb                     |  2 +-
 app/models/supplemental_file.rb               |  2 +-
 4 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/app/controllers/supplemental_files_controller.rb b/app/controllers/supplemental_files_controller.rb
index 8dbf123291..bcface71f2 100644
--- a/app/controllers/supplemental_files_controller.rb
+++ b/app/controllers/supplemental_files_controller.rb
@@ -145,9 +145,8 @@ def edit_file_information
         @supplemental_file.tags -= ['machine_generated']
       end
       @supplemental_file.label = supplemental_file_params[:label]
-      if supplemental_file_params[:language]
-        @supplemental_file.language = LanguageTerm.find(supplemental_file_params[:language]).code
-      end
+      return unless supplemental_file_params[:language]
+      @supplemental_file.language = LanguageTerm.find(supplemental_file_params[:language]).code
     end
 
     def object_supplemental_file_path
diff --git a/app/models/concerns/master_file_behavior.rb b/app/models/concerns/master_file_behavior.rb
index ef6f7a0676..103d2b5847 100644
--- a/app/models/concerns/master_file_behavior.rb
+++ b/app/models/concerns/master_file_behavior.rb
@@ -34,7 +34,7 @@ def succeeded?
   def stream_details
     flash, hls = [], []
 
-    common, captions_path, captions_format = nil, nil, nil, nil, nil
+    common, caption_paths = nil, nil
 
     derivatives.each do |d|
       common = { quality: d.quality,
@@ -58,11 +58,9 @@ def stream_details
     poster_path = Rails.application.routes.url_helpers.poster_master_file_path(self)
     if has_captions?
       caption_paths = []
-      self.supplemental_file_captions.each { |c| caption_paths.append(build_caption_hash(c)) }
+      supplemental_file_captions.each { |c| caption_paths.append(build_caption_hash(c)) }
 
-      if self.captions
-        caption_paths.append(build_caption_hash(self.captions))
-      end
+      caption_paths.append(build_caption_hash(captions)) if captions
 
       caption_paths
     end
@@ -148,16 +146,20 @@ def sort_streams array
   end
 
   def build_caption_hash(caption)
-    path = if caption.kind_of?(IndexedFile)
-             Rails.application.routes.url_helpers.captions_master_file_path(self)
-           elsif caption.kind_of?(SupplementalFile)
-             Rails.application.routes.url_helpers.master_file_supplemental_file_path(master_file_id: self.id, id: caption.id)
-           end
+    if caption.is_a?(IndexedFile)
+      path = Rails.application.routes.url_helpers.captions_master_file_path(self)
+      language = "en"
+      label = nil
+    elsif caption.is_a?(SupplementalFile)
+      path = Rails.application.routes.url_helpers.master_file_supplemental_file_path(master_file_id: self.id, id: caption.id)
+      language = caption.language
+      label = caption.label
+    end
     {
       path: path,
       mime_type: caption.mime_type,
-      language: caption.kind_of?(SupplementalFile) ? caption.language : "en",
-      label: caption.kind_of?(SupplementalFile) ? caption.label : nil
+      language: language,
+      label: label
     }
   end
 end
diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index 55846a87ba..73551bbf2c 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -480,7 +480,7 @@ def self.post_processing_move_filename(oldpath, options = {})
   attr_writer :supplemental_file_captions
 
   def supplemental_file_captions
-    self.supplemental_files(tag: 'caption')
+    supplemental_files(tag: 'caption')
   end
 
   def has_audio?
diff --git a/app/models/supplemental_file.rb b/app/models/supplemental_file.rb
index c9aae38155..e0704b447a 100644
--- a/app/models/supplemental_file.rb
+++ b/app/models/supplemental_file.rb
@@ -28,7 +28,7 @@ def attach_file(new_file)
   end
 
   def mime_type
-    self.file.content_type
+    file.content_type
   end
 
   def machine_generated?

From 9e31c7a0b16002da1c04850aba14bf9fd8f31bc4 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Tue, 23 May 2023 14:31:20 -0400
Subject: [PATCH 056/396] Add labels to edit fields for captions

This PR also includes a minor fix to the MasterFileIntercom to get tests
passing.
---
 app/assets/javascripts/supplemental_files.js  | 24 +++++++++++++++----
 app/assets/stylesheets/avalon.scss            |  6 +++++
 app/models/concerns/master_file_intercom.rb   |  2 +-
 .../_supplemental_files_list.html.erb         | 10 +++++++-
 4 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/app/assets/javascripts/supplemental_files.js b/app/assets/javascripts/supplemental_files.js
index 15b85618c2..4aab621ee1 100644
--- a/app/assets/javascripts/supplemental_files.js
+++ b/app/assets/javascripts/supplemental_files.js
@@ -1,12 +1,12 @@
-/* 
+/*
  * Copyright 2011-2023, The Trustees of Indiana University and Northwestern
  *   University.  Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
- * 
+ *
  * You may obtain a copy of the License at
- * 
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software distributed
  *   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
  *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -20,6 +20,13 @@
 $('button[name="edit_label"]').on('click', (e) => {
   const $row = getHTMLInfo(e, '.supplemental-file-data');
   const { masterfileId, fileId } = $row[0].dataset;
+  var captionLabel = document.getElementById("caption-label");
+  var captionLanguage = document.getElementById("caption-language");
+
+  if ($row.parent().hasClass("captions")) {
+    captionLabel.style.display = "";
+    captionLanguage.style.display = "";
+  }
 
   $row.addClass('is-editing');
   $row
@@ -32,7 +39,16 @@ $('button[name="edit_label"]').on('click', (e) => {
 /* Hide form when form is cancelled */
 $('button[name="cancel_edit_label"]').on('click', (e) => {
   const $row = getHTMLInfo(e, '.supplemental-file-data');
+  var captionLabel = document.getElementById("caption-label");
+  var captionLanguage = document.getElementById("caption-language");
+
   $row.removeClass('is-editing');
+  var editing = document.getElementsByClassName("is-editing")
+
+  if (editing.length === 0) {
+    captionLabel.style.display = "none";
+    captionLanguage.style.display = "none";
+  }
 });
 
 /* After editing, close the form and show the new label */
diff --git a/app/assets/stylesheets/avalon.scss b/app/assets/stylesheets/avalon.scss
index a18502fcf1..111824bb99 100644
--- a/app/assets/stylesheets/avalon.scss
+++ b/app/assets/stylesheets/avalon.scss
@@ -924,6 +924,12 @@ h5.card-title {
       }
     }
 
+    .caption-form-label {
+      display: inline-block;
+      margin-left: 10px;
+      width: 48%;
+    }
+
     div.supplemental-file-data {
       height: 1.5rem;
 
diff --git a/app/models/concerns/master_file_intercom.rb b/app/models/concerns/master_file_intercom.rb
index 233c4dfb6c..92579838c5 100644
--- a/app/models/concerns/master_file_intercom.rb
+++ b/app/models/concerns/master_file_intercom.rb
@@ -33,7 +33,7 @@ def to_ingest_api_hash(include_structure = true, remove_identifiers: false)
       file_checksum: file_checksum,
       file_format: file_format,
       other_identifier: (remove_identifiers ? [] : identifier.to_a),
-      captions: captions,
+      captions: captions.content,
       caption_type: captions.mime_type,
       supplemental_file_captions: supplemental_file_captions,
       comment: comment.to_a,
diff --git a/app/views/media_objects/_supplemental_files_list.html.erb b/app/views/media_objects/_supplemental_files_list.html.erb
index ab1a89ff91..65822eb1d9 100644
--- a/app/views/media_objects/_supplemental_files_list.html.erb
+++ b/app/views/media_objects/_supplemental_files_list.html.erb
@@ -15,7 +15,15 @@ Unless required by applicable law or agreed to in writing, software distributed
 %>
           <% if section.supplemental_files_json.present? %>
             <% files=tag.empty? ? section.supplemental_files(tag: nil) : section.supplemental_files(tag: tag) %>
-            
    
+            
    captions<% end %>">
+              <% if tag == "caption" %>
+              
+              
+              <% end %>
               <% files.each do |file| %>
                 
    
                     " class="display-item"><%= file.label %>

From 361b973439f03b86e0f58fae0733d6a4d08a5c74 Mon Sep 17 00:00:00 2001
From: dananji 
Date: Tue, 30 May 2023 15:40:22 -0400
Subject: [PATCH 057/396] Fix CSS for inline edit forms

---
 app/assets/javascripts/supplemental_files.js  | 27 +++-------
 .../_supplemental_files_list.html.erb         | 50 ++++++++++---------
 2 files changed, 34 insertions(+), 43 deletions(-)

diff --git a/app/assets/javascripts/supplemental_files.js b/app/assets/javascripts/supplemental_files.js
index 4aab621ee1..f5cd464a65 100644
--- a/app/assets/javascripts/supplemental_files.js
+++ b/app/assets/javascripts/supplemental_files.js
@@ -19,15 +19,9 @@
 /* Show form to edit label */
 $('button[name="edit_label"]').on('click', (e) => {
   const $row = getHTMLInfo(e, '.supplemental-file-data');
-  const { masterfileId, fileId } = $row[0].dataset;
-  var captionLabel = document.getElementById("caption-label");
-  var captionLanguage = document.getElementById("caption-language");
-
-  if ($row.parent().hasClass("captions")) {
-    captionLabel.style.display = "";
-    captionLanguage.style.display = "";
-  }
+  const { masterfileId, fileId, tag } = $row[0].dataset;
 
+  if(tag == 'caption') { $('#edit-label-row').addClass('is-editing') };
   $row.addClass('is-editing');
   $row
     .find(
@@ -39,16 +33,9 @@ $('button[name="edit_label"]').on('click', (e) => {
 /* Hide form when form is cancelled */
 $('button[name="cancel_edit_label"]').on('click', (e) => {
   const $row = getHTMLInfo(e, '.supplemental-file-data');
-  var captionLabel = document.getElementById("caption-label");
-  var captionLanguage = document.getElementById("caption-language");
 
+  $('#edit-label-row').removeClass('is-editing');
   $row.removeClass('is-editing');
-  var editing = document.getElementsByClassName("is-editing")
-
-  if (editing.length === 0) {
-    captionLabel.style.display = "none";
-    captionLanguage.style.display = "none";
-  }
 });
 
 /* After editing, close the form and show the new label */
@@ -79,10 +66,10 @@ $('.supplemental-file-form')
     var newLabel = $row
       .find(
         'input[id="supplemental_file_input_' +
-          masterfileId +
-          '_' +
-          fileId +
-          '"]'
+        masterfileId +
+        '_' +
+        fileId +
+        '"]'
       )
       .val();
     $row
diff --git a/app/views/media_objects/_supplemental_files_list.html.erb b/app/views/media_objects/_supplemental_files_list.html.erb
index 65822eb1d9..ccf8004c16 100644
--- a/app/views/media_objects/_supplemental_files_list.html.erb
+++ b/app/views/media_objects/_supplemental_files_list.html.erb
@@ -17,25 +17,42 @@ Unless required by applicable law or agreed to in writing, software distributed
             <% files=tag.empty? ? section.supplemental_files(tag: nil) : section.supplemental_files(tag: tag) %>
             
    captions<% end %>">
               <% if tag == "caption" %>
-              
-              
+                
    
+                  
    
+                    
+                  
    
+                  
    
+                    
+                  
    
+                
    
               <% end %>
               <% files.each do |file| %>
-                
    
+                
    
                     " class="display-item"><%= file.label %>
                     <%= form_for :supplemental_file, url: object_supplemental_file_path(section, file), remote: true,
                           html: { method: "put", class: "supplemental-file-form edit-item", id: "form-#{file.id}" },
                           data: { file_id: file.id, masterfile_id: section.id } do |form| %>
-                        
    
-                          
    
+                        
    
+                          
    
                             <%= form.text_field :label, id: "supplemental_file_input_#{section.id}_#{file.id}", value: file.label %>
                           
    
+                          <% if tag == 'transcript' %>
+                            
+                              <%= label_tag "machine_generated_#{file.id}", class: "ml-3" do %>
+                                <%= check_box_tag "machine_generated_#{file.id}", '1', file.machine_generated? %>
+                                Machine Generated
+                              <% end %>
+                            
+                          <% end %>
+                          <% if tag == 'caption' %>
+                            
    
+                              <%= form.text_field :language, id: "supplemental_file_language_#{section.id}_#{file.id}", value: LanguageTerm.find(file.language).text,
+                                  class: "typeahead from-model form-control",
+                                  data: { model: 'languageTerm', validate: false } %>
+                            
    
+                          <% end %>
                         
    
-                        
    
+                        
    
                           
    
                             <%= button_tag name: 'save_label', :class => "btn btn-outline btn-sm edit-item" do %>
                                Save
@@ -44,19 +61,6 @@ Unless required by applicable law or agreed to in writing, software distributed
                                Cancel
                             <% end %>
                           
    
-                          <% if tag == 'transcript' %>
-                          
-                            <%= label_tag "machine_generated_#{file.id}" do %>
-                            <%= check_box_tag "machine_generated_#{file.id}", '1', file.machine_generated? %>
-                            Machine Generated
-                            <% end %>
-                          
-                          <% end %>
-                          <% if tag == 'caption' %>
-                          <%= form.text_field :language, id: "supplemental_file_language_#{section.id}_#{file.id}", value: LanguageTerm.find(file.language).text,
-                              class: "typeahead from-model form-control",
-                              data: { model: 'languageTerm', validate: false } %>
-                          <% end %>
                         
    
                     <% end %>
                     

From 6713b7f04b6e3191a1dcbf3d55168d7f20216dc9 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 31 May 2023 11:36:31 -0400
Subject: [PATCH 058/396] Adjust display/hide of caption edit labels

---
 app/assets/javascripts/supplemental_files.js | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/assets/javascripts/supplemental_files.js b/app/assets/javascripts/supplemental_files.js
index f5cd464a65..081a41b7d7 100644
--- a/app/assets/javascripts/supplemental_files.js
+++ b/app/assets/javascripts/supplemental_files.js
@@ -33,9 +33,13 @@ $('button[name="edit_label"]').on('click', (e) => {
 /* Hide form when form is cancelled */
 $('button[name="cancel_edit_label"]').on('click', (e) => {
   const $row = getHTMLInfo(e, '.supplemental-file-data');
+  const { tag } = $row[0].dataset;
 
-  $('#edit-label-row').removeClass('is-editing');
   $row.removeClass('is-editing');
+
+  if(document.getElementsByClassName('is-editing').length === 1){
+    $('#edit-label-row').removeClass('is-editing');
+  }
 });
 
 /* After editing, close the form and show the new label */
@@ -45,6 +49,10 @@ $('button[name="save_label"]').on('click', (e) => {
 
   $row.removeClass('is-editing');
 
+  if(document.getElementsByClassName('is-editing').length === 1){
+    $('#edit-label-row').removeClass('is-editing');
+  }
+
   // Remove feedback message after 5 seconds
   setTimeout(function () {
     var alert = $row.find(

From 77b7821b86731fd55ae483ce36b6cb558881197b Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 31 May 2023 14:54:53 -0400
Subject: [PATCH 059/396] Use SpeedyAF::Proxy for faster ability checks

The proxy may need to be reified for non end-user actions but this is much faster for items with many sections when fetching the HLS manifest.  The speedup comes from a faster ability check where `can? :read, @master_file` ends up checking `can? :read, @master_file.media_object`.  For non-SpeedyAF objects this will fetch the large media object from fedora which could be very slow.
---
 app/controllers/master_files_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/master_files_controller.rb b/app/controllers/master_files_controller.rb
index cbb9756a4a..a1f1691657 100644
--- a/app/controllers/master_files_controller.rb
+++ b/app/controllers/master_files_controller.rb
@@ -389,7 +389,7 @@ def set_masterfile
     if params[:id].blank? || (not MasterFile.exists?(params[:id]))
       flash[:notice] = "MasterFile #{params[:id]} does not exist"
     end
-    @master_file = MasterFile.find(params[:id])
+    @master_file = SpeedyAF::Proxy::MasterFile.find(params[:id])
   end
 
   # return deflated waveform content. deflate only if necessary

From 78a2f04643ac36d9f678788c5f43a265a6dc6b85 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 1 Jun 2023 10:10:37 -0400
Subject: [PATCH 060/396] Set MasterFile duration from ActiveEncode technical
 metadata

Co-authored-by: Chris Colvard 
---
 app/models/master_file.rb | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index e0cf0c9f52..8d104a7502 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -170,6 +170,9 @@ def error
   # Generate the waveform after proessing is complete but before master file management
   after_transcoding :generate_waveform
   after_transcoding :update_ingest_batch
+  # Generate and set the poster and thumbnail
+  after_transcoding :set_poster_offset
+  after_transcoding :update_stills_from_offset!
 
   after_processing :post_processing_file_management
 
@@ -290,6 +293,7 @@ def update_progress_on_success!(encode)
     #Set date ingested to now if it wasn't preset (by batch, for example)
     #TODO pull this from the encode
     self.date_digitized ||= Time.now.utc.iso8601
+    self.duration ||= encode.input.duration
 
     outputs = Array(encode.output).collect do |output|
       {
@@ -723,12 +727,6 @@ def reloadTechnicalMetadata!
                          'Unknown'
                        end
 
-    self.duration = begin
-      @mediainfo.duration.to_s
-    rescue
-      nil
-    end
-
     unless @mediainfo.video.streams.empty?
       display_aspect_ratio_s = @mediainfo.video.streams.first.display_aspect_ratio
       if ':'.in? display_aspect_ratio_s
@@ -737,10 +735,13 @@ def reloadTechnicalMetadata!
         self.display_aspect_ratio = display_aspect_ratio_s
       end
       self.original_frame_size = @mediainfo.video.streams.first.frame_size
-      self.poster_offset = [2000,self.duration.to_i].min
     end
   end
 
+  def set_poster_offset
+    self.poster_offset = [2000,self.duration.to_i].min
+  end
+
   def post_processing_file_management
     logger.debug "Finished processing"
 

From 47d5459139c57289c3fc8a67614f394ead06c48c Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 1 Jun 2023 15:18:51 -0400
Subject: [PATCH 061/396] Readd duration assignment to reloadTechnicalMetadata!

---
 app/models/master_file.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index 8d104a7502..897c30000b 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -727,6 +727,12 @@ def reloadTechnicalMetadata!
                          'Unknown'
                        end
 
+    self.duration = begin
+      @mediainfo.duration.to_s
+    rescue
+      nil
+    end
+
     unless @mediainfo.video.streams.empty?
       display_aspect_ratio_s = @mediainfo.video.streams.first.display_aspect_ratio
       if ':'.in? display_aspect_ratio_s

From c0d86a5e1dfd9005f2e3220416e4655381fd3ede Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 2 Jun 2023 16:46:42 -0400
Subject: [PATCH 062/396] Update #set_default_poster_offset

---
 app/models/master_file.rb | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index 897c30000b..a7e58ca1ca 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -171,7 +171,7 @@ def error
   after_transcoding :generate_waveform
   after_transcoding :update_ingest_batch
   # Generate and set the poster and thumbnail
-  after_transcoding :set_poster_offset
+  after_transcoding :set_default_poster_offset
   after_transcoding :update_stills_from_offset!
 
   after_processing :post_processing_file_management
@@ -293,6 +293,9 @@ def update_progress_on_success!(encode)
     #Set date ingested to now if it wasn't preset (by batch, for example)
     #TODO pull this from the encode
     self.date_digitized ||= Time.now.utc.iso8601
+
+    # Set duration after transcode if mediainfo fails to find.
+    # e.x. WebM files missing technical metadata
     self.duration ||= encode.input.duration
 
     outputs = Array(encode.output).collect do |output|
@@ -744,8 +747,11 @@ def reloadTechnicalMetadata!
     end
   end
 
-  def set_poster_offset
-    self.poster_offset = [2000,self.duration.to_i].min
+  # This should only be getting called by the :after_transcode hook. Ensure that
+  # poster_offset is only set if it has not already been manually set via
+  # BatchEntry or another method.
+  def set_default_poster_offset
+    self.poster_offset = [2000,self.duration.to_i].min if self._poster_offset.nil?
   end
 
   def post_processing_file_management

From 6a4d4bccec7f4a5aa75428b6cf3392f429928cc8 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Mon, 5 Jun 2023 14:43:50 -0400
Subject: [PATCH 063/396] Clean up

---
 .../media_player_wrapper/avalon_player_new.es6         |  2 +-
 app/controllers/supplemental_files_controller.rb       |  2 +-
 app/models/concerns/master_file_behavior.rb            | 10 +++++-----
 app/models/concerns/master_file_intercom.rb            |  2 +-
 app/models/master_file.rb                              |  2 --
 5 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6 b/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6
index 7462fcf29c..9163d0323c 100644
--- a/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6
+++ b/app/assets/javascripts/media_player_wrapper/avalon_player_new.es6
@@ -267,7 +267,7 @@ class MEJSPlayer {
 
     if (this.currentStreamInfo.caption_paths) {
       for (c in this.currentStreamInfo.caption_paths) {
-        markup += ``;
+        markup += ``;
       }
     }
 
diff --git a/app/controllers/supplemental_files_controller.rb b/app/controllers/supplemental_files_controller.rb
index bcface71f2..07a699158e 100644
--- a/app/controllers/supplemental_files_controller.rb
+++ b/app/controllers/supplemental_files_controller.rb
@@ -145,7 +145,7 @@ def edit_file_information
         @supplemental_file.tags -= ['machine_generated']
       end
       @supplemental_file.label = supplemental_file_params[:label]
-      return unless supplemental_file_params[:language]
+      return unless supplemental_file_params[:language].present?
       @supplemental_file.language = LanguageTerm.find(supplemental_file_params[:language]).code
     end
 
diff --git a/app/models/concerns/master_file_behavior.rb b/app/models/concerns/master_file_behavior.rb
index 103d2b5847..77854b8ff7 100644
--- a/app/models/concerns/master_file_behavior.rb
+++ b/app/models/concerns/master_file_behavior.rb
@@ -146,14 +146,14 @@ def sort_streams array
   end
 
   def build_caption_hash(caption)
-    if caption.is_a?(IndexedFile)
-      path = Rails.application.routes.url_helpers.captions_master_file_path(self)
-      language = "en"
-      label = nil
-    elsif caption.is_a?(SupplementalFile)
+    if caption.is_a?(SupplementalFile)
       path = Rails.application.routes.url_helpers.master_file_supplemental_file_path(master_file_id: self.id, id: caption.id)
       language = caption.language
       label = caption.label
+    else
+      path = Rails.application.routes.url_helpers.captions_master_file_path(self)
+      language = "en"
+      label = nil
     end
     {
       path: path,
diff --git a/app/models/concerns/master_file_intercom.rb b/app/models/concerns/master_file_intercom.rb
index 92579838c5..107fcc121b 100644
--- a/app/models/concerns/master_file_intercom.rb
+++ b/app/models/concerns/master_file_intercom.rb
@@ -34,7 +34,7 @@ def to_ingest_api_hash(include_structure = true, remove_identifiers: false)
       file_format: file_format,
       other_identifier: (remove_identifiers ? [] : identifier.to_a),
       captions: captions.content,
-      caption_type: captions.mime_type,
+      captions_type: captions.mime_type,
       supplemental_file_captions: supplemental_file_captions,
       comment: comment.to_a,
       display_aspect_ratio: display_aspect_ratio,
diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index 73551bbf2c..5beada6776 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -477,8 +477,6 @@ def self.post_processing_move_filename(oldpath, options = {})
     end
   end
 
-  attr_writer :supplemental_file_captions
-
   def supplemental_file_captions
     supplemental_files(tag: 'caption')
   end

From ec3a17c356a13fb79a8af2615f80ee6713df4f0f Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 7 Jun 2023 10:54:02 -0400
Subject: [PATCH 064/396] Move waveform to seeAlso

---
 app/models/iiif_canvas_presenter.rb | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 95bdf1d9a9..9dcc89da8b 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -40,7 +40,11 @@ def display_content
   end
 
   def sequence_rendering
-    supplemental_files_rendering(master_file) + waveform_rendering
+    supplemental_files_rendering(master_file)
+  end
+
+  def see_also
+    waveform_see_also
   end
 
   def placeholder_content
@@ -79,7 +83,7 @@ def stream_urls
       end
     end
 
-    def waveform_rendering
+    def waveform_see_also
       [
         {
           "@id" => "#{@master_file.waveform_master_file_url(@master_file.id)}.json",

From 082ee4fc04d9da36fe71702d78cdf8617ce04190 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 7 Jun 2023 14:35:01 -0400
Subject: [PATCH 065/396] Validate caption file type

---
 .../supplemental_files_controller.rb          |  4 +-
 app/models/supplemental_file.rb               |  9 ++++
 spec/factories/supplemental_file.rb           |  6 ++-
 spec/models/supplemental_file_spec.rb         | 46 ++++++++++++++++---
 4 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/app/controllers/supplemental_files_controller.rb b/app/controllers/supplemental_files_controller.rb
index 07a699158e..815a5df74c 100644
--- a/app/controllers/supplemental_files_controller.rb
+++ b/app/controllers/supplemental_files_controller.rb
@@ -18,7 +18,7 @@ class SupplementalFilesController < ApplicationController
   before_action :authorize_object
 
   rescue_from Avalon::SaveError do |exception|
-    message = "An error occurred when saving the supplemental file: #{exception.full_message}"
+    message = "An error occurred when saving the supplemental file: #{exception.message}"
     handle_error(message: message, status: 500)
   end
 
@@ -44,7 +44,7 @@ def create
     # Raise errror if file wasn't attached
     raise Avalon::SaveError, "File could not be attached." unless @supplemental_file.file.attached?
 
-    raise Avalon::SaveError, @supplemental_files.errors.full_messages unless @supplemental_file.save
+    raise Avalon::SaveError, @supplemental_file.errors.full_messages unless @supplemental_file.save
 
     @object.supplemental_files += [@supplemental_file]
     raise Avalon::SaveError, @object.errors[:supplemental_files_json].full_messages unless @object.save
diff --git a/app/models/supplemental_file.rb b/app/models/supplemental_file.rb
index e0704b447a..89db5f3584 100644
--- a/app/models/supplemental_file.rb
+++ b/app/models/supplemental_file.rb
@@ -18,9 +18,14 @@ class SupplementalFile < ApplicationRecord
   # TODO: the empty tag should represent a generic supplemental file
   validates :tags, array_inclusion: ['transcript', 'caption', 'machine_generated', '', nil]
   validates :language, inclusion: { in: LanguageTerm.map.keys }
+  validate  :validate_file_type, if: :caption?
 
   serialize :tags, Array
 
+  def validate_file_type
+    errors.add(:file_type, "Uploaded file is not a recognized captions file") unless ['text/vtt', 'text/srt'].include? file.content_type
+  end
+
   def attach_file(new_file)
     file.attach(new_file)
     self.label = file.filename.to_s if label.blank?
@@ -31,6 +36,10 @@ def mime_type
     file.content_type
   end
 
+  def caption?
+    tags.include?('caption')
+  end
+
   def machine_generated?
     tags.include?('machine_generated')
   end
diff --git a/spec/factories/supplemental_file.rb b/spec/factories/supplemental_file.rb
index 4652799aac..5b02a8ee6a 100644
--- a/spec/factories/supplemental_file.rb
+++ b/spec/factories/supplemental_file.rb
@@ -22,7 +22,11 @@
     end
 
     trait :with_transcript_file do
-      file { fixture_file_upload(Rails.root.join('spec', 'fixtures', 'captions.vtt'), 'text/vtt')}
+      file { fixture_file_upload(Rails.root.join('spec', 'fixtures', 'captions.vtt'), 'text/vtt') }
+    end
+
+    trait :with_caption_file do
+      file { fixture_file_upload(Rails.root.join('spec', 'fixtures', 'captions.vtt'), 'text/vtt') }
     end
 
     trait :with_transcript_tag do
diff --git a/spec/models/supplemental_file_spec.rb b/spec/models/supplemental_file_spec.rb
index 2f8d7dc58a..d2775509a2 100644
--- a/spec/models/supplemental_file_spec.rb
+++ b/spec/models/supplemental_file_spec.rb
@@ -16,12 +16,50 @@
 
 describe SupplementalFile do
   let(:subject) { FactoryBot.create(:supplemental_file) }
-  
+
+  describe 'validations' do
+    describe 'file type' do
+      context 'non-caption file' do
+        let(:subject) { FactoryBot.create(:supplemental_file, :with_attached_file) }
+        it 'should skip validation' do
+          expect(subject.valid?).to be_truthy
+        end
+      end
+      context 'VTT/SRT caption file' do
+        let(:subject) { FactoryBot.create(:supplemental_file, :with_caption_file, :with_caption_tag) }
+        it 'should validate' do
+          expect(subject.valid?).to be_truthy
+        end
+      end
+      context 'non-VTT/non-SRT caption file' do
+        let(:subject) { FactoryBot.create(:supplemental_file, :with_attached_file) }
+        it 'should not validate' do
+          subject.tags = ['caption']
+          subject.save
+          expect(subject.valid?).to be_falsey
+          expect(subject.errors[:file_type]).not_to be_empty
+        end
+      end
+    end
+
+    describe 'language' do
+      it 'should validate valid language' do
+        subject.language = 'eng'
+        expect(subject.valid?).to be_truthy
+      end
+      it 'should not validate invalid language' do
+        subject.language = 'engl'
+        expect(subject.valid?).to be_falsey
+      end
+    end
+  end
+
   it "stores no tags by default" do
     expect(subject.tags).to match_array([])
   end
 
   context "with valid tags" do
+    let(:subject) { FactoryBot.create(:supplemental_file, :with_caption_file)}
     let(:tags) { ["transcript", "caption", "machine_generated"] }
 
     it "can store tags" do
@@ -47,11 +85,5 @@
       subject.save
       expect(subject.reload.language).to eq "ger"
     end
-
-    it "is limited to ISO 639-2 language codes" do
-      subject.language = 'English'
-      subject.save
-      expect(subject).to_not be_valid
-    end
   end
 end

From b0ebd014eb163fdb0a059ec75b81c15eec019877 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 7 Jun 2023 16:05:48 -0400
Subject: [PATCH 066/396] Fix tests

Adding validation for caption file type broke some tests that did not
explicitly assign a file to a supplemental file tagged as a caption.
---
 spec/models/iiif_canvas_presenter_spec.rb         | 2 +-
 spec/models/master_file_spec.rb                   | 2 +-
 spec/support/supplemental_file_shared_examples.rb | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index adb90a0d4d..cc3a45612b 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -118,7 +118,7 @@
     let(:master_file) { FactoryBot.build(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
     let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
     let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
-    let(:caption_file) { FactoryBot.create(:supplemental_file, tags: ['caption', 'machine_generated']) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, tags: ['caption', 'machine_generated']) }
     let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
     let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
 
diff --git a/spec/models/master_file_spec.rb b/spec/models/master_file_spec.rb
index ec9b4a42f6..3ea99904e0 100644
--- a/spec/models/master_file_spec.rb
+++ b/spec/models/master_file_spec.rb
@@ -722,7 +722,7 @@
   end
 
   describe 'supplemental_file_captions' do
-    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_tag) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, :with_caption_tag) }
     let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
     let(:master_file) { FactoryBot.create(:master_file, supplemental_files: [caption_file, transcript_file]) }
     it 'has a caption' do
diff --git a/spec/support/supplemental_file_shared_examples.rb b/spec/support/supplemental_file_shared_examples.rb
index 918ff3f70e..bfef023eac 100644
--- a/spec/support/supplemental_file_shared_examples.rb
+++ b/spec/support/supplemental_file_shared_examples.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -16,7 +16,7 @@
   let(:object) { FactoryBot.build(described_class.model_name.singular.to_sym) }
   let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
   let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
-  let(:caption_file) { FactoryBot.create(:supplemental_file, tags: ['caption', 'machine_generated']) }
+  let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, tags: ['caption', 'machine_generated']) }
   let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
   let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
 

From 772faf2d20374300e0ac7de3bb176927ee5b40b0 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 9 Jun 2023 15:35:47 -0400
Subject: [PATCH 067/396] Improve behavior of public and token timelines

---
 app/controllers/timelines_controller.rb       |  10 +-
 spec/controllers/timelines_controller_spec.rb | 104 +++++++++++++++---
 spec/factories/timeline.rb                    |  10 +-
 3 files changed, 101 insertions(+), 23 deletions(-)

diff --git a/app/controllers/timelines_controller.rb b/app/controllers/timelines_controller.rb
index 8dca8aa444..a222137d7b 100644
--- a/app/controllers/timelines_controller.rb
+++ b/app/controllers/timelines_controller.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -104,10 +104,10 @@ def show
     respond_to do |format|
       format.html do
         url_fragment = "noHeader=true&noFooter=true&noSourceLink=false&noVideo=false"
-        if current_user == @timeline.user
+        if @timeline.visibility == 'public' || current_user == @timeline.user
           url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
           url_fragment += "&callback=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
-        elsif current_user
+        elsif current_user || @timeline.valid_token?(@timeline.access_token)
           url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json, token: @timeline.access_token), /[:\/?=]/)}"
           url_fragment += "&callback=#{Addressable::URI.escape_component(timelines_url, /[:\/?=]/)}"
         end
diff --git a/spec/controllers/timelines_controller_spec.rb b/spec/controllers/timelines_controller_spec.rb
index e9dd16f73d..f7f9c42bdc 100644
--- a/spec/controllers/timelines_controller_spec.rb
+++ b/spec/controllers/timelines_controller_spec.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -222,12 +222,75 @@
   end
 
   describe "GET #show" do
-    it "renders the timeliner tool" do
-      timeline = Timeline.create! valid_attributes
-      get :show, params: {id: timeline.to_param}, session: valid_session
-      expect(response).to be_successful
+    let(:cataloger) { FactoryBot.create(:cataloger)}
+    let(:timeline) { FactoryBot.create(:timeline, user: cataloger) }
+    let(:encoded_manifest_url) do
+      [controller.default_url_options[:protocol],
+      "%3A%2F%2F",
+      controller.default_url_options[:host],
+      "%2Ftimelines%2F",
+      timeline.id,
+      "%2Fmanifest.json"].join
+    end
+
+    context 'private timeline' do
+      context 'unauthenticated user' do
+        it 'redirects to sign-in' do
+          expect(get :show, params: { id: timeline.to_param }, session: valid_session ).to render_template('errors/restricted_pid')
+        end
+      end
+
+      context 'non-owner' do
+        before do
+          login_as :student
+        end
+
+        it 'renders restricted content page' do
+          expect(get :show, params: { id: timeline.to_param }, session: valid_session ).to render_template('errors/restricted_pid')
+        end
+      end
+
+      context 'owner' do
+        before do
+          sign_in cataloger
+        end
+
+        render_views
+
+        it "renders the timeliner tool" do
+          get :show, params: { id: timeline.to_param }, session: valid_session
+          expect(response).to be_successful
+          expect(response.body).to include "resource=#{encoded_manifest_url}"
+        end
+      end
     end
 
+    context 'public timeline' do
+      let(:timeline) { FactoryBot.create(:timeline, visibility: Timeline::PUBLIC) }
+
+      render_views
+
+      context 'unauthenticated user' do
+        it "renders the timeliner tool" do
+          get :show, params: {id: timeline.to_param}, session: valid_session
+          expect(response).to be_successful
+          expect(response.body).to include "resource=#{encoded_manifest_url}"
+        end
+      end
+
+      context 'authenticated user' do
+        before do
+          login_as :user
+        end
+        it "renders the timeliner tool" do
+          get :show, params: {id: timeline.to_param}, session: valid_session
+          expect(response).to be_successful
+          expect(response.body).to include "resource=#{encoded_manifest_url}"
+        end
+      end
+    end
+
+
     context 'with token auth' do
       let(:timeline) { FactoryBot.create(:timeline, :with_access_token) }
       let(:encoded_manifest_url) do
@@ -240,15 +303,30 @@
         timeline.access_token].join
       end
 
-      before do
-        user
+      render_views
+
+      context 'unauthenticated user' do
+        it "correctly encodes tokenized timeline urls" do
+          get :show, params: {id: timeline.to_param, token: timeline.access_token}
+          expect(response.body).to include "resource=#{encoded_manifest_url}"
+        end
       end
 
-      render_views
+      context 'authenticated user' do
+        before do
+          login_as :user
+        end
+
+        it "correctly encodes tokenized timeline urls" do
+          get :show, params: {id: timeline.to_param, token: timeline.access_token}
+          expect(response.body).to include "resource=#{encoded_manifest_url}"
+        end
+      end
 
-      it "correctly encodes tokenized timeline urls" do
-        get :show, params: {id: timeline.to_param, token: timeline.access_token}
-        expect(response.body).to include "resource=#{encoded_manifest_url}"
+      context 'wrong token' do
+        it 'renders restricted content page' do
+          expect(get :show, params: { id: timeline.to_param, token: 'faketoken' }).to render_template('errors/restricted_pid')
+        end
       end
     end
 
diff --git a/spec/factories/timeline.rb b/spec/factories/timeline.rb
index 80dea5dd52..edd5958a93 100644
--- a/spec/factories/timeline.rb
+++ b/spec/factories/timeline.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -17,13 +17,13 @@
     user { FactoryBot.create(:user) }
     title { "Timeline: #{Faker::Lorem.word}" }
     description { Faker::Lorem.sentence }
-    visibility { Playlist::PRIVATE }
+    visibility { Timeline::PRIVATE }
     manifest { }
     source { }
     tags { [Faker::Lorem.word, Faker::Lorem.word] }
 
     trait :with_access_token do
-      visibility { Playlist::PRIVATE_WITH_TOKEN }
+      visibility { Timeline::PRIVATE_WITH_TOKEN }
       access_token { Faker::Lorem.characters(number: 10) }
     end
   end

From 6217ba35bf9c0246dcfd91724bb5cf088c33c48f Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 9 Jun 2023 16:02:03 -0400
Subject: [PATCH 068/396] Move #show url_fragment generation to own method

---
 app/controllers/timelines_controller.rb | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/app/controllers/timelines_controller.rb b/app/controllers/timelines_controller.rb
index a222137d7b..9889a9653a 100644
--- a/app/controllers/timelines_controller.rb
+++ b/app/controllers/timelines_controller.rb
@@ -103,14 +103,7 @@ def show
     authorize! :read, @timeline
     respond_to do |format|
       format.html do
-        url_fragment = "noHeader=true&noFooter=true&noSourceLink=false&noVideo=false"
-        if @timeline.visibility == 'public' || current_user == @timeline.user
-          url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
-          url_fragment += "&callback=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
-        elsif current_user || @timeline.valid_token?(@timeline.access_token)
-          url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json, token: @timeline.access_token), /[:\/?=]/)}"
-          url_fragment += "&callback=#{Addressable::URI.escape_component(timelines_url, /[:\/?=]/)}"
-        end
+        url_fragment = build_url_fragment
         @timeliner_iframe_url = timeliner_path + "##{url_fragment}"
       end
       format.json do
@@ -360,6 +353,18 @@ def load_timeline_token
       current_ability.options[:timeline_token] = @timeline_token
     end
 
+    def build_url_fragment
+      url_fragment = "noHeader=true&noFooter=true&noSourceLink=false&noVideo=false"
+
+      if @timeline.visibility == 'public' || current_user == @timeline.user
+        url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
+        url_fragment += "&callback=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
+      elsif current_user || @timeline.valid_token?(@timeline.access_token)
+        url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json, token: @timeline.access_token), /[:\/?=]/)}"
+        url_fragment += "&callback=#{Addressable::URI.escape_component(timelines_url, /[:\/?=]/)}"
+      end
+    end
+
     # Never trust parameters from the scary internet, only allow the white list through.
     def timeline_params
       new_params = params.fetch(:timeline, {}).permit(:title, :visibility, :description, :access_token, :tags, :source, :manifest, :include_structure)

From 4ce538092ad8133c8431b77fb18a685ea337d61e Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 9 Jun 2023 16:15:42 -0400
Subject: [PATCH 069/396] Fix for codeclimate

---
 app/controllers/timelines_controller.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/controllers/timelines_controller.rb b/app/controllers/timelines_controller.rb
index 9889a9653a..44f7c95c1e 100644
--- a/app/controllers/timelines_controller.rb
+++ b/app/controllers/timelines_controller.rb
@@ -363,6 +363,8 @@ def build_url_fragment
         url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json, token: @timeline.access_token), /[:\/?=]/)}"
         url_fragment += "&callback=#{Addressable::URI.escape_component(timelines_url, /[:\/?=]/)}"
       end
+
+      url_fragment
     end
 
     # Never trust parameters from the scary internet, only allow the white list through.

From 2f2f102a67217c915238a788f93e8741206f93ca Mon Sep 17 00:00:00 2001
From: Mason Ballengee <68433277+masaball@users.noreply.github.com>
Date: Mon, 12 Jun 2023 17:34:43 -0400
Subject: [PATCH 070/396] Simplify negative caption validation test

Co-authored-by: Chris Colvard 
---
 spec/models/supplemental_file_spec.rb | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/spec/models/supplemental_file_spec.rb b/spec/models/supplemental_file_spec.rb
index d2775509a2..9171f45f50 100644
--- a/spec/models/supplemental_file_spec.rb
+++ b/spec/models/supplemental_file_spec.rb
@@ -32,10 +32,8 @@
         end
       end
       context 'non-VTT/non-SRT caption file' do
-        let(:subject) { FactoryBot.create(:supplemental_file, :with_attached_file) }
+        let(:subject) { FactoryBot.build(:supplemental_file, :with_attached_file, :with_caption_tag) }
         it 'should not validate' do
-          subject.tags = ['caption']
-          subject.save
           expect(subject.valid?).to be_falsey
           expect(subject.errors[:file_type]).not_to be_empty
         end

From 5edf0ce7141ad4a1630619e309375aa1b471716c Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 13 Jun 2023 10:59:44 -0400
Subject: [PATCH 071/396] Small refactor and test for save callback url

Co-authored-by: Mason Ballengee 
---
 app/controllers/timelines_controller.rb       | 14 ++++-
 spec/controllers/timelines_controller_spec.rb | 60 +++++++++++++++----
 2 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/app/controllers/timelines_controller.rb b/app/controllers/timelines_controller.rb
index 44f7c95c1e..20eceada34 100644
--- a/app/controllers/timelines_controller.rb
+++ b/app/controllers/timelines_controller.rb
@@ -356,12 +356,20 @@ def load_timeline_token
     def build_url_fragment
       url_fragment = "noHeader=true&noFooter=true&noSourceLink=false&noVideo=false"
 
-      if @timeline.visibility == 'public' || current_user == @timeline.user
+      # Manifest url - include token if present
+      if @timeline_token.present?
+        url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json, token: @timeline_token), /[:\/?=]/)}"
+      else
         url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
+      end
+
+      # Save callback - Owner uses manifest url, logged in users use timeline create url, unauthenticated do not have a callback
+      if current_user == @timeline.user
         url_fragment += "&callback=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json), /[:\/?=]/)}"
-      elsif current_user || @timeline.valid_token?(@timeline.access_token)
-        url_fragment += "&resource=#{Addressable::URI.escape_component(manifest_timeline_url(@timeline, format: :json, token: @timeline.access_token), /[:\/?=]/)}"
+      elsif current_user
         url_fragment += "&callback=#{Addressable::URI.escape_component(timelines_url, /[:\/?=]/)}"
+      else
+        # no-op
       end
 
       url_fragment
diff --git a/spec/controllers/timelines_controller_spec.rb b/spec/controllers/timelines_controller_spec.rb
index f7f9c42bdc..fb7c719eaa 100644
--- a/spec/controllers/timelines_controller_spec.rb
+++ b/spec/controllers/timelines_controller_spec.rb
@@ -232,6 +232,12 @@
       timeline.id,
       "%2Fmanifest.json"].join
     end
+    let(:timeline_create_url) do
+      [controller.default_url_options[:protocol],
+      "%3A%2F%2F",
+      controller.default_url_options[:host],
+      "%2Ftimelines"].join
+    end
 
     context 'private timeline' do
       context 'unauthenticated user' do
@@ -261,39 +267,54 @@
           get :show, params: { id: timeline.to_param }, session: valid_session
           expect(response).to be_successful
           expect(response.body).to include "resource=#{encoded_manifest_url}"
+          expect(response.body).to include "callback=#{encoded_manifest_url}"
         end
       end
     end
 
     context 'public timeline' do
-      let(:timeline) { FactoryBot.create(:timeline, visibility: Timeline::PUBLIC) }
+      let(:timeline) { FactoryBot.create(:timeline, visibility: Timeline::PUBLIC, user: cataloger) }
 
       render_views
 
       context 'unauthenticated user' do
-        it "renders the timeliner tool" do
+        it "renders the timeliner tool without save feature" do
           get :show, params: {id: timeline.to_param}, session: valid_session
           expect(response).to be_successful
           expect(response.body).to include "resource=#{encoded_manifest_url}"
+          expect(response.body).not_to include "callback="
         end
       end
 
-      context 'authenticated user' do
+      context 'non-owner' do
         before do
           login_as :user
         end
-        it "renders the timeliner tool" do
+        it "renders the timeliner tool with save to new timeline" do
           get :show, params: {id: timeline.to_param}, session: valid_session
           expect(response).to be_successful
           expect(response.body).to include "resource=#{encoded_manifest_url}"
+          expect(response.body).to include "callback=#{timeline_create_url}"
         end
       end
-    end
 
+      context 'owner' do
+        before do
+          sign_in cataloger
+        end
+
+        it "renders the timeliner tool" do
+          get :show, params: { id: timeline.to_param }, session: valid_session
+          expect(response).to be_successful
+          expect(response.body).to include "resource=#{encoded_manifest_url}"
+          expect(response.body).to include "callback=#{encoded_manifest_url}"
+        end
+      end
+    end
 
     context 'with token auth' do
-      let(:timeline) { FactoryBot.create(:timeline, :with_access_token) }
-      let(:encoded_manifest_url) do
+      let(:timeline) { FactoryBot.create(:timeline, :with_access_token, user: cataloger) }
+      let(:encoded_manifest_url_with_token) do
         [controller.default_url_options[:protocol],
         "%3A%2F%2F",
         controller.default_url_options[:host],
@@ -306,24 +327,39 @@
       render_views
 
       context 'unauthenticated user' do
-        it "correctly encodes tokenized timeline urls" do
+        it "renders the timeliner tool without save feature" do
           get :show, params: {id: timeline.to_param, token: timeline.access_token}
-          expect(response.body).to include "resource=#{encoded_manifest_url}"
+          expect(response.body).to include "resource=#{encoded_manifest_url_with_token}"
+          expect(response.body).not_to include "callback="
         end
       end
 
-      context 'authenticated user' do
+      context 'non-owner' do
         before do
           login_as :user
         end
 
-        it "correctly encodes tokenized timeline urls" do
+        it "renders the timeliner tool with save to new timeline" do
           get :show, params: {id: timeline.to_param, token: timeline.access_token}
+          expect(response.body).to include "resource=#{encoded_manifest_url_with_token}"
+          expect(response.body).to include "callback=#{timeline_create_url}"
+        end
+      end
+
+      context 'owner' do
+        before do
+          sign_in cataloger
+        end
+
+        it "renders the timeliner tool" do
+          get :show, params: { id: timeline.to_param }, session: valid_session
+          expect(response).to be_successful
           expect(response.body).to include "resource=#{encoded_manifest_url}"
+          expect(response.body).to include "callback=#{encoded_manifest_url}"
         end
       end
 
-      context 'wrong token' do
+      context 'invalid token' do
         it 'renders restricted content page' do
           expect(get :show, params: { id: timeline.to_param, token: 'faketoken' }).to render_template('errors/restricted_pid')
         end

From fc12e335156996a6bd1024b4a25902fea5689afc Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 13 Jun 2023 14:02:17 -0400
Subject: [PATCH 072/396] Add ability checks for SpeedyAF::Base objects; only
 use proxy for end-user actions; fix tests

---
 app/controllers/master_files_controller.rb    | 10 +++++++-
 app/models/ability.rb                         | 17 +++++++++++++
 .../speedy_af/proxy/structural_metadata.rb    | 24 +++++++++++++++++++
 .../master_files_controller_spec.rb           | 10 ++++----
 4 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/app/controllers/master_files_controller.rb b/app/controllers/master_files_controller.rb
index a1f1691657..1a48487e7e 100644
--- a/app/controllers/master_files_controller.rb
+++ b/app/controllers/master_files_controller.rb
@@ -21,7 +21,8 @@ class MasterFilesController < ApplicationController
   include NoidValidator
 
   before_action :authenticate_user!, :only => [:create]
-  before_action :set_masterfile, except: [:create, :oembed]
+  before_action :set_masterfile_proxy, except: [:create, :oembed, :attach_structure, :attach_captions, :delete_structure, :delete_captions, :destroy, :update]
+  before_action :set_masterfile, only: [:attach_structure, :attach_captions, :delete_structure, :delete_captions, :destroy, :update]
   before_action :ensure_readable_filedata, :only => [:create]
   skip_before_action :verify_authenticity_token, only: [:set_structure, :delete_structure]
 
@@ -389,6 +390,13 @@ def set_masterfile
     if params[:id].blank? || (not MasterFile.exists?(params[:id]))
       flash[:notice] = "MasterFile #{params[:id]} does not exist"
     end
+    @master_file = MasterFile.find(params[:id])
+  end
+
+  def set_masterfile_proxy
+    if params[:id].blank? || SpeedyAF::Proxy::MasterFile.find(params[:id]).nil?
+      flash[:notice] = "MasterFile #{params[:id]} does not exist"
+    end
     @master_file = SpeedyAF::Proxy::MasterFile.find(params[:id])
   end
 
diff --git a/app/models/ability.rb b/app/models/ability.rb
index 1ec76affe2..2b93ab36c7 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -24,6 +24,15 @@ class Ability
                          :timeline_permissions,
                          :checkout_permissions]
 
+  # Override to add handling of SpeedyAF proxy objects
+  def read_permissions
+    super
+
+    can :read, SpeedyAF::Base do |obj|
+      test_read(obj.id)
+    end
+  end
+
   def encode_dashboard_permissions
     can :read, :encode_dashboard if is_administrator?
   end
@@ -68,10 +77,18 @@ def custom_permissions(user=nil, session=nil)
         !(test_read(media_object.id) && media_object.published?) && !test_edit(media_object.id)
       end
 
+      cannot :read, SpeedyAF::Proxy::MediaObject do |media_object|
+        !(test_read(media_object.id) && media_object.published?) && !test_edit(media_object.id)
+      end
+
       can :read, MasterFile do |master_file|
         can? :read, master_file.media_object
       end
 
+      can :read, SpeedyAF::Proxy::MasterFile do |master_file|
+        can? :read, master_file.media_object
+      end
+
       can :read, Derivative do |derivative|
         can? :read, derivative.masterfile.media_object
       end
diff --git a/app/presenters/speedy_af/proxy/structural_metadata.rb b/app/presenters/speedy_af/proxy/structural_metadata.rb
index f3d24e71c4..fa15aeee0a 100644
--- a/app/presenters/speedy_af/proxy/structural_metadata.rb
+++ b/app/presenters/speedy_af/proxy/structural_metadata.rb
@@ -22,4 +22,28 @@ def ng_xml
   def section_title
     xpath('/Item/@label').text
   end
+
+  def as_json
+    root_node = xpath('//Item')[0]
+    root_node.present? ? node_xml_to_json(root_node) : {}
+  end
+
+  protected
+
+    def node_xml_to_json(node)
+      if node.name.casecmp("div").zero? || node.name.casecmp('item').zero?
+        {
+          type: 'div',
+          label: node.attribute('label').value,
+          items: node.children.reject(&:blank?).collect { |n| node_xml_to_json n }
+        }
+      elsif node.name.casecmp('span').zero?
+        {
+          type: 'span',
+          label: node.attribute('label').value,
+          begin: node.attribute('begin').present? ? node.attribute('begin').value : '0',
+          end: node.attribute('end').present? ? node.attribute('end').value : '0'
+        }
+      end
+    end
 end
diff --git a/spec/controllers/master_files_controller_spec.rb b/spec/controllers/master_files_controller_spec.rb
index 8a8556d829..b2fe430b3f 100644
--- a/spec/controllers/master_files_controller_spec.rb
+++ b/spec/controllers/master_files_controller_spec.rb
@@ -615,11 +615,11 @@ class << file
 
     context 'master file has been deleted' do
       before do
-        allow(MasterFile).to receive(:find).and_raise(Ldp::Gone)
+        master_file.destroy
       end
 
       it 'returns gone (403)' do
-        expect(get('hls_manifest', params: { id: "deleted", quality: 'auto' })).to have_http_status(:gone)
+        expect(get('hls_manifest', params: { id: master_file.id, quality: 'auto' })).to have_http_status(:unauthorized)
       end
     end
 
@@ -652,11 +652,11 @@ class << file
 
     context 'master file has been deleted' do
       before do
-        allow(MasterFile).to receive(:find).and_raise(Ldp::Gone)
+        master_file.destroy
       end
 
-      it 'returns gone (403)' do
-        expect(get('caption_manifest', params: { id: master_file.id }, xhr: true)).to have_http_status(:gone)
+      it 'returns unauthorized (401)' do
+        expect(get('caption_manifest', params: { id: master_file.id }, xhr: true)).to have_http_status(:unauthorized)
       end
     end
 

From 4d05e3017e98455f76a3979aa7cacff0d6212af8 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 14 Jun 2023 12:05:49 -0400
Subject: [PATCH 073/396] Adjust tests

---
 spec/models/iiif_canvas_presenter_spec.rb | 28 +++++++++++++++++++----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index adb90a0d4d..141c3009ec 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -124,17 +124,35 @@
 
     subject { presenter.sequence_rendering }
 
-    it 'includes waveform' do
-      expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq true
-    end
-
     it 'includes supplemental files' do
       expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq true
     end
 
-    it 'does not include transcripts or captions' do
+    it 'does not include waveform, transcripts, or captions' do
+      expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq false
       expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{transcript_file.id}/ }).to eq false
       expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{caption_file.id}/ }).to eq false
     end
   end
+
+  describe '#see_also' do
+    let(:master_file) { FactoryBot.build(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
+    let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
+    let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, tags: ['caption', 'machine_generated']) }
+    let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
+    let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
+
+    subject { presenter.see_also }
+
+    it 'includes waveform' do
+      expect(subject.any? { |sa| sa["label"]["en"] == ["waveform.json"] }).to eq true
+    end
+
+    it 'does not include supplemental files, transcripts, or captions' do
+      expect(subject.any? { |sa| sa["@id"] =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq false
+      expect(subject.any? { |sa| sa["@id"] =~ /supplemental_files\/#{transcript_file.id}/ }).to eq false
+      expect(subject.any? { |sa| sa["@id"] =~ /supplemental_files\/#{caption_file.id}/ }).to eq false
+    end
+  end
 end

From 7106011ba0a63f707b335914e72dc2af55e5baa0 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 14 Jun 2023 15:30:41 -0400
Subject: [PATCH 074/396] Slight refactor of #see_also

Currently we only intend to provide waveforms through seeAlso so there
is no need to have a method separate from #see_also generating the
waveform hash. May need to be split out again if we add more to seeAlso
later.
---
 app/models/iiif_canvas_presenter.rb | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 9dcc89da8b..0d428e007d 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -44,7 +44,14 @@ def sequence_rendering
   end
 
   def see_also
-    waveform_see_also
+    [
+      {
+        "@id" => "#{@master_file.waveform_master_file_url(@master_file.id)}.json",
+        "type" => "Dataset",
+        "label" => { "en" => ["waveform.json"] },
+        "format" => "application/json"
+      }
+    ]
   end
 
   def placeholder_content
@@ -83,17 +90,6 @@ def stream_urls
       end
     end
 
-    def waveform_see_also
-      [
-        {
-          "@id" => "#{@master_file.waveform_master_file_url(@master_file.id)}.json",
-          "type" => "Dataset",
-          "label" => { "en" => ["waveform.json"] },
-          "format" => "application/json"
-        }
-      ]
-    end
-
     def simple_iiif_range(label = stream_info[:embed_title])
       # TODO: embed_title?
       IiifManifestRange.new(

From 0dd5135124e52496f15473bd5ba2a6906633ac8d Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 15 Jun 2023 11:20:52 -0400
Subject: [PATCH 075/396] Bump iiif_manifest

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index ec47eba561..edc4422096 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -67,7 +67,7 @@ GIT
 
 GIT
   remote: https://github.com/samvera-labs/iiif_manifest.git
-  revision: 33db00d04abbb0c66a6947bf24f8d913c5319c4f
+  revision: e5d8a2d3b775665e59dc123bd461cac5ac7c9cce
   branch: main
   specs:
     iiif_manifest (1.3.1)

From 4f30a7d939327c59431dcb00fdf8a03eac5fcfc7 Mon Sep 17 00:00:00 2001
From: dananji 
Date: Thu, 15 Jun 2023 14:50:38 -0400
Subject: [PATCH 076/396] New SME build using seeAlso property for waveform
 information

---
 yarn.lock | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 72761be0ef..58c055d704 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -894,9 +894,9 @@
     "@babel/plugin-transform-react-pure-annotations" "^7.14.5"
 
 "@babel/runtime@^7.1.2", "@babel/runtime@^7.15.4", "@babel/runtime@^7.2.0", "@babel/runtime@^7.3.1", "@babel/runtime@^7.4.4", "@babel/runtime@^7.8.3", "@babel/runtime@^7.9.2":
-  version "7.21.5"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.21.5.tgz#8492dddda9644ae3bda3b45eabe87382caee7200"
-  integrity sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==
+  version "7.22.5"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.5.tgz#8564dd588182ce0047d55d7a75e93921107b57ec"
+  integrity sha512-ecjvYlnAaZ/KVneE/OdKYBYfgXV3Ptu6zQWmgEF7vwKhQnvVS6bjMD2XYgj+SNvQ1GfK/pjgokfPkC/2CO8CuA==
   dependencies:
     regenerator-runtime "^0.13.11"
 
@@ -1348,9 +1348,9 @@
     redux "^4.0.0"
 
 "@types/react-transition-group@^4.2.0":
-  version "4.4.5"
-  resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.5.tgz#aae20dcf773c5aa275d5b9f7cdbca638abc5e416"
-  integrity sha512-juKD/eiSM3/xZYzjuzH6ZwpP+/lejltmiS3QEzV/vmb/Q8+HfDmxu+Baga8UEMGBqV88Nbg4l2hY/K2DkyaLLA==
+  version "4.4.6"
+  resolved "https://registry.yarnpkg.com/@types/react-transition-group/-/react-transition-group-4.4.6.tgz#18187bcda5281f8e10dfc48f0943e2fdf4f75e2e"
+  integrity sha512-VnCdSxfcm08KjsJVQcfBmhEQAPnLB8G08hAxn39azX1qYBQ/5RVQuoHuKIcfKOdncuaUvEpFKFzEvbtIMsfVew==
   dependencies:
     "@types/react" "*"
 
@@ -3869,9 +3869,9 @@ hex-color-regex@^1.1.0:
   integrity sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==
 
 hls.js@^1.1.2:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/hls.js/-/hls.js-1.4.0.tgz#e73af3e7c8d4310a2bcc92d5ade362dfb781fef7"
-  integrity sha512-VEjg7Rx5FlE9TB3MIn0HPgq3J+vR7EoQnjaqMCk/ISEaCOSZlAFh4g867f1QkSxZiq9kHeUZo+iH16X7VS3jKA==
+  version "1.4.5"
+  resolved "https://registry.yarnpkg.com/hls.js/-/hls.js-1.4.5.tgz#33b2d6cbebb4797bd61763c87c2f15c7ef080632"
+  integrity sha512-xb7IiSM9apU3tJWb5rdSStobXPNJJykHTwSy7JnLF5y/kLJXWjoR/fEpNBlwYxkKcDiiSfO9SQI8yFravZJxIg==
 
 "hls.js@https://github.com/avalonmediasystem/hls.js#stricter_ts_probing":
   version "0.13.1"
@@ -4649,9 +4649,9 @@ klona@^2.0.4:
   integrity sha512-ZRbnvdg/NxqzC7L9Uyqzf4psi1OM4Cuc+sJAkQPjO6XkQIJTNbfK2Rsmbw8fx1p2mkZdp2FZYo2+LwXYY/uwIA==
 
 konva@^9.0.0:
-  version "9.0.1"
-  resolved "https://registry.yarnpkg.com/konva/-/konva-9.0.1.tgz#92b4171beaa94273b108bd87b08c4d3e51a0da22"
-  integrity sha512-wzpkprJ8idE42TDF9Lu9RNjVVYNXrj0apvTK3pujdHQhX1iNV+MUquSxYN8HqjYSG95QQ51jhFzRLWhnhf44Mw==
+  version "9.2.0"
+  resolved "https://registry.yarnpkg.com/konva/-/konva-9.2.0.tgz#3739e539724b0e6b76d697a322efdaa01baa1508"
+  integrity sha512-+woI76Sk+VFVl9z7zPkuTnN2zFpEYg27YWz8BCdQXpt5IS3pdnSPAPQVPPMidcbDi9/G5b/IOIp35/KqMGiYPA==
 
 last-call-webpack-plugin@^3.0.0:
   version "3.0.0"
@@ -6597,7 +6597,7 @@ react-redux@^7.2.6:
 
 "react-structural-metadata-editor@https://github.com/avalonmediasystem/react-structural-metadata-editor":
   version "1.1.0"
-  resolved "https://github.com/avalonmediasystem/react-structural-metadata-editor#50dbd6d38fde3ccdaf174c2048a7c4f7e3fcbe36"
+  resolved "https://github.com/avalonmediasystem/react-structural-metadata-editor#ba45fcb5b5198aa83fdb9bf41aee081f2b62baee"
   dependencies:
     "@babel/runtime" "^7.4.4"
     "@fortawesome/fontawesome-svg-core" "^1.2.4"

From 18afdca91cc5daf087187450d901fc493180ad7b Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 16 Jun 2023 10:52:01 -0400
Subject: [PATCH 077/396] Fix failing #see_also tests

---
 spec/models/iiif_canvas_presenter_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 1f79445c1b..9b80e7cebe 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -139,7 +139,7 @@
     let(:master_file) { FactoryBot.build(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
     let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
     let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
-    let(:caption_file) { FactoryBot.create(:supplemental_file, tags: ['caption', 'machine_generated']) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, tags: ['caption', 'machine_generated']) }
     let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
     let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
 

From b9c9885b17d1470f6ba103259ecddebb48b4de89 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 16 Jun 2023 14:09:57 -0400
Subject: [PATCH 078/396] Avoid reifying from fedora when rendering thumbnail
 on search results page

---
 app/models/indexed_file.rb                    | 28 +++++++++++++++++++
 .../speedy_af/proxy/indexed_file.rb           | 28 +++++++++++++++++++
 .../speedy_af/proxy/media_object.rb           |  4 +++
 3 files changed, 60 insertions(+)
 create mode 100644 app/presenters/speedy_af/proxy/indexed_file.rb

diff --git a/app/models/indexed_file.rb b/app/models/indexed_file.rb
index 8bc23bd106..3de4ae20c8 100644
--- a/app/models/indexed_file.rb
+++ b/app/models/indexed_file.rb
@@ -14,9 +14,37 @@
 
 class IndexedFile < ActiveFedora::File
   include SpeedyAF::IndexedContent
+  MAX_CONTENT_SIZE = 32000
 
   # Override
   def original_name
     super&.force_encoding("UTF-8")
   end
+
+  # Override to add binary content handling
+  def to_solr(solr_doc = {}, opts = {})
+    return solr_doc unless opts[:external_index]
+    solr_doc.tap do |doc|
+      doc[:id] = id
+      doc[:has_model_ssim] = self.class.name
+      doc[:uri_ss] = uri.to_s
+      doc[:mime_type_ss] = mime_type
+      doc[:original_name_ss] = original_name
+      doc[:size_is] = content.present? ? content.size : 0
+      doc[:'empty?_bs'] = content.blank?
+      if index_content?
+        doc[:content_ss] = binary_content? ? Base64.encode64(content) : content
+      end
+    end
+  end
+
+  protected
+
+  def index_content?
+    has_content? && size < MAX_CONTENT_SIZE
+  end
+
+  def binary_content?
+    has_content? && mime_type !~ /(^text\/)|([\/\+]xml$)/
+  end
 end
diff --git a/app/presenters/speedy_af/proxy/indexed_file.rb b/app/presenters/speedy_af/proxy/indexed_file.rb
new file mode 100644
index 0000000000..88f4f6c5d3
--- /dev/null
+++ b/app/presenters/speedy_af/proxy/indexed_file.rb
@@ -0,0 +1,28 @@
+# Copyright 2011-2023, The Trustees of Indiana University and Northwestern
+#   University.  Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+# 
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software distributed
+#   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+#   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+#   specific language governing permissions and limitations under the License.
+# ---  END LICENSE_HEADER BLOCK  ---
+
+class SpeedyAF::Proxy::IndexedFile < SpeedyAF::Base
+  # If necessary, decode binary content that is base 64 encoded
+  def content
+    binary_content? ? Base64.decode64(attrs[:content]) : attrs[:content]
+  end
+
+  def has_content?
+    attrs[:content].present?
+  end
+
+  def binary_content?
+    has_content? && mime_type !~ /(^text\/)|([\/\+]xml$)/
+  end
+end
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index ac8e540879..770b2dce80 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -29,6 +29,10 @@ def to_param
     id
   end
 
+  def published?
+    !avalon_publisher.blank?
+  end
+
   # @return [SupplementalFile]
   def supplemental_files(tag: '*')
     return [] if supplemental_files_json.blank?

From 35042d4f034fc4f144929e144accd2d18810eaf2 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 16 Jun 2023 14:51:47 -0400
Subject: [PATCH 079/396] Avoid reifying on hls_manifest requests when master
 file title isn't set

---
 app/presenters/speedy_af/proxy/master_file.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/presenters/speedy_af/proxy/master_file.rb b/app/presenters/speedy_af/proxy/master_file.rb
index 56740e662d..6feff784ac 100644
--- a/app/presenters/speedy_af/proxy/master_file.rb
+++ b/app/presenters/speedy_af/proxy/master_file.rb
@@ -26,6 +26,11 @@ def find_encoder_class(klass_name)
     klass if klass&.ancestors&.include?(ActiveEncode::Base)
   end
 
+  # We know that title will be indexed if present so return presence to avoid reifying
+  def title
+    attrs[:title].presence
+  end
+
   def display_title
     mf_title = if has_structuralMetadata?
                  structuralMetadata.section_title

From c0b519ceb82989b16e89671b0c985945fe2dc68a Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 16 Jun 2023 17:13:50 -0400
Subject: [PATCH 080/396] Don't load from fedora on item view page or IIIF
 manifest Refactored shared methods out of MediaObject into new
 MediaObjectBehavior concern to allow for reuse in
 SpeedyAF::Proxy::MediaObject Moved setting defaults into presenters
 initializers instead of SpeedyAF#find calls

---
 app/controllers/media_objects_controller.rb   | 24 ++----
 app/helpers/security_helper.rb                |  2 +-
 app/models/checkout.rb                        |  2 +-
 app/models/concerns/media_object_behavior.rb  | 58 ++++++++++++++
 app/models/iiif_manifest_presenter.rb         | 10 +--
 app/models/media_object.rb                    | 42 +---------
 .../speedy_af/proxy/admin/collection.rb       | 10 +++
 app/presenters/speedy_af/proxy/master_file.rb |  4 +
 .../speedy_af/proxy/media_object.rb           | 76 ++++++++++++++++++-
 config/initializers/presenter_config.rb       | 57 +++++++++++++-
 10 files changed, 216 insertions(+), 69 deletions(-)
 create mode 100644 app/models/concerns/media_object_behavior.rb

diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb
index 7249094022..8883deef00 100644
--- a/app/controllers/media_objects_controller.rb
+++ b/app/controllers/media_objects_controller.rb
@@ -24,6 +24,7 @@ class MediaObjectsController < ApplicationController
   include SecurityHelper
 
   before_action :authenticate_user!, except: [:show, :set_session_quality, :show_stream_details, :manifest]
+  before_action :load_resource, except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview]
   load_and_authorize_resource except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview]
   authorize_resource only: [:create]
 
@@ -470,7 +471,7 @@ def tree
   end
 
   def manifest
-    @media_object = MediaObject.find(params[:id])
+    @media_object = SpeedyAF::Proxy::MediaObject.find(params[:id])
     authorize! :read, @media_object
 
     master_files = master_file_presenters
@@ -531,22 +532,13 @@ def move_preview
 
   protected
 
+  def load_resource
+    @media_object = SpeedyAF::Proxy::MediaObject.find(params[:id])
+  end
+
   def master_file_presenters
-    # NOTE: Defaults are set on returned SpeedyAF::Base objects if field isn't present in the solr doc.
-    # This is important otherwise speedy_af will reify from fedora when trying to access this field.
-    # When adding a new property to the master file model that will be used in the interface,
-    # add it to the default below to avoid reifying for master files lacking a value for the property.
-    SpeedyAF::Proxy::MasterFile.where("isPartOf_ssim:#{@media_object.id}",
-                                      order: -> { @media_object.indexed_master_file_ids },
-                                      defaults: {
-                                        permalink: nil,
-                                        title: nil,
-                                        encoder_classname: nil,
-                                        workflow_id: nil,
-                                        comment: [],
-                                        supplemental_files_json: nil
-                                      },
-                                      load_reflections: true)
+    # Assume that @media_object is a SpeedyAF::Proxy::MediaObject
+    @media_object.ordered_master_files
   end
 
   def load_master_files(mode = :rw)
diff --git a/app/helpers/security_helper.rb b/app/helpers/security_helper.rb
index 57e99e14ea..bc670b5d38 100644
--- a/app/helpers/security_helper.rb
+++ b/app/helpers/security_helper.rb
@@ -38,6 +38,6 @@ def add_stream_url(stream_info)
   private
 
     def not_checked_out?(media_object_id)
-      lending_enabled?(MediaObject.find(media_object_id)) && Checkout.checked_out_to_user(media_object_id, current_user&.id).empty?
+      lending_enabled?(SpeedyAF::Proxy::MediaObject.find(media_object_id)) && Checkout.checked_out_to_user(media_object_id, current_user&.id).empty?
     end
 end
diff --git a/app/models/checkout.rb b/app/models/checkout.rb
index 9d0812d668..7016eefc55 100644
--- a/app/models/checkout.rb
+++ b/app/models/checkout.rb
@@ -25,7 +25,7 @@ class Checkout < ApplicationRecord
   scope :checked_out_to_user, ->(media_object_id, user_id) { where("media_object_id = ? AND user_id = ? AND return_time > now()", media_object_id, user_id) }
 
   def media_object
-    MediaObject.find(media_object_id)
+    SpeedyAF::Proxy::MediaObject.find(media_object_id)
   end
 
   private
diff --git a/app/models/concerns/media_object_behavior.rb b/app/models/concerns/media_object_behavior.rb
new file mode 100644
index 0000000000..e11aa9c782
--- /dev/null
+++ b/app/models/concerns/media_object_behavior.rb
@@ -0,0 +1,58 @@
+# Copyright 2011-2023, The Trustees of Indiana University and Northwestern
+#   University.  Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed
+#   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+#   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+#   specific language governing permissions and limitations under the License.
+# ---  END LICENSE_HEADER BLOCK  ---
+
+# This module contains methods which transform stored values for use either on the MediaObject or the SpeedyAF presenter
+module MediaObjectBehavior
+  def published?
+    !avalon_publisher.blank?
+  end
+
+  def access_text
+    actors = []
+    if visibility == "public"
+      actors << "the public"
+    else
+      actors << "collection staff" if visibility == "private"
+      actors << "specific users" if read_users.any? || leases('user').any?
+
+      if visibility == "restricted"
+        actors << "logged-in users"
+      elsif virtual_read_groups.any? || local_read_groups.any? || leases('external').any? || leases('local').any?
+        actors << "users in specific groups"
+      end
+
+      actors << "users in specific IP Ranges" if ip_read_groups.any? || leases('ip').any?
+    end
+
+    "This item is accessible by: #{actors.join(', ')}."
+  end
+
+  # CDL methods
+  def lending_status
+    Checkout.active_for_media_object(id).any? ? "checked_out" : "available"
+  end
+
+  def return_time
+    Checkout.active_for_media_object(id).first&.return_time
+  end
+
+  def cdl_enabled?
+    collection&.cdl_enabled?
+  end
+
+  def current_checkout(user_id)
+    checkouts = Checkout.active_for_media_object(id)
+    checkouts.select{ |ch| ch.user_id == user_id  }.first
+  end
+end
diff --git a/app/models/iiif_manifest_presenter.rb b/app/models/iiif_manifest_presenter.rb
index 7b648e745a..185b6bad57 100644
--- a/app/models/iiif_manifest_presenter.rb
+++ b/app/models/iiif_manifest_presenter.rb
@@ -53,7 +53,7 @@ def manifest_metadata
   end
 
   def thumbnail
-    @thumbnail ||= image_for(media_object.to_solr)
+    @thumbnail ||= thumbnail_url
   end
 
   def ranges
@@ -179,11 +179,11 @@ def iiif_metadata_fields
     fields
   end
 
-  def image_for(document)
-    master_file_id = document["section_id_ssim"].try :first
+  def thumbnail_url
+    master_file_id = media_object.ordered_master_file_ids.try :first
 
-    video_count = document["avalon_resource_type_ssim"]&.count { |m| m.start_with?('moving image'.titleize) } || 0
-    audio_count = document["avalon_resource_type_ssim"]&.count { |m| m.start_with?('sound recording'.titleize) } || 0
+    video_count = media_object.avalon_resource_type.map(&:titleize)&.count { |m| m.start_with?('moving image'.titleize) } || 0
+    audio_count = media_object.avalon_resource_type.map(&:titleize)&.count { |m| m.start_with?('sound recording'.titleize) } || 0
 
     if master_file_id
       if video_count > 0
diff --git a/app/models/media_object.rb b/app/models/media_object.rb
index 527abd67de..c15d3f1afc 100644
--- a/app/models/media_object.rb
+++ b/app/models/media_object.rb
@@ -25,6 +25,7 @@ class MediaObject < ActiveFedora::Base
   include SpeedyAF::OrderedAggregationIndex
   include MediaObjectIntercom
   include SupplementalFileBehavior
+  include MediaObjectBehavior
   require 'avalon/controlled_vocabulary'
 
   include Kaminari::ActiveFedoraModelExtension
@@ -124,10 +125,6 @@ def validate_date(date_field)
 
   accepts_nested_attributes_for :master_files, :allow_destroy => true
 
-  def published?
-    !avalon_publisher.blank?
-  end
-
   def destroy
     # attempt to stop the matterhorn processing job
     self.master_files.each(&:destroy)
@@ -376,48 +373,11 @@ def merge!(media_objects)
     [mergeds, faileds]
   end
 
-  def access_text
-    actors = []
-    if visibility == "public"
-      actors << "the public"
-    else
-      actors << "collection staff" if visibility == "private"
-      actors << "specific users" if read_users.any? || leases('user').any?
-
-      if visibility == "restricted"
-        actors << "logged-in users"
-      elsif virtual_read_groups.any? || local_read_groups.any? || leases('external').any? || leases('local').any?
-        actors << "users in specific groups"
-      end
-
-      actors << "users in specific IP Ranges" if ip_read_groups.any? || leases('ip').any?
-    end
-
-    "This item is accessible by: #{actors.join(', ')}."
-  end
-
-  def lending_status
-    Checkout.active_for_media_object(id).any? ? "checked_out" : "available"
-  end
-
-  def return_time
-    Checkout.active_for_media_object(id).first&.return_time
-  end
-
   alias_method :'_lending_period', :'lending_period'
   def lending_period
     self._lending_period || collection&.default_lending_period
   end
 
-  def cdl_enabled?
-    collection&.cdl_enabled?
-  end
-
-  def current_checkout(user_id)
-    checkouts = Checkout.active_for_media_object(id)
-    checkouts.select{ |ch| ch.user_id == user_id  }.first
-  end
-
   # Override to reset memoized fields
   def reload
     @master_file_docs = nil
diff --git a/app/presenters/speedy_af/proxy/admin/collection.rb b/app/presenters/speedy_af/proxy/admin/collection.rb
index b57434d9ba..09608afb6a 100644
--- a/app/presenters/speedy_af/proxy/admin/collection.rb
+++ b/app/presenters/speedy_af/proxy/admin/collection.rb
@@ -28,4 +28,14 @@ def model_name
   def to_param
     id
   end
+
+  def cdl_enabled?
+    if cdl_enabled.nil?
+      Settings.controlled_digital_lending.collections_enabled
+    elsif cdl_enabled != Settings.controlled_digital_lending.collections_enabled
+      cdl_enabled
+    else
+      Settings.controlled_digital_lending.collections_enabled
+    end
+  end
 end
diff --git a/app/presenters/speedy_af/proxy/master_file.rb b/app/presenters/speedy_af/proxy/master_file.rb
index 6feff784ac..386614d326 100644
--- a/app/presenters/speedy_af/proxy/master_file.rb
+++ b/app/presenters/speedy_af/proxy/master_file.rb
@@ -13,6 +13,10 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class SpeedyAF::Proxy::MasterFile < SpeedyAF::Base
+  def to_param
+    id
+  end
+
   def encoder_class
     find_encoder_class(encoder_classname) ||
       find_encoder_class("#{workflow_name}_encode".classify) ||
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index 770b2dce80..d3c2ed922e 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -13,6 +13,21 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class SpeedyAF::Proxy::MediaObject < SpeedyAF::Base
+  # Override to handle section_id specially
+  def initialize(solr_document, instance_defaults = {})
+    instance_defaults ||= {}
+    @model = SpeedyAF::Base.model_for(solr_document)
+    @attrs = self.class.defaults.merge(instance_defaults)
+    solr_document.each_pair do |k, v|
+     attr_name, value = parse_solr_field(k, v)
+      @attrs[attr_name.to_sym] = value
+    end
+#byebug
+    # Handle this case here until a better fix can be found for multiple solr fields which don't have a model property
+    @attrs[:section_id] = solr_document["section_id_ssim"]
+    # TODO Need to convert hidden_bsi into discover_groups?
+  end
+
   def to_model
     self
   end
@@ -29,10 +44,6 @@ def to_param
     id
   end
 
-  def published?
-    !avalon_publisher.blank?
-  end
-
   # @return [SupplementalFile]
   def supplemental_files(tag: '*')
     return [] if supplemental_files_json.blank?
@@ -46,4 +57,61 @@ def supplemental_files(tag: '*')
       files.select { |file| Array(tag).all? { |t| file.tags.include?(t) } }
     end
   end
+
+  def master_file_ids
+    if real?
+      real_object.indexed_master_file_ids
+    elsif section_id.nil? # No master files or not indexed yet
+      ActiveFedora::Base.logger.warn("Reifying MediaObject because master_files not indexed")
+      real_object.indexed_master_file_ids
+    else
+      section_id
+    end
+  end
+  alias_method :indexed_master_file_ids, :master_file_ids
+  alias_method :ordered_master_file_ids, :master_file_ids
+
+  def master_files
+    # NOTE: Defaults are set on returned SpeedyAF::Base objects if field isn't present in the solr doc.
+    # This is important otherwise speedy_af will reify from fedora when trying to access this field.
+    # When adding a new property to the master file model that will be used in the interface,
+    # add it to the default below to avoid reifying for master files lacking a value for the property.
+    SpeedyAF::Proxy::MasterFile.where("isPartOf_ssim:#{id}",
+                                      order: -> { master_file_ids },
+                                      load_reflections: true)
+  end
+  alias_method :indexed_master_files, :master_files
+  alias_method :ordered_master_files, :master_files
+
+  def collection
+    SpeedyAF::Proxy::Admin::Collection.find(collection_id)
+  end
+
+  def lending_period
+    attrs[:lending_period].presence || collection&.default_lending_period
+  end
+
+  # Copied from Hydra-Access-Controls
+  def visibility
+    if read_groups.include? Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC
+      Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_PUBLIC
+    elsif read_groups.include? Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED
+      Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_AUTHENTICATED
+    else
+      Hydra::AccessControls::AccessRight::VISIBILITY_TEXT_VALUE_PRIVATE
+    end
+  end
+
+  def represented_visibility
+    [Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_AUTHENTICATED,
+     Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC]
+  end
+
+  def leases(scope=:all)
+    governing_policies.select { |gp| gp.is_a?(SpeedyAF::Proxy::Lease) and (scope == :all or gp.lease_type == scope) }
+  end
+
+  def governing_policies
+    @governing_policies ||= Array(attrs[:isGovernedBy]).collect { |id| SpeedyAF::Base.find(id) }
+  end
 end
diff --git a/config/initializers/presenter_config.rb b/config/initializers/presenter_config.rb
index c28196826e..f71321fcce 100644
--- a/config/initializers/presenter_config.rb
+++ b/config/initializers/presenter_config.rb
@@ -1,15 +1,70 @@
 Rails.application.config.to_prepare do
   SpeedyAF::Base.tap do |sp|
     sp.config MasterFile do
-      self.defaults = { permalink: nil }
+      self.defaults = { 
+                        permalink: nil,
+                        title: nil,
+                        encoder_classname: nil,
+                        workflow_id: nil,
+                        comment: [],
+                        supplemental_files_json: nil,
+                        width: nil,
+                        height: nil
+                      }
       include MasterFileBehavior
       include Rails.application.routes.url_helpers
     end
 
+    sp.config MediaObject do
+      self.defaults = {
+                        permalink: nil,
+                        abstract: nil,
+			genre: [],
+                        subject: [],
+			statement_of_responsibility: nil,
+			avalon_publisher: nil,
+			creator: [],
+			discover_groups: [],
+			read_groups: [],
+                        read_users: [],
+			edit_groups: [],
+			edit_users: [],
+                        supplemental_files_json: nil,
+                        contributor: [],
+                        publisher: [],
+                        temporal_subject: [],
+                        geographic_subject: [],
+                        language: [],
+                        terms_of_use: nil,
+                        physical_description: [],
+                        related_item_url: [],
+                        note: [],
+                        other_identifier: [],
+                        rights_statement: nil,
+                        table_of_contents: [],
+                        bibliographic_id: nil,
+                        comment: [],
+                        date_issued: nil
+                      }
+      include VirtualGroups
+      include MediaObjectBehavior
+      include Rails.application.routes.url_helpers
+    end
+
+    sp.config Admin::Collection do
+      self.defaults = {
+                        cdl_enabled: nil
+                      }
+    end
+
     sp.config Derivative do
       include DerivativeBehavior
     end
 
+    sp.config Lease do
+      self.defaults = { lease_type: nil }
+    end
+
     sp.config StructuralMetadata do
       def ng_xml
 	@ng_xml ||= Nokogiri::XML(content)

From 542928d0cb133428a0039e4dd3735f026e495a3d Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 20 Jun 2023 14:34:48 -0400
Subject: [PATCH 081/396] Load from fedora when doing edit, update, or
 json_update

---
 app/controllers/media_objects_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb
index 8883deef00..8f79ce22c3 100644
--- a/app/controllers/media_objects_controller.rb
+++ b/app/controllers/media_objects_controller.rb
@@ -24,7 +24,7 @@ class MediaObjectsController < ApplicationController
   include SecurityHelper
 
   before_action :authenticate_user!, except: [:show, :set_session_quality, :show_stream_details, :manifest]
-  before_action :load_resource, except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview]
+  before_action :load_resource, except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview, :edit, :update, :json_update]
   load_and_authorize_resource except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview]
   authorize_resource only: [:create]
 

From 17a04aceb73a76072f45f8366cd338b84e2435ae Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 20 Jun 2023 15:19:16 -0400
Subject: [PATCH 082/396] Assume solr fields are multiple if they include the m
 suffix except if explicitly marked as singular in property definitions

---
 app/models/concerns/media_object_behavior.rb  | 18 +++++++++
 app/models/media_object.rb                    | 18 ---------
 .../speedy_af/proxy/media_object.rb           | 40 ++++++++++++++++++-
 config/initializers/presenter_config.rb       |  2 +
 4 files changed, 59 insertions(+), 19 deletions(-)

diff --git a/app/models/concerns/media_object_behavior.rb b/app/models/concerns/media_object_behavior.rb
index e11aa9c782..58071d6b22 100644
--- a/app/models/concerns/media_object_behavior.rb
+++ b/app/models/concerns/media_object_behavior.rb
@@ -14,6 +14,24 @@
 
 # This module contains methods which transform stored values for use either on the MediaObject or the SpeedyAF presenter
 module MediaObjectBehavior
+  def as_json(options={})
+    {
+      id: id,
+      title: title,
+      collection: collection.name,
+      unit: collection.unit,
+      main_contributors: creator,
+      publication_date: date_created,
+      published_by: avalon_publisher,
+      published: published?,
+      summary: abstract,
+      visibility: visibility,
+      read_groups: read_groups,
+      lending_period: lending_period,
+      lending_status: lending_status,
+    }.merge(to_ingest_api_hash(options.fetch(:include_structure, false)))
+  end
+
   def published?
     !avalon_publisher.blank?
   end
diff --git a/app/models/media_object.rb b/app/models/media_object.rb
index c15d3f1afc..6e8380fb78 100644
--- a/app/models/media_object.rb
+++ b/app/models/media_object.rb
@@ -292,24 +292,6 @@ def to_solr(include_child_fields: false)
     end
   end
 
-  def as_json(options={})
-    {
-      id: id,
-      title: title,
-      collection: collection.name,
-      unit: collection.unit,
-      main_contributors: creator,
-      publication_date: date_created,
-      published_by: avalon_publisher,
-      published: published?,
-      summary: abstract,
-      visibility: visibility,
-      read_groups: read_groups,
-      lending_period: lending_period,
-      lending_status: lending_status,
-    }.merge(to_ingest_api_hash(options.fetch(:include_structure, false)))
-  end
-
   # Other validation to consider adding into future iterations is the ability to
   # validate against a known controlled vocabulary. This one will take some thought
   # and research as opposed to being able to just throw something together in an ad hoc
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index d3c2ed922e..ad2788204a 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -13,6 +13,8 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class SpeedyAF::Proxy::MediaObject < SpeedyAF::Base
+  SINGULAR_FIELDS = [:title, :statement_of_responsibility, :date_created, :date_issued, :copyright_date, :abstract, :terms_of_use, :rights_statement]
+
   # Override to handle section_id specially
   def initialize(solr_document, instance_defaults = {})
     instance_defaults ||= {}
@@ -22,10 +24,12 @@ def initialize(solr_document, instance_defaults = {})
      attr_name, value = parse_solr_field(k, v)
       @attrs[attr_name.to_sym] = value
     end
-#byebug
     # Handle this case here until a better fix can be found for multiple solr fields which don't have a model property
     @attrs[:section_id] = solr_document["section_id_ssim"]
     # TODO Need to convert hidden_bsi into discover_groups?
+    SINGULAR_FIELDS.each do |field_name|
+      @attrs[field_name] = Array(@attrs[field_name]).first
+    end
   end
 
   def to_model
@@ -91,6 +95,14 @@ def lending_period
     attrs[:lending_period].presence || collection&.default_lending_period
   end
 
+  def format
+    # TODO figure out how to memoize this
+    mime_types = master_files.reject { |mf| mf.file_location.blank? }.collect do |mf|
+      Rack::Mime.mime_type(File.extname(mf.file_location))
+    end.uniq
+    mime_types.empty? ? nil : mime_types
+  end
+
   # Copied from Hydra-Access-Controls
   def visibility
     if read_groups.include? Hydra::AccessControls::AccessRight::PERMISSION_TEXT_VALUE_PUBLIC
@@ -114,4 +126,30 @@ def leases(scope=:all)
   def governing_policies
     @governing_policies ||= Array(attrs[:isGovernedBy]).collect { |id| SpeedyAF::Base.find(id) }
   end
+
+  protected
+
+  # Overrides from SpeedyAF::Base
+  def parse_solr_field(k, v)
+    # :nocov:
+    transforms = {
+      'dt' => ->(m) { Time.parse(m) },
+      'b' => ->(m) { m },
+      'db' => ->(m) { m.to_f },
+      'f' => ->(m) { m.to_f },
+      'i' => ->(m) { m.to_i },
+      'l' => ->(m) { m.to_i },
+      nil => ->(m) { m }
+    }
+    # :nocov:
+    attr_name, type, _stored, _indexed, multi = k.scan(/^(.+)_(.+)(s)(i?)(m?)$/).first
+    return [k, v] if attr_name.nil?
+    value = Array(v).map { |m| transforms.fetch(type, transforms[nil]).call(m) }
+    value = value.first if !multi || (@model.respond_to?(:properties) && singular?(@model.properties[attr_name]))
+    [attr_name, value]
+  end
+
+  def singular?(prop)
+    (prop.present? && prop.respond_to?(:multiple?) && !prop.multiple?) || belongs_to_reflections.values.collect(&:predicate_for_solr)
+  end
 end
diff --git a/config/initializers/presenter_config.rb b/config/initializers/presenter_config.rb
index f71321fcce..699f0e3ffc 100644
--- a/config/initializers/presenter_config.rb
+++ b/config/initializers/presenter_config.rb
@@ -11,6 +11,7 @@
                         width: nil,
                         height: nil
                       }
+      include MasterFileIntercom
       include MasterFileBehavior
       include Rails.application.routes.url_helpers
     end
@@ -47,6 +48,7 @@
                         date_issued: nil
                       }
       include VirtualGroups
+      include MediaObjectIntercom
       include MediaObjectBehavior
       include Rails.application.routes.url_helpers
     end

From 1e30d2ac67f675207798127fa0181b9aba82a29a Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 11:08:38 -0400
Subject: [PATCH 083/396] Mirror previous behavior by raising not found if
 SpeedyAF can't find the MediaObject

---
 app/models/checkout.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/models/checkout.rb b/app/models/checkout.rb
index 7016eefc55..f2d7977a04 100644
--- a/app/models/checkout.rb
+++ b/app/models/checkout.rb
@@ -25,7 +25,9 @@ class Checkout < ApplicationRecord
   scope :checked_out_to_user, ->(media_object_id, user_id) { where("media_object_id = ? AND user_id = ? AND return_time > now()", media_object_id, user_id) }
 
   def media_object
-    SpeedyAF::Proxy::MediaObject.find(media_object_id)
+    @media_object ||= SpeedyAF::Proxy::MediaObject.find(media_object_id)
+    raise ActiveFedora::ObjectNotFoundError if @media_object.nil?
+    @media_object
   end
 
   private

From 358cb50d60180612c6b8c38eef6f4a67e9881908 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Tue, 30 May 2023 14:10:07 -0400
Subject: [PATCH 084/396] Add captions and transcripts to iiif manifest

---
 Gemfile                             |  2 +-
 Gemfile.lock                        |  4 ++--
 app/models/iiif_canvas_presenter.rb | 22 ++++++++++++++++++++++
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/Gemfile b/Gemfile
index 7ecca6a435..13c5b31c23 100644
--- a/Gemfile
+++ b/Gemfile
@@ -53,7 +53,7 @@ gem 'avalon-about', git: 'https://github.com/avalonmediasystem/avalon-about.git'
 #gem 'bootstrap-sass', '< 3.4.1' # Pin to less than 3.4.1 due to change in behavior with popovers
 gem 'bootstrap-toggle-rails'
 gem 'bootstrap_form'
-gem 'iiif_manifest', git: 'https://github.com/samvera-labs/iiif_manifest.git', branch: 'main'
+gem 'iiif_manifest', git: 'https://github.com/samvera-labs/iiif_manifest.git', branch: 'supplementing_annotations'
 gem 'rack-cors', require: 'rack/cors'
 gem 'rails_same_site_cookie'
 gem 'recaptcha', require: 'recaptcha/rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index edc4422096..43c11335cb 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -67,8 +67,8 @@ GIT
 
 GIT
   remote: https://github.com/samvera-labs/iiif_manifest.git
-  revision: e5d8a2d3b775665e59dc123bd461cac5ac7c9cce
-  branch: main
+  revision: dca6d509eeadf2ef78039668572deb4e95cf9a02
+  branch: supplementing_annotations
   specs:
     iiif_manifest (1.3.1)
       activesupport (>= 4)
diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 0d428e007d..0dc05c3b53 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -39,6 +39,10 @@ def display_content
     master_file.is_video? ? video_content : audio_content
   end
 
+  def supplementing_content
+    supplemental_captions_transcripts.collect { |file| supplementing_content_data(file) }
+  end
+
   def sequence_rendering
     supplemental_files_rendering(master_file)
   end
@@ -84,12 +88,21 @@ def audio_display_content(quality)
                                            **manifest_attributes(quality, 'Sound'))
     end
 
+    def supplementing_content_data(file)
+      IIIFManifest::V3::SupplementingContent.new(Rails.application.routes.url_helpers.master_file_supplemental_file_url(master_file.id, file.id),
+                                                 **supplemental_attributes(file))
+    end
+
     def stream_urls
       stream_info[:stream_hls].collect do |d|
         [d[:quality], d[:url]]
       end
     end
 
+    def supplemental_captions_transcripts
+      master_file.supplemental_files(tag: 'caption') + master_file.supplemental_files(tag: 'transcript')
+    end
+
     def simple_iiif_range(label = stream_info[:embed_title])
       # TODO: embed_title?
       IiifManifestRange.new(
@@ -158,6 +171,15 @@ def manifest_attributes(quality, media_type)
       end
     end
 
+    def supplemental_attributes(file)
+      supplement_hash = {
+        label: file.label,
+        type: 'Text',
+        format: file.file.content_type,
+        language: file.language
+      }
+    end
+
     # Note that the method returns empty Nokogiri Document instead of nil when structure_tesim doesn't exist or is empty.
     def structure_ng_xml
       # TODO: The XML parser should handle invalid XML files, for ex, if a non-leaf node has no valid "Div" or "Span" children,

From d99955719588f02faaef85160ed20af6b32d2ea8 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 2 Jun 2023 15:22:37 -0400
Subject: [PATCH 085/396] Add tests

---
 Gemfile                                   |  2 +-
 Gemfile.lock                              |  4 +--
 app/models/iiif_canvas_presenter.rb       |  2 +-
 spec/models/iiif_canvas_presenter_spec.rb | 36 ++++++++++++++++++-----
 4 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/Gemfile b/Gemfile
index 13c5b31c23..7ecca6a435 100644
--- a/Gemfile
+++ b/Gemfile
@@ -53,7 +53,7 @@ gem 'avalon-about', git: 'https://github.com/avalonmediasystem/avalon-about.git'
 #gem 'bootstrap-sass', '< 3.4.1' # Pin to less than 3.4.1 due to change in behavior with popovers
 gem 'bootstrap-toggle-rails'
 gem 'bootstrap_form'
-gem 'iiif_manifest', git: 'https://github.com/samvera-labs/iiif_manifest.git', branch: 'supplementing_annotations'
+gem 'iiif_manifest', git: 'https://github.com/samvera-labs/iiif_manifest.git', branch: 'main'
 gem 'rack-cors', require: 'rack/cors'
 gem 'rails_same_site_cookie'
 gem 'recaptcha', require: 'recaptcha/rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index 43c11335cb..edc4422096 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -67,8 +67,8 @@ GIT
 
 GIT
   remote: https://github.com/samvera-labs/iiif_manifest.git
-  revision: dca6d509eeadf2ef78039668572deb4e95cf9a02
-  branch: supplementing_annotations
+  revision: e5d8a2d3b775665e59dc123bd461cac5ac7c9cce
+  branch: main
   specs:
     iiif_manifest (1.3.1)
       activesupport (>= 4)
diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 0dc05c3b53..b511881b39 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -176,7 +176,7 @@ def supplemental_attributes(file)
         label: file.label,
         type: 'Text',
         format: file.file.content_type,
-        language: file.language
+        language: file.language || 'en'
       }
     end
 
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 9b80e7cebe..4c5eeb7c22 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -114,7 +114,7 @@
     end
   end
 
-  describe '#sequence_rendering' do
+  describe 'Supplemental file handling' do
     let(:master_file) { FactoryBot.build(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
     let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
     let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
@@ -122,16 +122,36 @@
     let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
     let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
 
-    subject { presenter.sequence_rendering }
+    describe '#sequence_rendering' do
+      subject { presenter.sequence_rendering }
 
-    it 'includes supplemental files' do
-      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq true
+      it 'includes supplemental files' do
+        expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq true
+      end
+
+      it 'does not include waveform, transcripts, or captions' do
+        expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq false
+        expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{transcript_file.id}/ }).to eq false
+        expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{caption_file.id}/ }).to eq false
+      end
     end
 
-    it 'does not include waveform, transcripts, or captions' do
-      expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq false
-      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{transcript_file.id}/ }).to eq false
-      expect(subject.any? { |rendering| rendering["@id"] =~ /supplemental_files\/#{caption_file.id}/ }).to eq false
+    describe '#supplementing_content' do
+      subject { presenter.supplementing_content }
+
+      it 'converts file metadata into IIIF Manifest SupplementingContent' do
+        expect(subject).to all be_a(IIIFManifest::V3::SupplementingContent)
+      end
+
+      it 'includes transcript and caption files' do
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{transcript_file.id}/ }).to eq true
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{caption_file.id}/ }).to eq true
+      end
+
+      it 'does not include generic supplemental files or waveform' do
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq false
+        expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq false
+      end
     end
   end
 

From c5f4cc7f3b70287528c086bc94b6d3a3f6fba736 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Thu, 15 Jun 2023 14:46:33 -0400
Subject: [PATCH 086/396] Generate distinct IIIF IDs for captions and
 transcripts

---
 app/models/iiif_canvas_presenter.rb             | 10 ++++++++--
 config/application.rb                           |  1 +
 config/routes.rb                                |  7 ++++++-
 spec/models/iiif_canvas_presenter_spec.rb       |  7 ++++++-
 spec/routing/supplemental_files_routing_spec.rb | 14 +++++++++++---
 5 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index b511881b39..ccd7c26ba3 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -89,8 +89,14 @@ def audio_display_content(quality)
     end
 
     def supplementing_content_data(file)
-      IIIFManifest::V3::SupplementingContent.new(Rails.application.routes.url_helpers.master_file_supplemental_file_url(master_file.id, file.id),
-                                                 **supplemental_attributes(file))
+      url = if file.tags.include?('caption')
+              Rails.application.routes.url_helpers.caption_master_file_supplemental_file_url(master_file.id, file.id)
+            elsif file.tags.include?('transcript')
+              Rails.application.routes.url_helpers.transcript_master_file_supplemental_file_url(master_file.id, file.id)
+            else
+              Rails.application.routes.url_helpers.master_file_supplemental_file_url(master_file.id, file.id)\
+            end
+      IIIFManifest::V3::SupplementingContent.new(url, **supplemental_attributes(file))
     end
 
     def stream_urls
diff --git a/config/application.rb b/config/application.rb
index 3328dce071..8d3b4ff5f5 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -49,6 +49,7 @@ class Application < Rails::Application
         resource '/master_files/*/*.m3u8', headers: :any, credentials: true, methods: [:get, :head]
         resource '/master_files/*/caption_manifest', headers: :any, methods: [:get]
         resource '/master_files/*/captions', headers: :any, methods: [:get]
+        resource '/master_files/*/supplemental_files/*', headers: :any, methods: [:get]
         resource '/timelines/*/manifest.json', headers: :any, methods: [:get, :post]
       end
     end
diff --git a/config/routes.rb b/config/routes.rb
index 2b1f52b58c..621094496b 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -165,7 +165,12 @@
     end
 
     # Supplemental Files
-    resources :supplemental_files, except: [:new, :index, :edit]
+    resources :supplemental_files, except: [:new, :index, :edit] do
+      member do
+        get 'caption', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
+        get 'transcript', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
+      end
+    end
   end
 
   match "iiif_auth_token/:id", to: 'master_files#iiif_auth_token', via: [:get], as: :iiif_auth_token
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 4c5eeb7c22..8a60802b53 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -148,9 +148,14 @@
         expect(subject.any? { |content| content.url =~ /supplemental_files\/#{caption_file.id}/ }).to eq true
       end
 
+      it 'differentiates between transcript and caption files' do
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{transcript_file.id}\/transcript/ }).to eq true
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{caption_file.id}\/caption/ }).to eq true
+      end
+
       it 'does not include generic supplemental files or waveform' do
         expect(subject.any? { |content| content.url =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq false
-        expect(subject.any? { |rendering| rendering["label"]["en"] == ["waveform.json"] }).to eq false
+        expect(subject.any? { |content| content.label == { "en" => ["waveform.json"] } }).to eq false
       end
     end
   end
diff --git a/spec/routing/supplemental_files_routing_spec.rb b/spec/routing/supplemental_files_routing_spec.rb
index b3390daebf..d1933e7ca7 100644
--- a/spec/routing/supplemental_files_routing_spec.rb
+++ b/spec/routing/supplemental_files_routing_spec.rb
@@ -1,11 +1,11 @@
 # Copyright 2011-2023, The Trustees of Indiana University and Northwestern
 #   University.  Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
-# 
+#
 # You may obtain a copy of the License at
-# 
+#
 # http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software distributed
 #   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 #   CONDITIONS OF ANY KIND, either express or implied. See the License for the
@@ -28,5 +28,13 @@
     it "routes to #update" do
       expect(:put => "/master_files/abc1234/supplemental_files/edf567").to route_to("supplemental_files#update", master_file_id: 'abc1234', id: 'edf567')
     end
+    # Redirects are not testable from the routing spec out of the box.
+    # Forcing the tests to `type: :request` to keep routing tests in one place.
+    it "redirects to supplemental_files#show", type: :request do
+      get "/master_files/abc1234/supplemental_files/edf567/caption"
+      expect(response).to redirect_to("/master_files/abc1234/supplemental_files/edf567")
+      get "/master_files/abc1234/supplemental_files/edf567/transcript"
+      expect(response).to redirect_to("/master_files/abc1234/supplemental_files/edf567")
+    end
   end
 end

From faaf6aa924f3e4bccf8e51bf553970c790eafeca Mon Sep 17 00:00:00 2001
From: Mason Ballengee <68433277+masaball@users.noreply.github.com>
Date: Thu, 15 Jun 2023 14:58:26 -0400
Subject: [PATCH 087/396] Fix for codeclimate

---
 app/models/iiif_canvas_presenter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index ccd7c26ba3..50a16f7de2 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -178,7 +178,7 @@ def manifest_attributes(quality, media_type)
     end
 
     def supplemental_attributes(file)
-      supplement_hash = {
+      {
         label: file.label,
         type: 'Text',
         format: file.file.content_type,

From 1fd471ea25870fe8ac3789d5d721095d1ec251ae Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 16 Jun 2023 09:31:37 -0400
Subject: [PATCH 088/396] Add legacy master file captions to manifest

---
 app/models/iiif_canvas_presenter.rb           | 27 +++++++++++++------
 app/models/master_file.rb                     |  1 -
 config/routes.rb                              |  4 +--
 spec/models/iiif_canvas_presenter_spec.rb     | 12 +++++++--
 .../supplemental_files_routing_spec.rb        |  4 +--
 5 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index 50a16f7de2..a2844aaf56 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -89,12 +89,14 @@ def audio_display_content(quality)
     end
 
     def supplementing_content_data(file)
-      url = if file.tags.include?('caption')
-              Rails.application.routes.url_helpers.caption_master_file_supplemental_file_url(master_file.id, file.id)
+      url = if !file.is_a?(SupplementalFile)
+              Rails.application.routes.url_helpers.captions_master_file_url(master_file.id)
+            elsif file.tags.include?('caption')
+              Rails.application.routes.url_helpers.captions_master_file_supplemental_file_url(master_file.id, file.id)
             elsif file.tags.include?('transcript')
-              Rails.application.routes.url_helpers.transcript_master_file_supplemental_file_url(master_file.id, file.id)
+              Rails.application.routes.url_helpers.transcripts_master_file_supplemental_file_url(master_file.id, file.id)
             else
-              Rails.application.routes.url_helpers.master_file_supplemental_file_url(master_file.id, file.id)\
+              Rails.application.routes.url_helpers.master_file_supplemental_file_url(master_file.id, file.id)
             end
       IIIFManifest::V3::SupplementingContent.new(url, **supplemental_attributes(file))
     end
@@ -106,7 +108,7 @@ def stream_urls
     end
 
     def supplemental_captions_transcripts
-      master_file.supplemental_files(tag: 'caption') + master_file.supplemental_files(tag: 'transcript')
+      master_file.supplemental_files(tag: 'caption') + master_file.supplemental_files(tag: 'transcript') + [master_file.captions]
     end
 
     def simple_iiif_range(label = stream_info[:embed_title])
@@ -178,11 +180,20 @@ def manifest_attributes(quality, media_type)
     end
 
     def supplemental_attributes(file)
+      if file.is_a?(SupplementalFile)
+        label = file.label
+        format = file.file.content_type
+        language = file.language || 'en'
+      else
+        label = 'English'
+        format = file.mime_type
+        language = 'en'
+      end
       {
-        label: file.label,
+        label: label,
         type: 'Text',
-        format: file.file.content_type,
-        language: file.language || 'en'
+        format: format,
+        language: language
       }
     end
 
diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index ae8d4267f0..db677838fd 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -527,7 +527,6 @@ def to_solr *args
       solr_doc['has_thumbnail?_bs'] = has_thumbnail?
       solr_doc['has_structuralMetadata?_bs'] = has_structuralMetadata?
       solr_doc['identifier_ssim'] = identifier.map(&:downcase)
-
       solr_doc['percent_complete_ssi'] = percent_complete
       # solr_doc['percent_succeeded_ssi'] =  percent_succeeded
       # solr_doc['percent_failed_ssi'] = percent_failed
diff --git a/config/routes.rb b/config/routes.rb
index 621094496b..0cf7227609 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -167,8 +167,8 @@
     # Supplemental Files
     resources :supplemental_files, except: [:new, :index, :edit] do
       member do
-        get 'caption', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
-        get 'transcript', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
+        get 'captions', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
+        get 'transcripts', :to => redirect('/master_files/%{master_file_id}/supplemental_files/%{id}')
       end
     end
   end
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 8a60802b53..3c85a3a1e6 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -149,14 +149,22 @@
       end
 
       it 'differentiates between transcript and caption files' do
-        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{transcript_file.id}\/transcript/ }).to eq true
-        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{caption_file.id}\/caption/ }).to eq true
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{transcript_file.id}\/transcripts/ }).to eq true
+        expect(subject.any? { |content| content.url =~ /supplemental_files\/#{caption_file.id}\/captions/ }).to eq true
       end
 
       it 'does not include generic supplemental files or waveform' do
         expect(subject.any? { |content| content.url =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq false
         expect(subject.any? { |content| content.label == { "en" => ["waveform.json"] } }).to eq false
       end
+
+      context 'legacy master file captions' do
+        let(:master_file) { FactoryBot.build(:master_file, :with_waveform, :with_captions, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
+
+        it 'includes the master file captions' do
+          expect(subject.any? { |content| content.url =~ /master_file\/#{master_file.id}\/captions/ }).to eq true
+        end
+      end
     end
   end
 
diff --git a/spec/routing/supplemental_files_routing_spec.rb b/spec/routing/supplemental_files_routing_spec.rb
index d1933e7ca7..1e28d4dafb 100644
--- a/spec/routing/supplemental_files_routing_spec.rb
+++ b/spec/routing/supplemental_files_routing_spec.rb
@@ -31,9 +31,9 @@
     # Redirects are not testable from the routing spec out of the box.
     # Forcing the tests to `type: :request` to keep routing tests in one place.
     it "redirects to supplemental_files#show", type: :request do
-      get "/master_files/abc1234/supplemental_files/edf567/caption"
+      get "/master_files/abc1234/supplemental_files/edf567/captions"
       expect(response).to redirect_to("/master_files/abc1234/supplemental_files/edf567")
-      get "/master_files/abc1234/supplemental_files/edf567/transcript"
+      get "/master_files/abc1234/supplemental_files/edf567/transcripts"
       expect(response).to redirect_to("/master_files/abc1234/supplemental_files/edf567")
     end
   end

From b743e0db5a3267bc7db104681134d8884c56dcf4 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 21 Jun 2023 11:47:11 -0400
Subject: [PATCH 089/396] Fix typo in test

---
 spec/models/iiif_canvas_presenter_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 3c85a3a1e6..53e3356ece 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -162,7 +162,7 @@
         let(:master_file) { FactoryBot.build(:master_file, :with_waveform, :with_captions, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
 
         it 'includes the master file captions' do
-          expect(subject.any? { |content| content.url =~ /master_file\/#{master_file.id}\/captions/ }).to eq true
+          expect(subject.any? { |content| content.url =~ /master_files\/#{master_file.id}\/captions/ }).to eq true
         end
       end
     end

From a1bf501336d9eebb6c8b0fddaece947323e169d7 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 12:00:32 -0400
Subject: [PATCH 090/396] More refinement of singular fields

---
 app/presenters/speedy_af/proxy/media_object.rb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index ad2788204a..038775b2d7 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -30,6 +30,8 @@ def initialize(solr_document, instance_defaults = {})
     SINGULAR_FIELDS.each do |field_name|
       @attrs[field_name] = Array(@attrs[field_name]).first
     end
+    # Convert empty strings to nil
+    @attrs.transform_values! { |value| value == "" ? nil : value }
   end
 
   def to_model
@@ -145,11 +147,12 @@ def parse_solr_field(k, v)
     attr_name, type, _stored, _indexed, multi = k.scan(/^(.+)_(.+)(s)(i?)(m?)$/).first
     return [k, v] if attr_name.nil?
     value = Array(v).map { |m| transforms.fetch(type, transforms[nil]).call(m) }
-    value = value.first if !multi || (@model.respond_to?(:properties) && singular?(@model.properties[attr_name]))
+    value = value.first if multi.blank? || singular?(attr_name)
     [attr_name, value]
   end
 
-  def singular?(prop)
-    (prop.present? && prop.respond_to?(:multiple?) && !prop.multiple?) || belongs_to_reflections.values.collect(&:predicate_for_solr)
+  def singular?(attr_name)
+    prop = @model.properties[attr_name]
+    (prop.present? && prop.respond_to?(:multiple?) && !prop.multiple?) || belongs_to_reflections.values.collect(&:predicate_for_solr).include?(attr_name)
   end
 end

From 5b531a99ab5f41cdabe4c2a952316bbc965287bf Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 12:57:32 -0400
Subject: [PATCH 091/396] Fix tests

---
 app/helpers/media_objects_helper.rb            |  2 +-
 app/presenters/speedy_af/proxy/media_object.rb | 10 ++++++++++
 spec/models/checkout_spec.rb                   |  2 +-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/app/helpers/media_objects_helper.rb b/app/helpers/media_objects_helper.rb
index a6f44d32ff..d9c1db206d 100644
--- a/app/helpers/media_objects_helper.rb
+++ b/app/helpers/media_objects_helper.rb
@@ -53,7 +53,7 @@ def dropbox_url collection
 
       def combined_display_date media_object
         (issued,created) = case media_object
-        when MediaObject
+        when MediaObject, SpeedyAF::Proxy::MediaObject
           [media_object.date_issued, media_object.date_created]
         when Hash
           [media_object[:document]['date_ssi'], media_object[:document]['date_created_ssi']]
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index 038775b2d7..ac64f79bd7 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -26,6 +26,7 @@ def initialize(solr_document, instance_defaults = {})
     end
     # Handle this case here until a better fix can be found for multiple solr fields which don't have a model property
     @attrs[:section_id] = solr_document["section_id_ssim"]
+    @attrs[:date_issued] = solr_document["date_ssi"]
     # TODO Need to convert hidden_bsi into discover_groups?
     SINGULAR_FIELDS.each do |field_name|
       @attrs[field_name] = Array(@attrs[field_name]).first
@@ -34,6 +35,15 @@ def initialize(solr_document, instance_defaults = {})
     @attrs.transform_values! { |value| value == "" ? nil : value }
   end
 
+  # Override to skip clearing attrs when reifying to avoid breaking overridden methods which read from attrs
+  def real_object
+    if @real_object.nil?
+      @real_object = model.find(id)
+      # @attrs.clear
+    end
+    @real_object
+  end
+
   def to_model
     self
   end
diff --git a/spec/models/checkout_spec.rb b/spec/models/checkout_spec.rb
index 77c4d16928..fdd0413955 100644
--- a/spec/models/checkout_spec.rb
+++ b/spec/models/checkout_spec.rb
@@ -123,7 +123,7 @@
     let(:checkout) { FactoryBot.create(:checkout, media_object_id: media_object.id) }
 
     it 'returns the checked out MediaObject' do
-      expect(checkout.media_object).to eq media_object
+      expect(checkout.media_object.id).to eq media_object.id
     end
   end
 end

From 41a84be1a27f576c06e3dd2f86b16d6a1fe31aeb Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 12:59:31 -0400
Subject: [PATCH 092/396] Speedily show delete confirmation page

---
 app/views/media_objects/confirm_remove.html.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/media_objects/confirm_remove.html.erb b/app/views/media_objects/confirm_remove.html.erb
index c2f52e7e96..907b51be4f 100644
--- a/app/views/media_objects/confirm_remove.html.erb
+++ b/app/views/media_objects/confirm_remove.html.erb
@@ -26,7 +26,7 @@ Unless required by applicable law or agreed to in writing, software distributed
 					
    Are you certain you want to delete the following <%= 'resource'.pluralize(params[:id].count) %>?
    
 					
      
 						<% params[:id].each do |id| %>
-							<% @media_object = MediaObject.find(id)  %>
+							<% @media_object = SpeedyAF::Proxy::MediaObject.find(id)  %>
 							<% if can? :destroy, @media_object  %>
 								
    • <%= @media_object.title %> (<%= id %>)
      • 
 							<% else %>

From 5235bd8532c3a1fdf308da574d3ceb25632c06df Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 13:02:34 -0400
Subject: [PATCH 093/396] Handle more case statements looking specifically for
 MediaObject

---
 app/helpers/application_helper.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index ec643663ce..de71160ca3 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -29,7 +29,7 @@ def share_link_for(obj, only_path: false)
       obj.permalink
     else
       case obj
-      when MediaObject
+      when MediaObjectBehavior
         if only_path
           media_object_path(obj)
         else
@@ -50,8 +50,8 @@ def lti_share_url_for(obj, _opts = {})
       return I18n.t('share.empty_lti_share_url')
     end
     target = case obj
-             when MediaObject then obj.id
-             when MasterFile then obj.id
+             when MediaObjectBehavior then obj.id
+             when MasterFileBehavior then obj.id
              when Playlist then obj.to_gid_param
              when Timeline then obj.to_gid_param
              end

From 2cf91f37cfe28de68a509e67bfd5ab5225c898c1 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 13:44:43 -0400
Subject: [PATCH 094/396] Allow is_a? checks to work with SpeedyAF

---
 app/helpers/application_helper.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
index de71160ca3..bf44a1a177 100644
--- a/app/helpers/application_helper.rb
+++ b/app/helpers/application_helper.rb
@@ -248,7 +248,7 @@ def parent_layout(layout)
   def object_supplemental_file_path(object, file)
     if object.is_a?(MasterFile) || object.try(:model) == MasterFile
       master_file_supplemental_file_path(master_file_id: object.id, id: file.id)
-    elsif object.is_a? MediaObject
+    elsif object.is_a? MediaObject || object.try(:model) == MediaObject
       media_object_supplemental_file_path(media_object_id: object.id, id: file.id)
     else
       nil
@@ -256,9 +256,9 @@ def object_supplemental_file_path(object, file)
   end
 
   def object_supplemental_files_path(object)
-    if object.is_a? MasterFile
+    if object.is_a?(MasterFile) || object.try(:model) == MasterFile
       master_file_supplemental_files_path(object.id)
-    elsif object.is_a? MediaObject
+    elsif object.is_a? MediaObject || object.try(:model) == MediaObject
       media_object_supplemental_files_path(object.id)
     else
       nil

From 56dfb5ff17fa6477a50d72ba5b2516e0d962b37b Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 14:25:51 -0400
Subject: [PATCH 095/396] Authorize in #show_progress instead of
 load_and_authorize_resource for more control

---
 app/controllers/media_objects_controller.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb
index 8f79ce22c3..0d9996e5ef 100644
--- a/app/controllers/media_objects_controller.rb
+++ b/app/controllers/media_objects_controller.rb
@@ -25,7 +25,7 @@ class MediaObjectsController < ApplicationController
 
   before_action :authenticate_user!, except: [:show, :set_session_quality, :show_stream_details, :manifest]
   before_action :load_resource, except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview, :edit, :update, :json_update]
-  load_and_authorize_resource except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview]
+  load_and_authorize_resource except: [:create, :destroy, :update_status, :set_session_quality, :tree, :deliver_content, :confirm_remove, :show_stream_details, :add_to_playlist_form, :add_to_playlist, :intercom_collections, :manifest, :move_preview, :show_progress]
   authorize_resource only: [:create]
 
   before_action :inject_workflow_steps, only: [:edit, :update], unless: proc { request.format.json? }
@@ -351,6 +351,7 @@ def show_stream_details
   end
 
   def show_progress
+    authorize! :read, @media_object
     overall = { :success => 0, :error => 0 }
     encode_gids = master_file_presenters.collect { |mf| "gid://ActiveEncode/#{mf.encoder_class}/#{mf.workflow_id}" }
     result = Hash[

From 109823d82feee8ffd3509eaa352f20008623e86e Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 14:48:14 -0400
Subject: [PATCH 096/396] Refactor methods into concern for sharing between
 presenter and model

---
 app/models/admin/collection.rb                | 17 +---------
 .../concerns/admin_collection_behavior.rb     | 32 +++++++++++++++++++
 config/initializers/presenter_config.rb       |  5 ++-
 3 files changed, 37 insertions(+), 17 deletions(-)
 create mode 100644 app/models/concerns/admin_collection_behavior.rb

diff --git a/app/models/admin/collection.rb b/app/models/admin/collection.rb
index 2045c307c5..53154167f5 100644
--- a/app/models/admin/collection.rb
+++ b/app/models/admin/collection.rb
@@ -22,6 +22,7 @@ class Admin::Collection < ActiveFedora::Base
   include ActiveFedora::Associations
   include Identifier
   include MigrationTarget
+  include AdminCollectionBehavior
 
   has_many :media_objects, class_name: 'MediaObject', predicate: ActiveFedora::RDF::Fcrepo::RelsExt.isMemberOfCollection
 
@@ -86,10 +87,6 @@ def created_at
     @created_at ||= create_date
   end
 
-  def managers
-    edit_users & ( Avalon::RoleControls.users("manager") | (Avalon::RoleControls.users("administrator") || []) )
-  end
-
   def managers= users
     old_managers = managers
     users.each {|u| add_manager u}
@@ -110,10 +107,6 @@ def remove_manager user
     self.inherited_edit_users -= [user]
   end
 
-  def editors
-    edit_users - managers
-  end
-
   def editors= users
     old_editors = editors
     users.each {|u| add_editor u}
@@ -131,14 +124,6 @@ def remove_editor user
     self.inherited_edit_users -= [user]
   end
 
-  def editors_and_managers
-    edit_users
-  end
-
-  def depositors
-    read_users
-  end
-
   def depositors= users
     old_depositors = depositors
     users.each {|u| add_depositor u}
diff --git a/app/models/concerns/admin_collection_behavior.rb b/app/models/concerns/admin_collection_behavior.rb
new file mode 100644
index 0000000000..6c0acad75e
--- /dev/null
+++ b/app/models/concerns/admin_collection_behavior.rb
@@ -0,0 +1,32 @@
+# Copyright 2011-2023, The Trustees of Indiana University and Northwestern
+#   University.  Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed
+#   under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+#   CONDITIONS OF ANY KIND, either express or implied. See the License for the
+#   specific language governing permissions and limitations under the License.
+# ---  END LICENSE_HEADER BLOCK  ---
+
+# This module contains methods which transform stored values for use either on Admin::Collection or the SpeedyAF presenter
+module AdminCollectionBehavior
+  def managers
+    edit_users & ( Avalon::RoleControls.users("manager") | (Avalon::RoleControls.users("administrator") || []) )
+  end
+
+  def editors
+    edit_users - managers
+  end
+
+  def editors_and_managers
+    edit_users
+  end
+
+  def depositors
+    read_users
+  end
+end
diff --git a/config/initializers/presenter_config.rb b/config/initializers/presenter_config.rb
index 699f0e3ffc..28dfae3168 100644
--- a/config/initializers/presenter_config.rb
+++ b/config/initializers/presenter_config.rb
@@ -55,8 +55,11 @@
 
     sp.config Admin::Collection do
       self.defaults = {
-                        cdl_enabled: nil
+                        cdl_enabled: nil,
+                        read_users: [],
+                        edit_users: []
                       }
+      include AdminCollectionBehavior
     end
 
     sp.config Derivative do

From ae8e617a7a6ddce913cfe5727e70ca7308a93b82 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 21 Jun 2023 16:39:04 -0400
Subject: [PATCH 097/396] Test that fedora isn't called and final fixes to get
 tests to pass

---
 app/models/concerns/master_file_intercom.rb   |  4 +-
 config/initializers/presenter_config.rb       | 24 +++++-
 .../master_files_controller_spec.rb           | 86 ++++++++++++++++++-
 .../media_objects_controller_spec.rb          | 31 +++++++
 4 files changed, 140 insertions(+), 5 deletions(-)

diff --git a/app/models/concerns/master_file_intercom.rb b/app/models/concerns/master_file_intercom.rb
index 107fcc121b..eafd89bd72 100644
--- a/app/models/concerns/master_file_intercom.rb
+++ b/app/models/concerns/master_file_intercom.rb
@@ -33,8 +33,8 @@ def to_ingest_api_hash(include_structure = true, remove_identifiers: false)
       file_checksum: file_checksum,
       file_format: file_format,
       other_identifier: (remove_identifiers ? [] : identifier.to_a),
-      captions: captions.content,
-      captions_type: captions.mime_type,
+      captions: captions&.content,
+      captions_type: captions&.mime_type,
       supplemental_file_captions: supplemental_file_captions,
       comment: comment.to_a,
       display_aspect_ratio: display_aspect_ratio,
diff --git a/config/initializers/presenter_config.rb b/config/initializers/presenter_config.rb
index 28dfae3168..e1e2e25dac 100644
--- a/config/initializers/presenter_config.rb
+++ b/config/initializers/presenter_config.rb
@@ -9,7 +9,13 @@
                         comment: [],
                         supplemental_files_json: nil,
                         width: nil,
-                        height: nil
+                        height: nil,
+                        physical_description: nil,
+                        file_size: nil,
+                        date_digitized: nil,
+                        file_checksum: nil,
+                        captions: nil,
+                        supplemental_file_captions: nil
                       }
       include MasterFileIntercom
       include MasterFileBehavior
@@ -34,6 +40,7 @@
                         contributor: [],
                         publisher: [],
                         temporal_subject: [],
+                        topical_subject: [],
                         geographic_subject: [],
                         language: [],
                         terms_of_use: nil,
@@ -45,7 +52,14 @@
                         table_of_contents: [],
                         bibliographic_id: nil,
                         comment: [],
-                        date_issued: nil
+                        date_issued: nil,
+                        avalon_uploader: nil,
+                        identifier: [],
+                        alternative_title: [],
+                        translated_title: [],
+                        uniform_title: [],
+                        resource_type: [],
+                        record_identifier: []
                       }
       include VirtualGroups
       include MediaObjectIntercom
@@ -63,6 +77,12 @@
     end
 
     sp.config Derivative do
+      self.defaults = {
+                        track_id: nil,
+                        mime_type: nil,
+                        hls_track_id: nil 
+                      }
+      include DerivativeIntercom
       include DerivativeBehavior
     end
 
diff --git a/spec/controllers/master_files_controller_spec.rb b/spec/controllers/master_files_controller_spec.rb
index 2572f70626..bd1e695d27 100644
--- a/spec/controllers/master_files_controller_spec.rb
+++ b/spec/controllers/master_files_controller_spec.rb
@@ -16,6 +16,7 @@
 require 'equivalent-xml'
 
 describe MasterFilesController do
+  include ActiveJob::TestHelper
 
   describe "#create" do
     let(:media_object) { FactoryBot.create(:media_object) }
@@ -294,6 +295,15 @@ class << file
         expect(get(:embed, params: { id: fedora3_pid })).to redirect_to(embed_master_file_url(master_file.id))
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get(:embed, params: { id: master_file.id })
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#set_frame" do
@@ -357,6 +367,16 @@ class << file
         expect(response.headers['Content-Type']).to eq('image/jpeg')
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        mf
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get :get_frame, params: { id: mf.id, type: 'thumbnail', size: 'bar' }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   # rubocop:disable RSpec/ExampleLength
@@ -425,6 +445,16 @@ class << file
       expect(JSON.parse(response.body)).to eq(JSON.parse(structure_json))
       expect(response.headers['Content-Type']).to eq('application/json; charset=utf-8')
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        master_file
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get 'structure', params: { id: master_file.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#set_structure" do
@@ -471,6 +501,17 @@ class << file
       login_as :administrator
       expect(get('captions', params: { id: master_file.id })).to have_http_status(:ok)
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        master_file
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        login_as :administrator
+        get('captions', params: { id: master_file.id })
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#waveform" do
@@ -505,6 +546,17 @@ class << file
         expect(response['Content-Disposition']).to eq("attachment; filename=\"empty_waveform.json\"; filename*=UTF-8''empty_waveform.json")
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        master_file
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        login_as :administrator
+        get('waveform', params: { id: master_file.id })
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe '#iiif_auth_token' do
@@ -575,6 +627,17 @@ class << file
     it 'returns a manifest if public' do
       expect(get('hls_manifest', params: { id: public_master_file.id, quality: 'auto' })).to have_http_status(:ok)
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        master_file
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        login_as :administrator
+        get('hls_manifest', params: { id: master_file.id, quality: 'high' })
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe '#caption_manifest' do
@@ -606,6 +669,17 @@ class << file
     it 'returns a manifest if public' do
       expect(get('caption_manifest', params: { id: public_master_file.id }, xhr: true)).to have_http_status(:ok)
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        public_master_file
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        login_as :administrator
+        get('caption_manifest', params: { id: public_master_file.id }, xhr: true)
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe '#move' do
@@ -698,6 +772,16 @@ class << file
       expect(response).to have_http_status(:ok)
       expect(response.body.include? "Example captions").to be_truthy
     end
-  end
 
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        master_file
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        login_as :administrator
+        get('transcript', params: { use_route: 'master_files/:id/transcript', id: master_file.id, t_id: supplemental_file.id })
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
+  end
 end
diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb
index aad72c013f..5e4c31966f 100644
--- a/spec/controllers/media_objects_controller_spec.rb
+++ b/spec/controllers/media_objects_controller_spec.rb
@@ -1238,6 +1238,15 @@
           expect(json['files'].first['structure']).not_to eq master_file.structuralMetadata.content
         end
       end
+
+      context 'read from solr' do
+	it 'should not read from fedora' do
+	  perform_enqueued_jobs(only: MediaObjectIndexingJob)
+	  WebMock.reset_executed_requests!
+          get 'show', params: { id: media_object.id, format:'json' }
+	  expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+	end
+      end
     end
 
     context 'misformed NOID' do
@@ -1251,6 +1260,16 @@
         expect(response.response_code).to eq(404)
       end
     end
+
+    context 'read from solr' do
+      let!(:media_object) { FactoryBot.create(:published_media_object, :with_master_file, visibility: 'public') }
+      it 'should not read from fedora' do
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get 'show', params: { id: media_object.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#destroy" do
@@ -1784,4 +1803,16 @@
       end
     end
   end
+
+  describe '#manifest' do
+    context 'read from solr' do
+      let!(:media_object) { FactoryBot.create(:published_media_object, :with_master_file, visibility: 'public') }
+      it 'should not read from fedora' do
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get 'manifest', params: { id: media_object.id, format: 'json' }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
+  end
 end

From 140c1a5e52a97354894ba19445d6b3953ed389f9 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 23 Jun 2023 17:09:50 -0400
Subject: [PATCH 098/396] Fix failing tests

---
 app/models/concerns/master_file_intercom.rb   |  3 ++-
 app/models/derivative.rb                      |  8 +++++--
 .../speedy_af/proxy/admin/collection.rb       | 14 +++++++++++
 .../speedy_af/proxy/media_object.rb           | 11 +--------
 config/initializers/presenter_config.rb       | 24 ++++++++++++++-----
 5 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/app/models/concerns/master_file_intercom.rb b/app/models/concerns/master_file_intercom.rb
index eafd89bd72..c0cfdc2427 100644
--- a/app/models/concerns/master_file_intercom.rb
+++ b/app/models/concerns/master_file_intercom.rb
@@ -34,7 +34,8 @@ def to_ingest_api_hash(include_structure = true, remove_identifiers: false)
       file_format: file_format,
       other_identifier: (remove_identifiers ? [] : identifier.to_a),
       captions: captions&.content,
-      captions_type: captions&.mime_type,
+      # Captions (and all IndexedFile's) will return 'text/plain' when there isn't content including when it isn't persisted yet
+      captions_type: captions.try(:persisted?) ? captions&.mime_type : nil,
       supplemental_file_captions: supplemental_file_captions,
       comment: comment.to_a,
       display_aspect_ratio: display_aspect_ratio,
diff --git a/app/models/derivative.rb b/app/models/derivative.rb
index 4efb824daa..6eaf53f2e5 100644
--- a/app/models/derivative.rb
+++ b/app/models/derivative.rb
@@ -31,8 +31,12 @@ class Derivative < ActiveFedora::Base
   property :duration, predicate: ::RDF::Vocab::EBUCore.duration, multiple: false do |index|
     index.as :stored_sortable
   end
-  property :track_id, predicate: Avalon::RDFVocab::EBUCore.identifier, multiple: false
-  property :hls_track_id, predicate: Avalon::RDFVocab::Derivative.hlsTrackID, multiple: false
+  property :track_id, predicate: Avalon::RDFVocab::EBUCore.identifier, multiple: false do |index|
+    index.as :displayable
+  end
+  property :hls_track_id, predicate: Avalon::RDFVocab::Derivative.hlsTrackID, multiple: false do |index|
+    index.as :displayable
+  end
   property :managed, predicate: Avalon::RDFVocab::Derivative.isManaged, multiple: false do |index|
     index.as ActiveFedora::Indexing::Descriptor.new(:boolean, :stored, :indexed)
   end
diff --git a/app/presenters/speedy_af/proxy/admin/collection.rb b/app/presenters/speedy_af/proxy/admin/collection.rb
index 09608afb6a..9c60c8ed08 100644
--- a/app/presenters/speedy_af/proxy/admin/collection.rb
+++ b/app/presenters/speedy_af/proxy/admin/collection.rb
@@ -13,6 +13,20 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class SpeedyAF::Proxy::Admin::Collection < SpeedyAF::Base
+  # Override to handle section_id specially
+  def initialize(solr_document, instance_defaults = {})
+    instance_defaults ||= {}
+    @model = SpeedyAF::Base.model_for(solr_document)
+    @attrs = self.class.defaults.merge(instance_defaults)
+    solr_document.each_pair do |k, v|
+      attr_name, value = parse_solr_field(k, v)
+      @attrs[attr_name.to_sym] = value
+    end
+    # Handle this case here until a better fix can be found for multiple solr fields which don't have a model property
+    @attrs[:read_users] = solr_document["read_access_person_ssim"] || []
+    @attrs[:edit_users] = solr_document["edit_access_person_ssim"] || []
+  end
+
   def to_model
     self
   end
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index ac64f79bd7..45acb872fe 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -21,7 +21,7 @@ def initialize(solr_document, instance_defaults = {})
     @model = SpeedyAF::Base.model_for(solr_document)
     @attrs = self.class.defaults.merge(instance_defaults)
     solr_document.each_pair do |k, v|
-     attr_name, value = parse_solr_field(k, v)
+      attr_name, value = parse_solr_field(k, v)
       @attrs[attr_name.to_sym] = value
     end
     # Handle this case here until a better fix can be found for multiple solr fields which don't have a model property
@@ -35,15 +35,6 @@ def initialize(solr_document, instance_defaults = {})
     @attrs.transform_values! { |value| value == "" ? nil : value }
   end
 
-  # Override to skip clearing attrs when reifying to avoid breaking overridden methods which read from attrs
-  def real_object
-    if @real_object.nil?
-      @real_object = model.find(id)
-      # @attrs.clear
-    end
-    @real_object
-  end
-
   def to_model
     self
   end
diff --git a/config/initializers/presenter_config.rb b/config/initializers/presenter_config.rb
index e1e2e25dac..9aa746da6f 100644
--- a/config/initializers/presenter_config.rb
+++ b/config/initializers/presenter_config.rb
@@ -14,8 +14,7 @@
                         file_size: nil,
                         date_digitized: nil,
                         file_checksum: nil,
-                        captions: nil,
-                        supplemental_file_captions: nil
+                        supplemental_file_captions: []
                       }
       include MasterFileIntercom
       include MasterFileBehavior
@@ -69,9 +68,7 @@
 
     sp.config Admin::Collection do
       self.defaults = {
-                        cdl_enabled: nil,
-                        read_users: [],
-                        edit_users: []
+                        cdl_enabled: nil
                       }
       include AdminCollectionBehavior
     end
@@ -80,7 +77,11 @@
       self.defaults = {
                         track_id: nil,
                         mime_type: nil,
-                        hls_track_id: nil 
+                        hls_track_id: nil,
+                        video_bitrate: nil,
+                        video_codec: nil,
+                        audio_bitrate: nil,
+                        audio_codec: nil
                       }
       include DerivativeIntercom
       include DerivativeBehavior
@@ -100,4 +101,15 @@ def xpath(*args)
       end
     end
   end
+
+  SpeedyAF::Base.class_eval do
+    # Override to skip clearing attrs when reifying to avoid breaking overridden methods which read from attrs
+    def real_object
+      if @real_object.nil?
+	@real_object = model.find(id)
+	# @attrs.clear
+      end
+      @real_object
+    end
+  end
 end

From 5585f2f76117e933b3e15cd9a9d0a700c0fbd35c Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Fri, 23 Jun 2023 17:38:47 -0400
Subject: [PATCH 099/396] Don't reify if captions not found

---
 app/presenters/speedy_af/proxy/master_file.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/presenters/speedy_af/proxy/master_file.rb b/app/presenters/speedy_af/proxy/master_file.rb
index 386614d326..2ad2710ef5 100644
--- a/app/presenters/speedy_af/proxy/master_file.rb
+++ b/app/presenters/speedy_af/proxy/master_file.rb
@@ -60,4 +60,8 @@ def supplemental_files(tag: '*')
       files.select { |file| Array(tag).all? { |t| file.tags.include?(t) } }
     end
   end
+
+  def captions
+    load_subresource_content(:captions) rescue nil
+  end
 end

From 4370a2dd0f5c70690786bfc903ad13bdb1ce6f05 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Mon, 26 Jun 2023 16:40:46 -0400
Subject: [PATCH 100/396] Use the configured protocol of embed url instead of
 //

---
 app/models/concerns/master_file_behavior.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/concerns/master_file_behavior.rb b/app/models/concerns/master_file_behavior.rb
index 77854b8ff7..7da6527660 100644
--- a/app/models/concerns/master_file_behavior.rb
+++ b/app/models/concerns/master_file_behavior.rb
@@ -124,7 +124,7 @@ def embed_code(width, permalink_opts = {})
       url = if self.permalink.present?
         self.permalink_with_query(permalink_opts)
       else
-        embed_master_file_url(self.id, only_path: false, protocol: '//')
+        embed_master_file_url(self.id)
       end
       height = is_video? ? (width/display_aspect_ratio.to_f).floor : AUDIO_HEIGHT
       ""

From 38a08380aa8b25b8b27e0e59b70fc4c192da5777 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Mon, 26 Jun 2023 17:22:51 -0400
Subject: [PATCH 101/396] Avoid reading from fedora when loading playlist show
 view

---
 app/models/avalon_annotation.rb               |  2 +-
 .../playlist_items/_current_item.html.erb     |  4 +--
 app/views/playlists/_info.html.erb            |  4 +--
 app/views/playlists/refresh_info.js.erb       |  4 +--
 .../playlist_items_controller_spec.rb         | 31 +++++++++++++++++++
 spec/controllers/playlists_controller_spec.rb | 19 ++++++++++++
 6 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/app/models/avalon_annotation.rb b/app/models/avalon_annotation.rb
index 4cffe27073..545ae13cd3 100644
--- a/app/models/avalon_annotation.rb
+++ b/app/models/avalon_annotation.rb
@@ -58,7 +58,7 @@ def title_default!
 
   # Sets the class variable @master_file by finding the master referenced in the source uri
   def master_file
-    @master_file ||= MasterFile.find(CGI::unescape(self.source.split('/').last)) rescue nil if self.source
+    @master_file ||= SpeedyAF::Proxy::MasterFile.find(CGI::unescape(self.source.split('/').last)) if self.source
   end
 
   def master_file=(value)
diff --git a/app/views/playlist_items/_current_item.html.erb b/app/views/playlist_items/_current_item.html.erb
index cce75c27f1..ff891c61e2 100644
--- a/app/views/playlist_items/_current_item.html.erb
+++ b/app/views/playlist_items/_current_item.html.erb
@@ -15,8 +15,8 @@ Unless required by applicable law or agreed to in writing, software distributed
 %>
 <% @current_playlist_item = @playlist_item %>
 <% @current_clip = AvalonClip.find(@current_playlist_item.clip_id) %>
-<% @current_masterfile = MasterFile.find(@current_playlist_item.clip.source.split('/').last) %>
-<% @current_mediaobject = MediaObject.find(@current_masterfile.media_object_id) %>
+<% @current_masterfile = SpeedyAF::Proxy::MasterFile.find(@current_playlist_item.clip.source.split('/').last) %>
+<% @current_mediaobject = SpeedyAF::Proxy::MediaObject.find(@current_masterfile.media_object_id) %>
 
      
   
      
     <%= link_to master_file_path(@current_masterfile.id) do %>
diff --git a/app/views/playlists/_info.html.erb b/app/views/playlists/_info.html.erb
index afac78fcb5..ae7a002170 100644
--- a/app/views/playlists/_info.html.erb
+++ b/app/views/playlists/_info.html.erb
@@ -15,8 +15,8 @@ Unless required by applicable law or agreed to in writing, software distributed
 %>
 <%# @current_playlist_item ||= @playlist_item %>
 <%# @current_clip ||= AvalonClip.find(@current_playlist_item.clip_id) %>
-<%# @current_masterfile ||= MasterFile.find(@current_playlist_item.clip.source.split('/').last) %>
-<%# @current_mediaobject ||= MediaObject.find(@current_masterfile.media_object_id) %>
+<%# @current_masterfile ||= SpeedyAF::Proxy::MasterFile.find(@current_playlist_item.clip.source.split('/').last) %>
+<%# @current_mediaobject ||= SpeedyAF::Proxy::MediaObject.find(@current_masterfile.media_object_id) %>
 
 <% if can? :read, @current_masterfile %>
 
diff --git a/app/views/playlists/refresh_info.js.erb b/app/views/playlists/refresh_info.js.erb
index 53c3edbecf..f8cac4afe9 100644
--- a/app/views/playlists/refresh_info.js.erb
+++ b/app/views/playlists/refresh_info.js.erb
@@ -15,8 +15,8 @@ Unless required by applicable law or agreed to in writing, software distributed
 %>
 <% @current_playlist_item = @playlist_item %>
 <% @current_clip = AvalonClip.find(@current_playlist_item.clip_id) %>
-<% @current_masterfile = MasterFile.find(@current_playlist_item.clip.source.split('/').last) %>
-<% @current_mediaobject = MediaObject.find(@current_masterfile.media_object_id) %>
+<% @current_masterfile = SpeedyAF::Proxy::MasterFile.find(@current_playlist_item.clip.source.split('/').last) %>
+<% @current_mediaobject = SpeedyAF::Proxy::MediaObject.find(@current_masterfile.media_object_id) %>
 <% clip_start = @current_clip.start_time / 1000.0 %>
 <% clip_end = @current_clip.end_time / 1000.0 %>
 <% clip_frag = "?t=#{clip_start},#{clip_end}" %>
diff --git a/spec/controllers/playlist_items_controller_spec.rb b/spec/controllers/playlist_items_controller_spec.rb
index 5549411ec8..ec11e1af91 100644
--- a/spec/controllers/playlist_items_controller_spec.rb
+++ b/spec/controllers/playlist_items_controller_spec.rb
@@ -179,6 +179,16 @@
       expect(response).to have_http_status(:ok)
       expect(response).to render_template(:_current_item)
     end
+
+    context 'read from solr' do
+      render_views
+      it 'should not read from fedora' do
+        playlist_item
+        WebMock.reset_executed_requests!
+        get :source_details, params: { playlist_id: playlist.to_param, playlist_item_id: playlist_item.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe 'GET #markers' do
@@ -187,6 +197,16 @@
       expect(response).to have_http_status(:ok)
       expect(response).to render_template(:_markers)
     end
+
+    context 'read from solr' do
+      render_views
+      it 'should not read from fedora' do
+        playlist_item
+        WebMock.reset_executed_requests!
+        get :markers, params: { playlist_id: playlist.to_param, playlist_item_id: playlist_item.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe 'GET #related_items' do
@@ -196,5 +216,16 @@
       expect(response).to have_http_status(:ok)
       expect(response).to render_template(:_related_items)
     end
+
+    context 'read from solr' do
+      render_views
+      it 'should not read from fedora' do
+        playlist_item
+        WebMock.reset_executed_requests!
+        allow_any_instance_of(Playlist).to receive(:related_clips).and_return([clip]);
+        get :related_items, params: { playlist_id: playlist.to_param, playlist_item_id: playlist_item.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 end
diff --git a/spec/controllers/playlists_controller_spec.rb b/spec/controllers/playlists_controller_spec.rb
index a9b2373c9a..2ad2e0e8e4 100644
--- a/spec/controllers/playlists_controller_spec.rb
+++ b/spec/controllers/playlists_controller_spec.rb
@@ -33,6 +33,8 @@
 # that an instance is receiving a specific message.
 
 RSpec.describe PlaylistsController, type: :controller do
+  include ActiveJob::TestHelper
+
   # This should return the minimal set of attributes required to create a valid
   # Playlist. As you add validations to Playlist, be sure to
   # adjust the attributes here as well.
@@ -151,6 +153,23 @@
       expect(assigns(:playlist)).to eq(playlist)
     end
     # TODO: write tests for public/private playists
+
+    context 'read from solr' do
+      render_views
+
+      let!(:playlist) { FactoryBot.create(:playlist, items: [playlist_item], visibility: Playlist::PUBLIC) }
+      let(:playlist_item) { FactoryBot.create(:playlist_item, clip: clip) }
+      let(:clip) { FactoryBot.create(:avalon_clip, master_file: master_file) }
+      let(:master_file) { FactoryBot.create(:master_file, media_object: media_object) }
+      let(:media_object) { FactoryBot.create(:published_media_object, visibility: 'public') }
+
+      it 'should not read from fedora' do
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get :show, params: { id: playlist.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe 'GET #new' do

From 0da085a90ad042dd88e3ce34f2b15c6da692364c Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 27 Jun 2023 11:42:23 -0400
Subject: [PATCH 102/396] Avoid reifying when to_key is called on MediaObject

---
 app/presenters/speedy_af/proxy/media_object.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index 45acb872fe..93863918ca 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -51,6 +51,10 @@ def to_param
     id
   end
 
+  def to_key
+    [id]
+  end
+
   # @return [SupplementalFile]
   def supplemental_files(tag: '*')
     return [] if supplemental_files_json.blank?

From ee0c0663364dc148c4c65558e13d1b695fc597c2 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 27 Jun 2023 11:55:05 -0400
Subject: [PATCH 103/396] Use SpeedyAF proxy for MediaObject tree route

---
 app/controllers/media_objects_controller.rb       |  2 +-
 spec/controllers/media_objects_controller_spec.rb | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb
index 0d9996e5ef..0c368a8905 100644
--- a/app/controllers/media_objects_controller.rb
+++ b/app/controllers/media_objects_controller.rb
@@ -454,7 +454,7 @@ def update_status
   # Sets the published status for the object. If no argument is given then
   # it will just toggle the state.
   def tree
-    @media_object = MediaObject.find(params[:id])
+    @media_object = SpeedyAF::Proxy::MediaObject.find(params[:id])
     authorize! :inspect, @media_object
 
     respond_to do |format|
diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb
index 5e4c31966f..4840a0507a 100644
--- a/spec/controllers/media_objects_controller_spec.rb
+++ b/spec/controllers/media_objects_controller_spec.rb
@@ -1815,4 +1815,17 @@
       end
     end
   end
+
+  describe '#tree' do
+    context 'read from solr' do
+      let!(:media_object) { FactoryBot.create(:published_media_object, :with_master_file, visibility: 'public') }
+      it 'should not read from fedora' do
+        login_as(:administrator)
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get 'tree', params: { id: media_object.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
+  end
 end

From ac9c4a9deae88170bbc0051569845eb5f115fe2d Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 27 Jun 2023 13:51:04 -0400
Subject: [PATCH 104/396] Use SpeedyAF proxies for add to playlist routes

---
 app/controllers/media_objects_controller.rb   |  6 ++---
 app/models/concerns/master_file_behavior.rb   | 14 ++++++++++++
 app/models/master_file.rb                     | 14 ------------
 .../media_objects_controller_spec.rb          | 22 ++++++++++++++++++-
 4 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/app/controllers/media_objects_controller.rb b/app/controllers/media_objects_controller.rb
index 0d9996e5ef..9b9703e733 100644
--- a/app/controllers/media_objects_controller.rb
+++ b/app/controllers/media_objects_controller.rb
@@ -109,7 +109,7 @@ def new
 
   # POST /media_objects/avalon:1/add_to_playlist_form
   def add_to_playlist_form
-    @media_object = MediaObject.find(params[:id])
+    @media_object = SpeedyAF::Proxy::MediaObject.find(params[:id])
     authorize! :read, @media_object
     respond_to do |format|
       format.html do
@@ -120,7 +120,7 @@ def add_to_playlist_form
 
   # POST /media_objects/avalon:1/add_to_playlist
   def add_to_playlist
-    @media_object = MediaObject.find(params[:id])
+    @media_object = SpeedyAF::Proxy::MediaObject.find(params[:id])
     authorize! :read, @media_object
     masterfile_id = params[:post][:masterfile_id]
     playlist_id = params[:post][:playlist_id]
@@ -132,7 +132,7 @@ def add_to_playlist
     # If a single masterfile_id wasn't in the request, then create playlist_items for all masterfiles
     masterfile_ids = masterfile_id.present? ? [masterfile_id] : @media_object.ordered_master_file_ids
     masterfile_ids.each do |mf_id|
-      mf = MasterFile.find(mf_id)
+      mf = SpeedyAF::Proxy::MasterFile.find(mf_id)
       if playlistitem_scope=='structure' && mf.has_structuralMetadata? && mf.structuralMetadata.xpath('//Span').present?
         #create individual items for spans within structure
         mf.structuralMetadata.xpath('//Span').each do |s|
diff --git a/app/models/concerns/master_file_behavior.rb b/app/models/concerns/master_file_behavior.rb
index 77854b8ff7..56fcf53f54 100644
--- a/app/models/concerns/master_file_behavior.rb
+++ b/app/models/concerns/master_file_behavior.rb
@@ -162,4 +162,18 @@ def build_caption_hash(caption)
       label: label
     }
   end
+
+  # Supplies the route to the master_file as an rdf formatted URI
+  # @return [String] the route as a uri
+  # @example uri for a mf on avalon.iu.edu with a id of: avalon:1820
+  #   "my_master_file.rdf_uri" #=> "https://www.avalon.iu.edu/master_files/avalon:1820"
+  def rdf_uri
+    master_file_url(id)
+  end
+
+  # Returns the dctype of the master_file
+  # @return [String] either 'dctypes:MovingImage' or 'dctypes:Sound'
+  def rdf_type
+    is_video? ? 'dctypes:MovingImage' : 'dctypes:Sound'
+  end
 end
diff --git a/app/models/master_file.rb b/app/models/master_file.rb
index ae8d4267f0..6e2c66b8b8 100644
--- a/app/models/master_file.rb
+++ b/app/models/master_file.rb
@@ -461,20 +461,6 @@ def structural_metadata_labels
     structuralMetadata.xpath('//@label').collect{|a|a.value}
   end
 
-  # Supplies the route to the master_file as an rdf formatted URI
-  # @return [String] the route as a uri
-  # @example uri for a mf on avalon.iu.edu with a id of: avalon:1820
-  #   "my_master_file.rdf_uri" #=> "https://www.avalon.iu.edu/master_files/avalon:1820"
-  def rdf_uri
-    master_file_url(id)
-  end
-
-  # Returns the dctype of the master_file
-  # @return [String] either 'dctypes:MovingImage' or 'dctypes:Sound'
-  def rdf_type
-    is_video? ? 'dctypes:MovingImage' : 'dctypes:Sound'
-  end
-
   def self.post_processing_move_filename(oldpath, options = {})
     prefix = options[:id].tr(':', '_')
     if File.basename(oldpath).start_with?(prefix)
diff --git a/spec/controllers/media_objects_controller_spec.rb b/spec/controllers/media_objects_controller_spec.rb
index 5e4c31966f..bc58e473ae 100644
--- a/spec/controllers/media_objects_controller_spec.rb
+++ b/spec/controllers/media_objects_controller_spec.rb
@@ -1693,6 +1693,16 @@
       get :add_to_playlist_form, params: { id: media_object.id, scope: 'media_object', masterfile_id: media_object.ordered_master_file_ids[0] }
       expect(response.body).to include('Add Item to Playlist')
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        media_object
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        get :add_to_playlist_form, params: { id: media_object.id, scope: 'media_object', masterfile_id: media_object.ordered_master_file_ids[0] }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#add_to_playlist" do
@@ -1741,6 +1751,16 @@
       expect(response).to have_http_status(403)
       expect(JSON.parse(response.body).symbolize_keys).to eq({message: "
      You are not authorized to update this playlist.
      ", status: 403})
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        media_object
+        perform_enqueued_jobs(only: MediaObjectIndexingJob)
+        WebMock.reset_executed_requests!
+        post :add_to_playlist, params: { id: media_object.id, post: { playlist_id: playlist.id, masterfile_id: media_object.ordered_master_file_ids[0], playlistitem_scope: 'section' } }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe 'deliver_content' do
@@ -1796,7 +1816,7 @@
 
       let(:media_object) { FactoryBot.create(:published_media_object) }
 
-      it 'returns a json preview of the media object' do
+      it 'returns unauthorized' do
         get :move_preview, params: { id: media_object.id, format: 'json' }
         expect(response.status).to eq 401
         expect(response.content_type).to eq 'application/json'

From 6aacf21cf02d620cac23477ce8f10c128d1890e2 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 27 Jun 2023 14:59:55 -0400
Subject: [PATCH 105/396] Use SpeedyAF proxies for BookmarksController actions

---
 app/controllers/bookmarks_controller.rb       | 21 +++----
 app/models/ability.rb                         | 62 +++++++++----------
 spec/controllers/bookmarks_controller_spec.rb | 56 +++++++++++++++++
 3 files changed, 97 insertions(+), 42 deletions(-)

diff --git a/app/controllers/bookmarks_controller.rb b/app/controllers/bookmarks_controller.rb
index c08fca68d3..38c4627397 100644
--- a/app/controllers/bookmarks_controller.rb
+++ b/app/controllers/bookmarks_controller.rb
@@ -68,9 +68,8 @@ def verify_permissions
     @response, @documents = action_documents
     @valid_user_actions = [:delete, :unpublish, :publish, :merge, :move, :update_access_control, :add_to_playlist]
     @valid_user_actions += [:intercom_push] if Settings.intercom.present?
-    mos = @documents.collect { |doc| MediaObject.find( doc.id ) }
     @documents.each do |doc|
-      mo = MediaObject.find(doc.id)
+      mo = SpeedyAF::Proxy::MediaObject.find(doc.id)
       @valid_user_actions.delete :delete if @valid_user_actions.include? :delete and cannot? :destroy, mo
       @valid_user_actions.delete :unpublish if @valid_user_actions.include? :unpublish and cannot? :unpublish, mo
       @valid_user_actions.delete :publish if @valid_user_actions.include? :publish and cannot? :update, mo
@@ -124,7 +123,7 @@ def access_control_action documents
     errors = []
     success_ids = []
     Array(documents.map(&:id)).each do |id|
-      media_object = MediaObject.find(id)
+      media_object = SpeedyAF::Proxy::MediaObject.find(id)
       if cannot? :update_access_control, media_object
         errors += ["#{media_object.title} (#{id}) #{t('blacklight.messages.permission_denied')}."]
       else
@@ -141,7 +140,7 @@ def access_control_action documents
   def add_to_playlist_action documents
     playlist = Playlist.find(params[:target_playlist_id])
     Array(documents.map(&:id)).each do |id|
-      media_object = MediaObject.find(id)
+      media_object = SpeedyAF::Proxy::MediaObject.find(id)
       media_object.ordered_master_files.to_a.each do |mf|
         clip = AvalonClip.create(master_file: mf)
         PlaylistItem.create(clip: clip, playlist: playlist)
@@ -152,7 +151,7 @@ def add_to_playlist_action documents
   def status_action documents
     errors, success_ids = [], [], []
     Array(documents.map(&:id)).each do |id|
-      media_object = MediaObject.find(id)
+      media_object = SpeedyAF::Proxy::MediaObject.find(id)
       if cannot? :update, media_object
         errors += ["#{media_object.title} (#{id}) #{t('blacklight.messages.permission_denied')}."]
       else
@@ -181,7 +180,7 @@ def delete_action documents
     errors = []
     success_ids = []
     Array(documents.map(&:id)).each do |id|
-      media_object = MediaObject.find(id)
+      media_object = SpeedyAF::Proxy::MediaObject.find(id)
       if can? :destroy, media_object
         success_ids << id
       else
@@ -194,14 +193,14 @@ def delete_action documents
   end
 
   def move_action documents
-    collection = Admin::Collection.find( params[:target_collection_id] )
+    collection = SpeedyAF::Proxy::Admin::Collection.find( params[:target_collection_id] )
     if cannot? :read, collection
       flash[:error] =  t("blacklight.move.error", collection_name: collection.name)
     else
       errors = []
       success_ids = []
       Array(documents.map(&:id)).each do |id|
-        media_object = MediaObject.find(id)
+        media_object = SpeedyAF::Proxy::MediaObject.find(id)
         if cannot? :update, media_object
           errors += ["#{media_object.title} (#{id}) #{t('blacklight.messages.permission_denied')}."]
         else
@@ -222,7 +221,7 @@ def intercom_push_action documents
     session[:intercom_collections] = collections
     if intercom.collection_valid?(params[:collection_id])
       Array(documents.map(&:id)).each do |id|
-        media_object = MediaObject.find(id)
+        media_object = SpeedyAF::Proxy::MediaObject.find(id)
         if cannot? :intercom_push, media_object
           errors += ["#{media_object.title} (#{id}) #{t('blacklight.messages.permission_denied')}."]
         else
@@ -242,10 +241,10 @@ def intercom_push_action documents
 
   def merge_action documents
     errors = []
-    target = MediaObject.find params[:media_object]
+    target = SpeedyAF::Proxy::MediaObject.find params[:media_object]
     subject_ids = documents.collect(&:id)
     subject_ids.delete(target.id)
-    subject_ids.map { |id| MediaObject.find id }.each do |media_object|
+    subject_ids.map { |id| SpeedyAF::Proxy::MediaObject.find id }.each do |media_object|
       if cannot? :destroy, media_object
         errors += ["#{media_object.title || id} #{t('blacklight.messages.permission_denied')}."]
       end
diff --git a/app/models/ability.rb b/app/models/ability.rb
index 2b93ab36c7..c7956ccd94 100644
--- a/app/models/ability.rb
+++ b/app/models/ability.rb
@@ -25,6 +25,14 @@ class Ability
                          :checkout_permissions]
 
   # Override to add handling of SpeedyAF proxy objects
+  def edit_permissions
+    super
+
+    can [:edit, :update, :destroy], SpeedyAF::Base do |obj|
+      test_edit(obj.id)
+    end
+  end
+
   def read_permissions
     super
 
@@ -73,67 +81,59 @@ def create_permissions(user=nil, session=nil)
   def custom_permissions(user=nil, session=nil)
 
     unless (full_login? || is_api_request?) and is_administrator?
-      cannot :read, MediaObject do |media_object|
+      cannot :read, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
         !(test_read(media_object.id) && media_object.published?) && !test_edit(media_object.id)
       end
 
-      cannot :read, SpeedyAF::Proxy::MediaObject do |media_object|
-        !(test_read(media_object.id) && media_object.published?) && !test_edit(media_object.id)
-      end
-
-      can :read, MasterFile do |master_file|
-        can? :read, master_file.media_object
-      end
-
-      can :read, SpeedyAF::Proxy::MasterFile do |master_file|
+      can :read, [MasterFile, SpeedyAF::Proxy::MasterFile] do |master_file|
         can? :read, master_file.media_object
       end
 
-      can :read, Derivative do |derivative|
+      can :read, [Derivative, SpeedyAF::Proxy::Derivative] do |derivative|
         can? :read, derivative.masterfile.media_object
       end
 
-      cannot :read, Admin::Collection unless (full_login? || is_api_request?)
+      cannot :read, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] unless (full_login? || is_api_request?)
 
       if full_login? || is_api_request?
-        can [:read, :items], Admin::Collection do |collection|
+        can [:read, :items], [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           is_member_of?(collection)
         end
 
         unless (is_member_of_any_collection? or @user_groups.include? 'manager')
-          cannot :read, Admin::Collection
+          cannot :read, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection]
         end
 
-        can :update_access_control, MediaObject do |media_object|
+        can :update_access_control, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
           @user.in?(media_object.collection.managers) ||
             (is_editor_of?(media_object.collection) && !media_object.published?)
         end
 
-        can :unpublish, MediaObject do |media_object|
+        can :unpublish, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
           @user.in?(media_object.collection.managers)
         end
 
-        can :update, Admin::Collection do |collection|
+        can :update, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           is_editor_of?(collection)
         end
 
-        can :update_unit, Admin::Collection do |collection|
+        can :update_unit, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           @user.in?(collection.managers)
         end
 
-        can :update_access_control, Admin::Collection do |collection|
+        can :update_access_control, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           @user.in?(collection.managers)
         end
 
-        can :update_managers, Admin::Collection do |collection|
+        can :update_managers, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           @user.in?(collection.managers)
         end
 
-        can :update_editors, Admin::Collection do |collection|
+        can :update_editors, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           @user.in?(collection.managers)
         end
 
-        can :update_depositors, Admin::Collection do |collection|
+        can :update_depositors, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection|
           is_editor_of?(collection)
         end
 
@@ -141,20 +141,20 @@ def custom_permissions(user=nil, session=nil)
           @user.in?(collection.managers)
         end
 
-        can :inspect, MediaObject do |media_object|
+        can :inspect, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
           is_member_of?(media_object.collection)
         end
 
-        can :show_progress, MediaObject do |media_object|
+        can :show_progress, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
           is_member_of?(media_object.collection)
         end
 
-        can [:edit, :destroy, :update], MasterFile do |master_file|
+        can [:edit, :destroy, :update], [MasterFile, SpeedyAF::Proxy::MasterFile] do |master_file|
           can? :edit, master_file.media_object
         end
 
         # Users logged in through LTI cannot share
-        can :share, MediaObject
+        can :share, [MediaObject, SpeedyAF::Proxy::MediaObject]
       end
 
       # if is_api_request?
@@ -163,27 +163,27 @@ def custom_permissions(user=nil, session=nil)
       #   can :manage, Avalon::ControlledVocabulary
       # end
 
-      cannot :update, MediaObject do |media_object|
+      cannot :update, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
         (not (full_login? || is_api_request?)) || (!is_member_of?(media_object.collection)) ||
           ( media_object.published? && !@user.in?(media_object.collection.managers) )
       end
 
-      cannot :destroy, MediaObject do |media_object|
+      cannot :destroy, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
         # non-managers can only destroy media_object if it's unpublished
         (not (full_login? || is_api_request?)) || (!is_member_of?(media_object.collection)) ||
           ( media_object.published? && !@user.in?(media_object.collection.managers) )
       end
 
-      cannot :destroy, Admin::Collection do |collection, other_user_collections=[]|
+      cannot :destroy, [Admin::Collection, SpeedyAF::Proxy::Admin::Collection] do |collection, other_user_collections=[]|
         (not (full_login? || is_api_request?)) || !@user.in?(collection.managers)
       end
 
-      can :intercom_push, MediaObject do |media_object|
+      can :intercom_push, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
         # anyone who can edit a media_object can also push it
         can? :edit, media_object
       end
 
-      can :json_update, MediaObject do |media_object|
+      can :json_update, [MediaObject, SpeedyAF::Proxy::MediaObject] do |media_object|
         # anyone who can edit a media_object can also update it via the API
         is_api_request? && can?(:edit, media_object)
       end
diff --git a/spec/controllers/bookmarks_controller_spec.rb b/spec/controllers/bookmarks_controller_spec.rb
index cdcf526fc9..637af30689 100644
--- a/spec/controllers/bookmarks_controller_spec.rb
+++ b/spec/controllers/bookmarks_controller_spec.rb
@@ -57,6 +57,14 @@
       expect(flash[:success]).to eq(I18n.t("blacklight.delete.success", count: 11))
       media_objects.each {|mo| expect(MediaObject.exists?(mo.id)).to be_falsey }
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora', :no_perform_enqueued_jobs do
+        WebMock.reset_executed_requests!
+        post :delete
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#update_status" do
@@ -104,6 +112,14 @@
         end
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora', :no_perform_enqueued_jobs do
+        WebMock.reset_executed_requests!
+        post 'publish'
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "index" do
@@ -125,6 +141,14 @@
         expect(response.body).not_to have_css('#deleteLink')
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        WebMock.reset_executed_requests!
+        get 'index'
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#move" do
@@ -148,6 +172,14 @@
         end
       end
     end
+
+    context 'read from solr', :no_perform_enqueued_jobs do
+      it 'should not read from fedora' do
+        WebMock.reset_executed_requests!
+        post 'move', params: { target_collection_id: collection2.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#intercom_push" do
@@ -205,6 +237,14 @@
         expect(flash[:success]).to eq('Sucessfully started push of 3 media objects.')
       end
     end
+
+    context 'read from solr', :no_perform_enqueued_jobs do
+      it 'should not read from fedora' do
+        WebMock.reset_executed_requests!
+        post 'intercom_push', params: { collection_id: 'cupcake_collection' }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe '#update_access_control' do
@@ -423,6 +463,14 @@
         end
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora', :no_perform_enqueued_jobs do
+        WebMock.reset_executed_requests!
+        post 'update_access_control', params: { hidden: "true" }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#merge", :no_perform_enqueued_jobs do
@@ -450,6 +498,14 @@
         expect(flash[:success]).to start_with("Merging 2 items into")
       end
     end
+
+    context 'read from solr', :no_perform_enqueued_jobs do
+      it 'should not read from fedora' do
+        WebMock.reset_executed_requests!
+        post 'merge', params: { media_object: target.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 
   describe "#count" do

From e9c5ea37fcc9737ffa15da7220c32c5a16dc5057 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Tue, 27 Jun 2023 11:34:16 -0400
Subject: [PATCH 106/396] Index collection posters and serve from solr if <512K

---
 app/controllers/admin/collections_controller.rb       |  4 ++--
 app/models/indexed_file.rb                            |  2 +-
 app/presenters/speedy_af/proxy/indexed_file.rb        |  5 +++++
 lib/tasks/avalon.rake                                 | 11 ++++++++++-
 spec/controllers/admin_collections_controller_spec.rb |  8 ++++++++
 5 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/app/controllers/admin/collections_controller.rb b/app/controllers/admin/collections_controller.rb
index dc8bf92529..52f94d20b7 100644
--- a/app/controllers/admin/collections_controller.rb
+++ b/app/controllers/admin/collections_controller.rb
@@ -263,11 +263,11 @@ def remove_poster
   end
 
   def poster
-    @collection = Admin::Collection.find(params['id'])
+    @collection = SpeedyAF::Proxy::Admin::Collection.find(params['id'])
     authorize! :show, @collection
 
     file = @collection.poster
-    if file.nil? || file.new_record?
+    if file.nil? || file.empty?
       render plain: 'Collection Poster Not Found', status: :not_found
     else
       render plain: file.content, content_type: file.mime_type
diff --git a/app/models/indexed_file.rb b/app/models/indexed_file.rb
index 3de4ae20c8..11d331a337 100644
--- a/app/models/indexed_file.rb
+++ b/app/models/indexed_file.rb
@@ -14,7 +14,7 @@
 
 class IndexedFile < ActiveFedora::File
   include SpeedyAF::IndexedContent
-  MAX_CONTENT_SIZE = 32000
+  MAX_CONTENT_SIZE = 512000
 
   # Override
   def original_name
diff --git a/app/presenters/speedy_af/proxy/indexed_file.rb b/app/presenters/speedy_af/proxy/indexed_file.rb
index 88f4f6c5d3..6e26220b36 100644
--- a/app/presenters/speedy_af/proxy/indexed_file.rb
+++ b/app/presenters/speedy_af/proxy/indexed_file.rb
@@ -15,6 +15,11 @@
 class SpeedyAF::Proxy::IndexedFile < SpeedyAF::Base
   # If necessary, decode binary content that is base 64 encoded
   def content
+    # Reify if content exists but isn't stored in the index
+    if !empty? && attrs[:content].blank?
+      ActiveFedora::Base.logger.warn("Reifying #{model} because content not indexed")
+      return real_object.content
+    end
     binary_content? ? Base64.decode64(attrs[:content]) : attrs[:content]
   end
 
diff --git a/lib/tasks/avalon.rake b/lib/tasks/avalon.rake
index baf37b26c1..3263233101 100644
--- a/lib/tasks/avalon.rake
+++ b/lib/tasks/avalon.rake
@@ -24,8 +24,17 @@ namespace :avalon do
     `rails generate active_annotations:install`
   end
 
-  desc "Index MasterFiles and subresources to take advantage of SpeedyAF"
+  desc "Index Admin::Collections and MasterFiles and their subresources to take advantage of SpeedyAF"
   task index_for_speed: :environment do
+    Admin::Collection.find_each do |c|
+      $stderr.print "c["
+      c.update_index;
+      c.declared_attached_files.each_pair do |name, file|
+        $stderr.print name.to_s[0]
+        file.update_external_index if file.respond_to?(:update_external_index)
+      end
+      $stderr.print "]"
+    end
     MasterFile.find_each do |mf|
       $stderr.print "m["
       mf.update_index;
diff --git a/spec/controllers/admin_collections_controller_spec.rb b/spec/controllers/admin_collections_controller_spec.rb
index 1b828da96e..cbe83ca21d 100644
--- a/spec/controllers/admin_collections_controller_spec.rb
+++ b/spec/controllers/admin_collections_controller_spec.rb
@@ -686,5 +686,13 @@
       expect(response.content_type).to eq "image/png; charset=utf-8"
       expect(response.body).not_to be_blank
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        WebMock.reset_executed_requests!
+        get :poster, params: { id: collection.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 end

From 55be10fe84570c0ad8dfa926ad405fa99399f27d Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Wed, 28 Jun 2023 09:48:15 -0400
Subject: [PATCH 107/396] Check for existence of master_file.captions

---
 app/models/iiif_canvas_presenter.rb       | 4 +++-
 spec/models/iiif_canvas_presenter_spec.rb | 8 ++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index a2844aaf56..d0dcfeda63 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -108,7 +108,9 @@ def stream_urls
     end
 
     def supplemental_captions_transcripts
-      master_file.supplemental_files(tag: 'caption') + master_file.supplemental_files(tag: 'transcript') + [master_file.captions]
+      files = master_file.supplemental_files(tag: 'caption') + master_file.supplemental_files(tag: 'transcript')
+      files += [master_file.captions] if master_file.captions.present? && master_file.captions.persisted?
+      files
     end
 
     def simple_iiif_range(label = stream_info[:embed_title])
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 53e3356ece..4c0033ac97 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -115,7 +115,7 @@
   end
 
   describe 'Supplemental file handling' do
-    let(:master_file) { FactoryBot.build(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
+    let(:master_file) { FactoryBot.create(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
     let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
     let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
     let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, tags: ['caption', 'machine_generated']) }
@@ -158,8 +158,12 @@
         expect(subject.any? { |content| content.label == { "en" => ["waveform.json"] } }).to eq false
       end
 
+      it 'does not include master file captions url when legacy captions are not present' do
+        expect(subject.any? { |content| content.url =~ /master_files\/#{master_file.id}\/captions/ }).to eq false
+      end
+
       context 'legacy master file captions' do
-        let(:master_file) { FactoryBot.build(:master_file, :with_waveform, :with_captions, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
+        let(:master_file) { FactoryBot.create(:master_file, :with_waveform, :with_captions, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
 
         it 'includes the master file captions' do
           expect(subject.any? { |content| content.url =~ /master_files\/#{master_file.id}\/captions/ }).to eq true

From ed4286123a7528ea94390a55858a0c6825514279 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 28 Jun 2023 10:24:19 -0400
Subject: [PATCH 108/396] Use SpeedyAF when rendering bulk action modals

---
 app/controllers/application_controller.rb        |  6 +++---
 app/presenters/speedy_af/proxy/media_object.rb   |  1 +
 app/views/bookmarks/delete.html.erb              |  2 +-
 app/views/bookmarks/merge.html.erb               |  2 +-
 app/views/bookmarks/move.html.erb                |  1 -
 .../bookmarks/update_access_control.html.erb     |  2 +-
 spec/controllers/application_controller_spec.rb  | 16 ++++++++--------
 spec/controllers/bookmarks_controller_spec.rb    |  5 +++++
 8 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 25dd22dbb1..29281c6215 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -122,12 +122,12 @@ def get_user_collections(user = nil)
     # return all collections to admin, unless specific user is passed in
     if can?(:manage, Admin::Collection)
       if user.blank?
-        Admin::Collection.all
+        SpeedyAF::Proxy::Admin::Collection.where("*:*").to_a
       else
-        Admin::Collection.where("inheritable_edit_access_person_ssim" => user).to_a
+        SpeedyAF::Proxy::Admin::Collection.where("inheritable_edit_access_person_ssim:#{user}").to_a
       end
     else
-      Admin::Collection.where("inheritable_edit_access_person_ssim" => user_key).to_a
+      SpeedyAF::Proxy::Admin::Collection.where("inheritable_edit_access_person_ssim:#{user_key}").to_a
     end
   end
   helper_method :get_user_collections
diff --git a/app/presenters/speedy_af/proxy/media_object.rb b/app/presenters/speedy_af/proxy/media_object.rb
index 93863918ca..44004bb13e 100644
--- a/app/presenters/speedy_af/proxy/media_object.rb
+++ b/app/presenters/speedy_af/proxy/media_object.rb
@@ -27,6 +27,7 @@ def initialize(solr_document, instance_defaults = {})
     # Handle this case here until a better fix can be found for multiple solr fields which don't have a model property
     @attrs[:section_id] = solr_document["section_id_ssim"]
     @attrs[:date_issued] = solr_document["date_ssi"]
+    @attrs[:hidden?] = solr_document["hidden_bsi"]
     # TODO Need to convert hidden_bsi into discover_groups?
     SINGULAR_FIELDS.each do |field_name|
       @attrs[field_name] = Array(@attrs[field_name]).first
diff --git a/app/views/bookmarks/delete.html.erb b/app/views/bookmarks/delete.html.erb
index 6a104ba5b7..25e99a85f5 100644
--- a/app/views/bookmarks/delete.html.erb
+++ b/app/views/bookmarks/delete.html.erb
@@ -29,7 +29,7 @@ Unless required by applicable law or agreed to in writing, software distributed
           
      <%= t('blacklight.delete.confirm', count: @documents.count) %>
      
           
        
             <% @documents.each do |doc| %>
-              <% @mediaobject = MediaObject.find(doc.id)  %>
+              <% @mediaobject = SpeedyAF::Proxy::MediaObject.find(doc.id)  %>
               <% if can? :destroy, @mediaobject  %>
                 
      • <%= @mediaobject.title %> (<%= doc.id %>)<%=hidden_field_tag "id[]", doc.id %>
        • 
               <% else %>
diff --git a/app/views/bookmarks/merge.html.erb b/app/views/bookmarks/merge.html.erb
index 51440f3894..8fb29d69ca 100644
--- a/app/views/bookmarks/merge.html.erb
+++ b/app/views/bookmarks/merge.html.erb
@@ -41,7 +41,7 @@ Unless required by applicable law or agreed to in writing, software distributed
               
        
                 <%= radio_button_tag :media_object, d.id, style: 'width:100%;' %>
                 <%= label_tag "media_object_#{d.id}", "#{d["title_tesi"] || "No title"} - #{d.id}" %>
-                <% t('blacklight.merge.insufficient_rights') if cannot? :destroy, MediaObject.find(d.id) %>
+                <% t('blacklight.merge.insufficient_rights') if cannot? :destroy, SpeedyAF::Proxy::MediaObject.find(d.id) %>
               
        
             <% end %>
           
diff --git a/app/views/bookmarks/move.html.erb b/app/views/bookmarks/move.html.erb
index fc2a3cdf32..f1cc89df23 100644
--- a/app/views/bookmarks/move.html.erb
+++ b/app/views/bookmarks/move.html.erb
@@ -19,7 +19,6 @@ Unless required by applicable law or agreed to in writing, software distributed
     dropdownParent: $('#blacklight-modal')
   });
 
-
 <% @candidates = get_user_collections %>
 
        
   
        <%= t('blacklight.move.form.title') %>
        
diff --git a/app/views/bookmarks/update_access_control.html.erb b/app/views/bookmarks/update_access_control.html.erb
index cefc121221..f2536a6c1b 100644
--- a/app/views/bookmarks/update_access_control.html.erb
+++ b/app/views/bookmarks/update_access_control.html.erb
@@ -28,7 +28,7 @@ Unless required by applicable law or agreed to in writing, software distributed
   
        
 <% else %>
   <%
-   @media_objects = @documents.collect {|doc| MediaObject.find(doc.id)}
+   @media_objects = @documents.collect {|doc| SpeedyAF::Proxy::MediaObject.find(doc.id)}
    visibility = @media_objects.first.visibility
    @visibility = visibility if @media_objects.all? {|mo| mo.visibility == visibility}
    @shown = @media_objects.all? {|doc| doc.hidden? == false}
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index 1d8dc68f47..f03548f20e 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -47,18 +47,18 @@ def show
   end
 
   describe '#get_user_collections' do
-    let(:collection1) { FactoryBot.create(:collection) }
-    let(:collection2) { FactoryBot.create(:collection) }
+    let!(:collection1) { FactoryBot.create(:collection) }
+    let!(:collection2) { FactoryBot.create(:collection) }
 
     it 'returns all collections for an administrator' do
       login_as :administrator
-      expect(controller.get_user_collections).to include(collection1)
-      expect(controller.get_user_collections).to include(collection2)
+      expect(controller.get_user_collections).to include(have_attributes(id: collection1.id))
+      expect(controller.get_user_collections).to include(have_attributes(id: collection2.id))
     end
     it 'returns only relevant collections for a manager' do
       login_user collection1.managers.first
-      expect(controller.get_user_collections).to include(collection1)
-      expect(controller.get_user_collections).not_to include(collection2)
+      expect(controller.get_user_collections).to include(have_attributes(id: collection1.id))
+      expect(controller.get_user_collections).not_to include(have_attributes(id: collection2.id))
     end
     it 'returns no collections for an end-user' do
       login_as :user
@@ -70,8 +70,8 @@ def show
     end
     it 'returns requested user\'s collections for an administrator' do
       login_as :administrator
-      expect(controller.get_user_collections collection1.managers.first).to include(collection1)
-      expect(controller.get_user_collections collection1.managers.first).not_to include(collection2)
+      expect(controller.get_user_collections collection1.managers.first).to include(have_attributes(id: collection1.id))
+      expect(controller.get_user_collections collection1.managers.first).not_to include(have_attributes(id: collection2.id))
     end
   end
 
diff --git a/spec/controllers/bookmarks_controller_spec.rb b/spec/controllers/bookmarks_controller_spec.rb
index 637af30689..6e954cea69 100644
--- a/spec/controllers/bookmarks_controller_spec.rb
+++ b/spec/controllers/bookmarks_controller_spec.rb
@@ -61,6 +61,7 @@
     context 'read from solr' do
       it 'should not read from fedora', :no_perform_enqueued_jobs do
         WebMock.reset_executed_requests!
+        get :delete
         post :delete
         expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
       end
@@ -176,6 +177,7 @@
     context 'read from solr', :no_perform_enqueued_jobs do
       it 'should not read from fedora' do
         WebMock.reset_executed_requests!
+        get 'move'
         post 'move', params: { target_collection_id: collection2.id }
         expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
       end
@@ -241,6 +243,7 @@
     context 'read from solr', :no_perform_enqueued_jobs do
       it 'should not read from fedora' do
         WebMock.reset_executed_requests!
+        get 'intercom_push'
         post 'intercom_push', params: { collection_id: 'cupcake_collection' }
         expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
       end
@@ -467,6 +470,7 @@
     context 'read from solr' do
       it 'should not read from fedora', :no_perform_enqueued_jobs do
         WebMock.reset_executed_requests!
+        get 'update_access_control'
         post 'update_access_control', params: { hidden: "true" }
         expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
       end
@@ -502,6 +506,7 @@
     context 'read from solr', :no_perform_enqueued_jobs do
       it 'should not read from fedora' do
         WebMock.reset_executed_requests!
+        get 'merge'
         post 'merge', params: { media_object: target.id }
         expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
       end

From 47deff6215a5852294c04ae6c7617a86002f3fc7 Mon Sep 17 00:00:00 2001
From: Chris Colvard 
Date: Wed, 28 Jun 2023 12:06:15 -0400
Subject: [PATCH 109/396] Collection poster end-user route should use SpeedyAF
 like admin route

---
 app/controllers/collections_controller.rb       | 4 ++--
 spec/controllers/collections_controller_spec.rb | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/app/controllers/collections_controller.rb b/app/controllers/collections_controller.rb
index e601a8c48f..903a9eeb07 100644
--- a/app/controllers/collections_controller.rb
+++ b/app/controllers/collections_controller.rb
@@ -51,10 +51,10 @@ def poster
     # Only go on if params[:id] is in @response.documents
     raise CanCan::AccessDenied unless document
 
-    @collection = Admin::Collection.find(params['id'])
+    @collection = SpeedyAF::Proxy::Admin::Collection.find(params['id'])
 
     file = @collection.poster
-    if file.nil? || file.new_record?
+    if file.nil? || file.empty?
       render plain: 'Collection Poster Not Found', status: :not_found
     else
       render plain: file.content, content_type: file.mime_type
diff --git a/spec/controllers/collections_controller_spec.rb b/spec/controllers/collections_controller_spec.rb
index a0911aa689..e483529227 100644
--- a/spec/controllers/collections_controller_spec.rb
+++ b/spec/controllers/collections_controller_spec.rb
@@ -158,5 +158,13 @@
         expect(response).to be_not_found
       end
     end
+
+    context 'read from solr' do
+      it 'should not read from fedora' do
+        WebMock.reset_executed_requests!
+        get 'poster', params: { id: collection.id }
+        expect(a_request(:any, /#{ActiveFedora.fedora.base_uri}/)).not_to have_been_made
+      end
+    end
   end
 end

From 34c55fb57f37258bab4f875fd84ff291fdc5d8ef Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Mon, 12 Jun 2023 12:24:21 -0400
Subject: [PATCH 110/396] Indicate machine generated transcripts in iiif
 manifest

---
 app/models/iiif_canvas_presenter.rb       |  2 +-
 spec/models/iiif_canvas_presenter_spec.rb | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/app/models/iiif_canvas_presenter.rb b/app/models/iiif_canvas_presenter.rb
index d0dcfeda63..b34cbc627b 100644
--- a/app/models/iiif_canvas_presenter.rb
+++ b/app/models/iiif_canvas_presenter.rb
@@ -183,7 +183,7 @@ def manifest_attributes(quality, media_type)
 
     def supplemental_attributes(file)
       if file.is_a?(SupplementalFile)
-        label = file.label
+        label = file.tags.include?('machine_generated') ? file.label + ' (machine generated)' : file.label
         format = file.file.content_type
         language = file.language || 'en'
       else
diff --git a/spec/models/iiif_canvas_presenter_spec.rb b/spec/models/iiif_canvas_presenter_spec.rb
index 4c0033ac97..6b14e2cc60 100644
--- a/spec/models/iiif_canvas_presenter_spec.rb
+++ b/spec/models/iiif_canvas_presenter_spec.rb
@@ -118,7 +118,7 @@
     let(:master_file) { FactoryBot.create(:master_file, :with_waveform, supplemental_files_json: supplemental_files_json, media_object: media_object, derivatives: [derivative]) }
     let(:supplemental_file) { FactoryBot.create(:supplemental_file) }
     let(:transcript_file) { FactoryBot.create(:supplemental_file, :with_transcript_tag) }
-    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, tags: ['caption', 'machine_generated']) }
+    let(:caption_file) { FactoryBot.create(:supplemental_file, :with_caption_file, :with_caption_tag) }
     let(:supplemental_files) { [supplemental_file, transcript_file, caption_file] }
     let(:supplemental_files_json) { supplemental_files.map(&:to_global_id).map(&:to_s).to_s }
 
@@ -153,6 +153,11 @@
         expect(subject.any? { |content| content.url =~ /supplemental_files\/#{caption_file.id}\/captions/ }).to eq true
       end
 
+      it 'does not add " (machine generated)" to label of non-generated files' do
+        expect(subject.any? { |content| content.label =~ /#{transcript_file.label} \(machine generated\)/ }).to eq false
+        expect(subject.any? { |content| content.label =~ /#{caption_file.label} \(machine generated\)/ }).to eq false
+      end
+
       it 'does not include generic supplemental files or waveform' do
         expect(subject.any? { |content| content.url =~ /supplemental_files\/#{supplemental_file.id}/ }).to eq false
         expect(subject.any? { |content| content.label == { "en" => ["waveform.json"] } }).to eq false
@@ -169,6 +174,14 @@
           expect(subject.any? { |content| content.url =~ /master_files\/#{master_file.id}\/captions/ }).to eq true
         end
       end
+
+      context 'machine generated transcript' do
+        let(:transcript_file) { FactoryBot.create(:supplemental_file, tags: ['transcript', 'machine_generated']) }
+
+        it "adds '(machine generated)' to the label" do
+          expect(subject.any? { |content| content.label =~ /#{transcript_file.label} \(machine generated\)/ }).to eq true
+        end
+      end
     end
   end
 

From 2b16196281f629e0ce1764885e2ff62a5fe31569 Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Fri, 23 Jun 2023 11:23:25 -0400
Subject: [PATCH 111/396] Bump active encode

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index edc4422096..eef689e680 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -143,7 +143,7 @@ GEM
     active_elastic_job (3.2.0)
       aws-sdk-sqs (~> 1)
       rails (>= 5.2.6, < 7.1)
-    active_encode (1.2.0)
+    active_encode (1.2.1)
       addressable (~> 2.8)
       rails
     active_fedora-datastreams (0.5.0)

From 5da1d6f012d705b99c0b7b65d3aa95109ddcb4aa Mon Sep 17 00:00:00 2001
From: Mason Ballengee 
Date: Tue, 27 Jun 2023 16:09:55 -0400
Subject: [PATCH 112/396] Refactor access settings forms for collection edit

---
 .../admin/collections_controller.rb           |  23 ++-
 .../collections/_access_control.html.erb      | 111 ++++++++++
 .../admin/collections/_cdl_form.html.erb      |  69 +++++++
 app/views/admin/collections/show.html.erb     |  26 +--
 app/views/modules/_access_control.html.erb    | 193 ------------------
 app/views/modules/_special_access.html.erb    |  30 +++
 6 files changed, 243 insertions(+), 209 deletions(-)
 create mode 100644 app/views/admin/collections/_access_control.html.erb
 create mode 100644 app/views/admin/collections/_cdl_form.html.erb
 delete mode 100644 app/views/modules/_access_control.html.erb
 create mode 100644 app/views/modules/_special_access.html.erb

diff --git a/app/controllers/admin/collections_controller.rb b/app/controllers/admin/collections_controller.rb
index 52f94d20b7..a9852ba00c 100644
--- a/app/controllers/admin/collections_controller.rb
+++ b/app/controllers/admin/collections_controller.rb
@@ -319,10 +319,25 @@ def update_access(collection, params)
       end
     end
 
-    collection.default_visibility = params[:visibility] unless params[:visibility].blank?
-    collection.default_hidden = params[:hidden] == "1"
-    collection.cdl_enabled = params[:cdl] == "1"
-    if collection.cdl_enabled?
+    update_access_settings(collection, params)
+
+    update_default_lending_period(collection, params)
+  end
+
+  def update_access_settings(collection, params)
+    if params[:save_visibility]
+      collection.default_visibility = params[:visibility] unless params[:visibility].blank?
+    end
+    if params[:save_discovery]
+      collection.default_hidden = params[:hidden] == "1"
+    end
+    if params[:save_cdl]
+      collection.cdl_enabled = params[:cdl] == "1"
+    end
+  end
+
+  def update_default_lending_period(collection, params)
+    if collection.cdl_enabled? && params[:save_lending_period]
       lending_period = build_default_lending_period(collection)
       if lending_period.positive?
         collection.default_lending_period = lending_period
diff --git a/app/views/admin/collections/_access_control.html.erb b/app/views/admin/collections/_access_control.html.erb
new file mode 100644
index 0000000000..48cef06274
--- /dev/null
+++ b/app/views/admin/collections/_access_control.html.erb
@@ -0,0 +1,111 @@
+<%#
+Copyright 2011-2023, The Trustees of Indiana University and Northwestern
+  University.  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+  under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+  CONDITIONS OF ANY KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations under the License.
+---  END LICENSE_HEADER BLOCK  ---
+%>
+<% can_update = can?(:update_access_control, object) %>
+
+<% if can_update %>
+<%= bootstrap_form_for object, html: { id: "#{object.is_a?(Admin::Collection) ? 'discovery_form' : 'access_control_form'}" }  do |vid| %>
+<%= hidden_field_tag :step, @active_step %>
+
        
+  
        
+    
        Item discovery
        
+  
        
+  
        
+    
        
+      
        
+        <%= label_tag :hidden do %>
+        <%= check_box_tag :hidden, '1', (hidden) %>
+        Hide this item from search results
+        <% end %>
+      
        
+      <%= submit_tag "Save Setting", name: "save_discovery", class: "btn btn-primary", form: 'discovery_form' %>
+      <%= button_tag "Apply to All Existing Items", type: 'button', class: "btn btn-outline", data: { toggle:"modal", target:"#access_control_modal" } %>
+    
        
+  
        
+
        
+
+<% end %>
+
+<%= bootstrap_form_for object, html: { id: "#{object.is_a?(Admin::Collection) ? 'visibility_form' : 'access_control_form'}" }  do |vid| %>
+<%= hidden_field_tag :step, @active_step %>
+
+
        
+  
        
+    
        Item access
        
+  
        
+  
        
+    
        
+      
        
+        
+      
        
+      
        
+        
+      
        
+      
        
+        
+      
        
+      <%= submit_tag "Save Setting", name: "save_visibility", class: "btn btn-primary", form: 'visibility_form' %>
+      <%= button_tag "Apply to All Existing Items", type: 'button', class: "btn btn-outline", data: { toggle:"modal", target:"#access_control_modal" } %>
+    
        
+  
        
+
        
+
+<% end %>
+
+<%= render modal[:partial], modal_title: modal[:title] if defined? modal %>
+
+<% else %>
+
+
        
+  
        
+    
        Item discovery
        
+  
        
+  
        
+    
        
+      <%= hidden ? "Item is hidden from search results" :
+        "Item is not hidden from search results" %>
+    
        
+  
        
+
        
+
+
        
+  
        
+    
        Item access
        
+  
        
+  
        
+    
        
+      Item is viewable by
+      <%= case @visibility
+          when "public"
+            "the general public"
+          when "restricted"
+            "logged in users only"
+          when "private"
+            "collection staff only"
+          end %>
+    
        
+  
        
+
        
+
+<% end %>
diff --git a/app/views/admin/collections/_cdl_form.html.erb b/app/views/admin/collections/_cdl_form.html.erb
new file mode 100644
index 0000000000..8380822a03
--- /dev/null
+++ b/app/views/admin/collections/_cdl_form.html.erb
@@ -0,0 +1,69 @@
+<% if can?(:update_access_control, object) %>
+<%= bootstrap_form_for object, html: { id: "#{object.is_a?(Admin::Collection) ? 'cdl_form' : 'access_control_form'}" }  do |vid| %>
+<%= hidden_field_tag :step, @active_step %>
+  <% if Avalon::Configuration.controlled_digital_lending_enabled? && object.is_a?(Admin::Collection) %>
+    
        
+      
        
+        
        Controlled Digital Lending 
        
+      
        
+      
        
+        
        
+          
        
+            <%= label_tag :cdl do %>
+            <%= check_box_tag :cdl, '1', (cdl) %>
+            Enable controlled digital lending for this collection
+            <% end %>
+          
        
+        
        
+        <%= submit_tag "Save Setting", name: "save_cdl", class: "btn btn-primary", form: 'cdl_form' %>
+      
        
+    
        
+  <% end %>
+
+  <% if lending_enabled?(object) %>
+    
        
+      
        
+        
        Item lending period
        
+      
        
+      
        
+        
        
+          <%=  render partial: "modules/tooltip", locals: { form: vid, field: :lending_period, tooltip: t("access_control.#{:lending_period}"), options: {display_label: (t("access_control.#{:lending_period}label")+'*').html_safe} } %>
        
+          <% d, h = (lending_period/3600).divmod(24) %>
+          
        
+            
        
+              
+              <%= text_field_tag "add_lending_period_days", d ? d : 0, class: 'form-control' %>
+            
        
+            
        
+              
+              <%= text_field_tag "add_lending_period_hours", h ? h : 0, class: 'form-control' %>
+            
        
+          
        
+        
        
+        <%= submit_tag "Save Lending Period", name: "save_lending_period", class: "btn btn-primary", form: 'cdl_form' %>
+        <%= button_tag "Apply to All Existing Items", type: 'button', class: "btn btn-outline", data: { toggle:"modal", target:"#access_control_modal" } %>
+      
        
+    
        
+  <% end %>
+<% end %>
+
+<% else %>
+
+  <% if lending_enabled?(object) %>
+    
        
+      
        
+        
        Item lending period
        
+      
        
+      
        
+        
        
+          Item is available to be checked out for
+          <%= ActiveSupport::Duration.build(lending_period).to_day_hour_s %>
+        
        
+      
        
+    
        
+  <% end %>
+<% end %>
diff --git a/app/views/admin/collections/show.html.erb b/app/views/admin/collections/show.html.erb
index 6adcc862d4..f7daef2611 100644
--- a/app/views/admin/collections/show.html.erb
+++ b/app/views/admin/collections/show.html.erb
@@ -123,22 +123,24 @@ Unless required by applicable law or agreed to in writing, software distributed
   
      
 
    
 
-
    
-  
    Set Default Access Control for New Items
    
-  <%= render "modules/access_control", { object: @collection,
-                                         visibility: @collection.default_visibility,
-                                         hidden: @collection.default_hidden,
-                                         lending_period: @collection.default_lending_period,
-                                         cdl: @collection.cdl_enabled,
+
    
+  <%= render "cdl_form", { object: @collection,
+                           lending_period: @collection.default_lending_period,
+                           cdl: @collection.cdl_enabled,
+                           modal: { partial: "apply_access_control", title: "Apply current Default Access settings to all existing Items" } } %>
+
+  
    Set Default Access Control for New Items
    
+  <%= render "access_control", { object: @collection,
+                                 visibility: @collection.default_visibility,
+                                 hidden: @collection.default_hidden,
+                                 modal: { partial: "apply_access_control", title: "Apply current Default Access settings to all existing Items" } } %>
+
+  <%= render "modules/special_access", { object: @collection,
                                          modal: { partial: "apply_access_control", title: "Apply current Default Access settings to all existing Items" } } %>
 
-  <% if can? :update_access_control, @collection %>
-  <%= submit_tag "Save Access Settings", name: "save_access", class: "btn btn-primary", form: 'access_control_form' %>
-  <%= button_tag "Apply to All Existing Items", class: "btn btn-outline", data: { toggle:"modal", target:"#access_control_modal" } %>
-  <% end %>
 
    
 
-<%= render "form", modal_title: "Edit collection Information" %>
+<%= render "form", modal_title: "Edit Collection Information" %>
 
 <% content_for :page_scripts do %>
 
diff --git a/app/views/media_objects/_sections.html.erb b/app/views/media_objects/_sections.html.erb
deleted file mode 100644
index 4dfff4d855..0000000000
--- a/app/views/media_objects/_sections.html.erb
+++ /dev/null
@@ -1,129 +0,0 @@
-<%#
-Copyright 2011-2023, The Trustees of Indiana University and Northwestern
-  University.  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed
-  under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-  CONDITIONS OF ANY KIND, either express or implied. See the License for the
-  specific language governing permissions and limitations under the License.
----  END LICENSE_HEADER BLOCK  ---
-%>
-<% show_progress = show_progress?(sections) %>
-<% unless hide_sections?(sections) and not show_progress %>
-
-
    
-  
    
-    
-
-<% show_all_sections = can? :edit, @media_object %>
-<% sections.each_with_index do |section,i| %>
-<%   unless show_all_sections %>
-<%     next if section.derivatives.empty? %>
-<%   end %>
-<%=  structure_html(section,i,show_progress).html_safe %>
-<% end %>
-
-  
    
-
    
-
-
-
-
-<% content_for :page_scripts do %>
-
-<% end %>
-<% end %>
diff --git a/app/views/media_objects/_share.html.erb b/app/views/media_objects/_share.html.erb
index 74c6a89f66..340d318e88 100644
--- a/app/views/media_objects/_share.html.erb
+++ b/app/views/media_objects/_share.html.erb
@@ -32,21 +32,44 @@ Unless required by applicable law or agreed to in writing, software distributed
 <% content_for :page_scripts do %>
 
 
 
 <% end %>
diff --git a/app/views/media_objects/_timeline.html.erb b/app/views/media_objects/_timeline.html.erb
index e25e125a1e..d7801b3c25 100644
--- a/app/views/media_objects/_timeline.html.erb
+++ b/app/views/media_objects/_timeline.html.erb
@@ -20,7 +20,7 @@ Unless required by applicable law or agreed to in writing, software distributed
   
 
    
 
-