From 3bff6e4618daf58bb9583eb00f872a33b8be9cf3 Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Mon, 18 Nov 2024 13:07:08 -0500 Subject: [PATCH 1/4] Make override explicit by doing it in the original module in an initializer --- app/helpers/blacklight/local_blacklight_helper.rb | 4 ---- config/initializers/blacklight.rb | 8 ++++++++ 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 config/initializers/blacklight.rb diff --git a/app/helpers/blacklight/local_blacklight_helper.rb b/app/helpers/blacklight/local_blacklight_helper.rb index c11b38074e..10cde5281e 100644 --- a/app/helpers/blacklight/local_blacklight_helper.rb +++ b/app/helpers/blacklight/local_blacklight_helper.rb @@ -25,10 +25,6 @@ def facet_group_names blacklight_config.facet_fields.map {|facet,opts| opts[:group]}.uniq end - def url_for_document doc, options = {} - SpeedyAF::Base.for(doc.to_h.with_indifferent_access) - end - def contributor_index_display args args[:document][args[:field]].first(3).join("; ") end diff --git a/config/initializers/blacklight.rb b/config/initializers/blacklight.rb new file mode 100644 index 0000000000..2c484e8d57 --- /dev/null +++ b/config/initializers/blacklight.rb @@ -0,0 +1,8 @@ +require 'blacklight' +Rails.application.config.to_prepare do + module Blacklight::UrlHelperBehavior + def url_for_document doc, options = {} + SpeedyAF::Base.for(doc.to_h.with_indifferent_access) + end + end +end From bd4726078c812128540444c5dc9818f05f3ce098 Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Mon, 18 Nov 2024 15:45:09 -0500 Subject: [PATCH 2/4] Update cross-spawn and remove unneccessary resolutions --- package.json | 6 +----- yarn.lock | 60 ++++++---------------------------------------------- 2 files changed, 7 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 0b814ebd4f..e083c4b55c 100644 --- a/package.json +++ b/package.json @@ -48,11 +48,7 @@ "webpack-dev-server": "5" }, "resolutions": { - "glob-parent": "^6.0.2", - "unset-value": "^2.0.1", - "nth-check": "^2.1.1", - "set-value": "^4.1.0", - "kind-of": "^6.0.3" + "glob-parent": "^6.0.2" }, "scripts": { "start-collection-index": "webpack-dev-server --mode development --config config/webpack/collection_index.js --host 0.0.0.0", diff --git a/yarn.lock b/yarn.lock index bca9d14501..e168c71785 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2865,9 +2865,9 @@ cropperjs@^1.5.5: integrity sha512-re7UdjE5UnwdrovyhNzZ6gathI4Rs3KGCBSc8HCIjUo5hO42CtzyblmWLj6QWVw7huHyDMfpKxhiO2II77nhDw== cross-spawn@^7.0.0, cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" @@ -3773,13 +3773,6 @@ get-stream@^5.0.0, get-stream@^5.1.0: dependencies: pump "^3.0.0" -get-value@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/get-value/-/get-value-3.0.1.tgz#5efd2a157f1d6a516d7524e124ac52d0a39ef5a8" - integrity sha512-mKZj9JLQrwMBtj5wxi6MH8Z5eSKaERpAwjg43dPtlGI1ZVEgH/qC7T8/6R2OBSUA+zzHBZgICsVJaEIV2tKTDA== - dependencies: - isobject "^3.0.1" - getos@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5" @@ -3904,21 +3897,6 @@ has-tostringtag@^1.0.0: dependencies: has-symbols "^1.0.2" -has-value@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/has-value/-/has-value-2.0.2.tgz#d0f12e8780ba8e90e66ad1a21c707fdb67c25658" - integrity sha512-ybKOlcRsK2MqrM3Hmz/lQxXHZ6ejzSPzpNabKB45jb5qDgJvKPa3SdapTsTLwEb9WltgWpOmNax7i+DzNOk4TA== - dependencies: - get-value "^3.0.0" - has-values "^2.0.1" - -has-values@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/has-values/-/has-values-2.0.1.tgz#3876200ff86d8a8546a9264a952c17d5fc17579d" - integrity sha512-+QdH3jOmq9P8GfdjFg0eJudqx1FqU62NQJ4P16rOEHeRdl7ckgwn6uqQjzYE0ZoHVV/e5E2esuJ5Gl5+HUW19w== - dependencies: - kind-of "^6.0.2" - has@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" @@ -4321,11 +4299,6 @@ is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== -is-primitive@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/is-primitive/-/is-primitive-3.0.1.tgz#98c4db1abff185485a657fc2905052b940524d05" - integrity sha512-GljRxhWvlCNRfZyORiH77FwdFwGcMO620o37EOYC0ORWdq+WYNVqW0w2Juzew4M+L81l6/QS3t5gkkihyRqv9w== - is-stream@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" @@ -4370,11 +4343,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isomorphic-unfetch@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/isomorphic-unfetch/-/isomorphic-unfetch-3.1.0.tgz#87341d5f4f7b63843d468438128cb087b7c3e98f" @@ -4580,7 +4548,7 @@ keycode@2.2.0: resolved "https://registry.yarnpkg.com/keycode/-/keycode-2.2.0.tgz#3d0af56dc7b8b8e5cba8d0a97f107204eec22b04" integrity sha512-ps3I9jAdNtRpJrbBvQjpzyFbss/skHqzS+eu4RxKLaEAtFqkjZaB6TZMSivPbLxf4K7VI4SjR0P5mRCX5+Q25A== -kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@^6.0.2: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -4844,7 +4812,7 @@ methods@~1.1.2: resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= -micromatch@4.0.8, micromatch@^4.0.2: +micromatch@^4.0.2: version "4.0.8" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== @@ -5040,7 +5008,7 @@ npm-run-path@^4.0.0: dependencies: path-key "^3.0.0" -nth-check@^2.0.0, nth-check@^2.0.1, nth-check@^2.1.1: +nth-check@^2.0.0, nth-check@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== @@ -6304,14 +6272,6 @@ set-function-length@^1.2.1: gopd "^1.0.1" has-property-descriptors "^1.0.2" -set-value@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/set-value/-/set-value-4.1.0.tgz#aa433662d87081b75ad88a4743bd450f044e7d09" - integrity sha512-zTEg4HL0RwVrqcWs3ztF+x1vkxfm0lP+MQQFPiMJTKVceBwEV0A569Ou8l9IYQG8jOZdMVI1hGsc0tmeD2o/Lw== - dependencies: - is-plain-object "^2.0.4" - is-primitive "^3.0.1" - setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" @@ -6837,14 +6797,6 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= -unset-value@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" From c3d8884fa9f4119acb4f2ce84a643cc6311eb8d9 Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Mon, 18 Nov 2024 16:40:11 -0500 Subject: [PATCH 3/4] Fix processing of google drive files When we made the switch from mediainfo to ffprobe, we created a new class for processing the file. Because Google Drive utilizes headers/params for its authorization flow, this change resulted in the file information becoming divorced from the auth token. By explicitly passing the auth token through to the classes downloading and processing the file, we are able to fix this oversight. --- app/models/master_file.rb | 2 +- app/services/file_locator.rb | 14 +++++++++++--- lib/avalon/ffprobe.rb | 4 +++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/app/models/master_file.rb b/app/models/master_file.rb index 51e8706eae..df55db16d9 100644 --- a/app/models/master_file.rb +++ b/app/models/master_file.rb @@ -727,7 +727,7 @@ def saveDerivativesHash(derivative_hash) def reloadTechnicalMetadata! # Reset ffprobe - @ffprobe = Avalon::FFprobe.new(FileLocator.new(file_location)) + @ffprobe = Avalon::FFprobe.new(FileLocator.new(file_location, { auth_header: @auth_header })) # Formats like MP4 can be caught as both audio and video # so the case statement flows in the preferred order diff --git a/app/services/file_locator.rb b/app/services/file_locator.rb index 65041c88c3..0ef83705e2 100644 --- a/app/services/file_locator.rb +++ b/app/services/file_locator.rb @@ -16,7 +16,7 @@ require 'aws-sdk-s3' class FileLocator - attr_reader :source + attr_reader :source, :auth_header class S3File attr_reader :bucket, :key @@ -46,8 +46,9 @@ def download_url end end - def initialize(source) + def initialize(source, opts = {}) @source = source + @auth_header = opts[:auth_header] end def uri @@ -126,7 +127,14 @@ def reader # Prioritize using URI#open, attempt to fallback to Kernel#open # if URI fails. def open_uri - URI.open(uri, 'r') + # Google Drive download urls have the auth as a param + # so we need to make sure the auth gets passed in + # when we try to access files + if auth_header.present? && auth_header.is_a?(Hash) + URI.open(uri, 'r', auth_header) + else + URI.open(uri, 'r') + end rescue Kernel::open(uri.to_s, 'r') end diff --git a/lib/avalon/ffprobe.rb b/lib/avalon/ffprobe.rb index 0322b8ec60..2bea318517 100644 --- a/lib/avalon/ffprobe.rb +++ b/lib/avalon/ffprobe.rb @@ -23,7 +23,9 @@ def json_output return @json_output unless @json_output.nil? return @json_output = {} unless valid_content_type?(@media_file) ffprobe = Settings&.ffprobe&.path || 'ffprobe' - raw_output = `#{ffprobe} -i "#{@media_file.location}" -v quiet -show_format -show_streams -of json` + # Include authorization headers for cases like Google Drive + header = "-headers 'Authorization: #{@media_file.auth_header.fetch('Authorization')}'" if @media_file.auth_header.present? + raw_output = `#{ffprobe} #{header} -i "#{@media_file.location}" -v quiet -show_format -show_streams -of json` # $? is a variable for the exit status of the last executed process. # Success == 0, any other value means the command failed in some way. unless $?.exitstatus == 0 From 4b5f772f770f911b75890798741fe7daab54be6b Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Tue, 19 Nov 2024 15:07:25 -0500 Subject: [PATCH 4/4] Pin to specific revisions in github until releases of gems are made. --- Gemfile | 7 +++---- Gemfile.lock | 51 +++++++++++++-------------------------------------- 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/Gemfile b/Gemfile index 012299b183..4a6343df84 100644 --- a/Gemfile +++ b/Gemfile @@ -30,9 +30,8 @@ gem 'shakapacker' # Core Samvera #gem 'active-fedora', '~> 15.0' -#gem 'active_fedora-datastreams', '~> 0.5' -gem 'active-fedora', git: 'https://github.com/samvera/active_fedora.git', branch: 'fedora6_rebase' -gem 'active_fedora-datastreams', git: 'https://github.com/samvera-labs/active_fedora-datastreams.git', branch: 'fedora6_rebase' +gem 'active-fedora', git: 'https://github.com/samvera/active_fedora.git', ref: '0f5ccb1536224efec750941ce9a1f58f2e09cd3c' +gem 'active_fedora-datastreams', '~> 0.5' gem 'hydra-head', '~> 13.0' gem 'ldp', '~> 1.1.0' gem 'noid-rails', '~> 3.2' @@ -60,7 +59,7 @@ gem 'rack-cors', require: 'rack/cors' gem 'rails_same_site_cookie' gem 'recaptcha', require: 'recaptcha/rails' gem 'samvera-persona', '~> 0.5.0' -gem 'speedy-af', git: 'https://github.com/samvera-labs/speedy_af.git', branch: 'empty_reflection' +gem 'speedy-af', '~> 0.4.0' # Avalon Components gem 'avalon-workflow', git: "https://github.com/avalonmediasystem/avalon-workflow.git", tag: 'avalon-r8.0' diff --git a/Gemfile.lock b/Gemfile.lock index d4e74213ac..878b8a2d04 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,32 +57,10 @@ GIT ims-lti omniauth -GIT - remote: https://github.com/samvera-labs/active_fedora-datastreams.git - revision: 3e1b54f60e6d6316e114f608ee5c50c85d6598c6 - branch: fedora6_rebase - specs: - active_fedora-datastreams (0.5.0) - active-fedora (>= 11.0.0.pre) - activemodel (>= 5.2) - nom-xml (>= 0.5.1) - om (~> 3.1) - rdf (~> 3.2) - rdf-rdfa (= 3.2.0) - rdf-rdfxml (~> 3.2) - -GIT - remote: https://github.com/samvera-labs/speedy_af.git - revision: 223abe9e2a7cdc68b033398bf69922523aeff47e - branch: empty_reflection - specs: - speedy-af (0.3.0) - activesupport (> 5.2) - GIT remote: https://github.com/samvera/active_fedora.git revision: 0f5ccb1536224efec750941ce9a1f58f2e09cd3c - branch: fedora6_rebase + ref: 0f5ccb1536224efec750941ce9a1f58f2e09cd3c specs: active-fedora (15.0.1) active-triples (>= 0.11.0, < 2.0.0) @@ -159,6 +137,13 @@ GEM active_encode (1.2.3) addressable (~> 2.8) rails + active_fedora-datastreams (0.5.0) + active-fedora (>= 11.0.0.pre) + activemodel (>= 5.2) + nom-xml (>= 0.5.1) + om (~> 3.1) + rdf (~> 3.2) + rdf-rdfxml (~> 3.2) activejob (7.2.2) activesupport (= 7.2.2) globalid (>= 0.3.6) @@ -466,9 +451,6 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - haml (5.2.2) - temple (>= 0.8.0) - tilt hashdiff (1.1.1) hashie (5.0.0) hooks (0.4.1) @@ -740,20 +722,11 @@ GEM bcp47_spec (~> 0.2) bigdecimal (~> 3.1, >= 3.1.5) link_header (~> 0.0, >= 0.0.8) - rdf-aggregate-repo (3.3.0) - rdf (~> 3.3) rdf-isomorphic (3.3.0) rdf (~> 3.3) rdf-ldp (0.1.0) deprecation rdf - rdf-rdfa (3.2.0) - haml (~> 5.2) - htmlentities (~> 4.3) - rdf (~> 3.2) - rdf-aggregate-repo (~> 3.2) - rdf-vocab (~> 3.2) - rdf-xsd (~> 3.2) rdf-rdfxml (3.3.0) builder (~> 3.2, >= 3.2.4) htmlentities (~> 4.3) @@ -940,6 +913,9 @@ GEM nokogiri stomp xml-simple + speedy-af (0.4.0) + active-fedora (>= 11.0.0) + activesupport (> 5.2) sprockets (3.7.3) base64 concurrent-ruby (~> 1.0) @@ -967,7 +943,6 @@ GEM sxp (2.0.0) matrix (~> 0.4) rdf (~> 3.3) - temple (0.10.3) terser (1.2.3) execjs (>= 0.3.0, < 3) thor (1.3.2) @@ -1033,7 +1008,7 @@ DEPENDENCIES active_annotations (~> 0.5.0) active_elastic_job active_encode (~> 1.2) - active_fedora-datastreams! + active_fedora-datastreams (~> 0.5) activejob-traffic_control activejob-uniqueness activerecord-session_store (>= 2.0.0) @@ -1143,7 +1118,7 @@ DEPENDENCIES sidekiq-cron (~> 1.9) simplecov solr_wrapper (>= 0.16) - speedy-af! + speedy-af (~> 0.4.0) sprockets (~> 3.7.2) sprockets-es6 sqlite3