diff --git a/app/jobs/master_file_management_jobs.rb b/app/jobs/master_file_management_jobs.rb index f35de08acd..f01ddb76ab 100644 --- a/app/jobs/master_file_management_jobs.rb +++ b/app/jobs/master_file_management_jobs.rb @@ -18,34 +18,6 @@ module MasterFileManagementJobs class Move < ActiveJob::Base queue_as :master_file_management_move - def s3_to_s3(source, dest) - source_object = FileLocator::S3File.new(source.source).object - dest_object = FileLocator::S3File.new(dest.source).object - if dest_object.copy_from(source_object, multipart_copy: source_object.size > 15.megabytes) - source_object.delete if FileLocator.new(dest.source).exists? - end - end - - def s3_to_file(source, dest) - source_object = FileLocator::S3File.new(source.source).object - FileUtils.mkdir_p File.dirname(dest.uri.path) unless File.exist? File.dirname(dest.uri.path) - if source_object.download_file(dest.uri.path) - source_object.delete - end - end - - def file_to_s3(source, dest) - dest_object = FileLocator::S3File.new(dest.source).object - if dest_object.upload_file(source.uri.path) - FileUtils.rm(source.uri.path) - end - end - - def file_to_file(source, dest) - FileUtils.mkdir_p File.dirname(dest.location) unless File.exist? File.dirname(dest.location) - FileUtils.mv source.location, dest.location - end - def perform(id, newpath) Rails.logger.debug "Moving masterfile to #{newpath}" @@ -57,8 +29,7 @@ def perform(id, newpath) Rails.logger.info "Masterfile #{newpath} already moved" elsif old_locator.exists? new_locator = FileLocator.new(newpath) - copy_method = "#{old_locator.uri.scheme}_to_#{new_locator.uri.scheme}".to_sym - send(copy_method, old_locator, new_locator) + FileMover.move(old_locator, new_locator) masterfile.file_location = newpath masterfile.save Rails.logger.info "#{oldpath} has been moved to #{newpath}" diff --git a/app/models/master_file.rb b/app/models/master_file.rb index df55db16d9..3951ca10ee 100644 --- a/app/models/master_file.rb +++ b/app/models/master_file.rb @@ -208,9 +208,8 @@ def setContent(file, file_name: nil, file_size: nil, auth_header: nil, dropbox_d self.file_location = file.to_s self.file_size = FileLocator::S3File.new(file).object.size else - self.file_location = file.to_s - self.file_size = file_size - self.title = file_name + local_file = FileLocator.new(file, filename: file_name, auth_header: auth_header).local_location + saveOriginal(File.open(local_file), file_name, dropbox_dir) end else #Batch saveOriginal(file, File.basename(file.path), dropbox_dir) @@ -682,6 +681,7 @@ def ffmpeg_frame_options(file_source, output_path, offset, new_width, new_height def saveOriginal(file, original_name = nil, dropbox_dir = media_object.collection.dropbox_absolute_path) realpath = File.realpath(file.path) + self.file_size = file.size if original_name.present? # If we have a temp name from an upload, rename to the original name supplied by the user unless File.basename(realpath) == original_name @@ -694,14 +694,15 @@ def saveOriginal(file, original_name = nil, dropbox_dir = media_object.collectio path = File.join(parent_dir, duplicate_file_name(original_name, num)) num += 1 end - FileUtils.move(realpath, path) + old_locator = FileLocator.new(realpath) + new_locator = FileLocator.new(path) + FileMover.move(old_locator, new_locator) realpath = path end create_working_file!(realpath) end self.file_location = realpath - self.file_size = file.size.to_s ensure file.close end diff --git a/app/services/file_locator.rb b/app/services/file_locator.rb index 0ef83705e2..24e6045b12 100644 --- a/app/services/file_locator.rb +++ b/app/services/file_locator.rb @@ -16,7 +16,7 @@ require 'aws-sdk-s3' class FileLocator - attr_reader :source, :auth_header + attr_reader :source, :filename, :auth_header class S3File attr_reader :bucket, :key @@ -48,6 +48,7 @@ def download_url def initialize(source, opts = {}) @source = source + @filename = opts[:filename] @auth_header = opts[:auth_header] end @@ -89,17 +90,28 @@ def location end end - # If S3, download object to /tmp + # If S3 or http(s), download object to /tmp def local_location @local_location ||= begin - if uri.scheme == 's3' + case uri.scheme + when 's3' S3File.new(uri).local_file.path - else + when 'file' location + else + local_file.path end end end + def local_file + @local_file ||= Tempfile.new(filename) + File.binwrite(@local_file, reader.read) + @local_file + ensure + @local_file.close + end + def exist? case uri.scheme when 's3' diff --git a/app/services/file_mover.rb b/app/services/file_mover.rb new file mode 100644 index 0000000000..b5320cccd4 --- /dev/null +++ b/app/services/file_mover.rb @@ -0,0 +1,65 @@ +# Copyright 2011-2024, The Trustees of Indiana University and Northwestern +# University. Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# --- END LICENSE_HEADER BLOCK --- + +class FileMover + class << self + def move(source, dest, method: nil) + new(source, dest, method: method).move + end + end + + def initialize(source, dest, method: nil) + @source = source + @dest = dest + @method = method + end + + def move + send(method, @source, @dest) + end + + def method + @method || "#{@source.uri.scheme}_to_#{@dest.uri.scheme}".to_sym + end + + private + + def s3_to_s3(source, dest) + source_object = FileLocator::S3File.new(source.source).object + dest_object = FileLocator::S3File.new(dest.source).object + if dest_object.copy_from(source_object, multipart_copy: source_object.size > 15.megabytes) + source_object.delete if FileLocator.new(dest.source).exists? + end + end + + def s3_to_file(source, dest) + source_object = FileLocator::S3File.new(source.source).object + FileUtils.mkdir_p File.dirname(dest.uri.path) unless File.exist? File.dirname(dest.uri.path) + if source_object.download_file(dest.uri.path) + source_object.delete + end + end + + def file_to_s3(source, dest) + dest_object = FileLocator::S3File.new(dest.source).object + if dest_object.upload_file(source.uri.path) + FileUtils.rm(source.uri.path) + end + end + + def file_to_file(source, dest) + FileUtils.mkdir_p File.dirname(dest.location) unless File.exist? File.dirname(dest.location) + FileUtils.mv source.location, dest.location + end +end \ No newline at end of file diff --git a/spec/models/master_file_spec.rb b/spec/models/master_file_spec.rb index 9c00267597..f3fafea222 100644 --- a/spec/models/master_file_spec.rb +++ b/spec/models/master_file_spec.rb @@ -423,15 +423,50 @@ let(:file_name) { "sample.mp4" } let(:file_size) { 12345 } let(:auth_header) { {"Authorization"=>"Bearer ya29.a0AfH6SMC6vSj4D6po1aDxAr6JmY92azh3lxevSuPKxf9QPPSKmMzqbZvI7B3oIACqqMVono1P0XD2F1Jl_rkayoI6JGz-P2cpg44-55oJFcWychAvUliWeRKf1cifMo9JF10YmXxhIfrG5mu7Ahy9FZpudN92p2JhvTI"} } + let(:fixture) { File.expand_path('../../fixtures/videoshort.mp4',__FILE__) } + let(:tempfile) { Tempfile.new('foo') } + let(:media_path) { File.expand_path("../../master_files-#{SecureRandom.uuid}",__FILE__)} + let(:dropbox_path) { File.expand_path("../../collection-#{SecureRandom.uuid}",__FILE__)} + let(:upload) { ActionDispatch::Http::UploadedFile.new :tempfile => tempfile, :filename => original, :type => 'video/mp4' } + let!(:media_object) { FactoryBot.create(:media_object, sections: [subject]) } + let(:collection) { Admin::Collection.new } subject { FactoryBot.create(:master_file) } - it "should set the right properties" do + before(:each) do allow(subject).to receive(:reloadTechnicalMetadata!).and_return(nil) - subject.setContent(file, file_name: file_name, file_size: file_size, auth_header: auth_header) - expect(subject.file_location).to eq(file.to_s) + @old_media_path = Settings.encoding.working_file_path + FileUtils.mkdir_p media_path + FileUtils.cp fixture, tempfile + allow_any_instance_of(FileLocator).to receive(:local_location).and_return(tempfile.path) + allow_any_instance_of(File).to receive(:size).and_return(file_size) + allow(media_object).to receive(:collection).and_return(collection) + FileUtils.mkdir_p dropbox_path + allow(collection).to receive(:dropbox_absolute_path).and_return(File.absolute_path(dropbox_path)) + end + + after(:each) do + Settings.encoding.working_file_path = @old_media_path + File.unlink subject.file_location + FileUtils.rm_rf media_path + FileUtils.rm_rf dropbox_path + end + + it "should move an uploaded file into the root of the collection's dropbox" do + Settings.encoding.working_file_path = nil + subject.setContent(file, file_name: file_name, file_size: file_size, auth_header: auth_header, dropbox_dir: collection.dropbox_absolute_path) + expect(subject.file_location).to eq(File.realpath(File.join(collection.dropbox_absolute_path, file_name))) + end + + it "should copy an uploaded file to the media path" do + Settings.encoding.working_file_path = media_path + subject.setContent(file, file_name: file_name, file_size: file_size, auth_header: auth_header, dropbox_dir: collection.dropbox_absolute_path) + expect(File.fnmatch("#{media_path}/*/#{file_name}", subject.working_file_path.first)).to be true + end + + it "should set the right properties" do + subject.setContent(file, file_name: file_name, file_size: file_size, auth_header: auth_header, dropbox_dir: collection.dropbox_absolute_path) expect(subject.file_size).to eq(file_size) - expect(subject.title).to eq(file_name) expect(subject.instance_variable_get(:@auth_header)).to eq(auth_header) end end diff --git a/spec/services/file_mover_spec.rb b/spec/services/file_mover_spec.rb new file mode 100644 index 0000000000..8b8d9459bd --- /dev/null +++ b/spec/services/file_mover_spec.rb @@ -0,0 +1,81 @@ +# Copyright 2011-2024, The Trustees of Indiana University and Northwestern +# University. Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# --- END LICENSE_HEADER BLOCK --- + +require 'rails_helper' +require 'fakefs/safe' + +describe FileMover, type: :service do + describe '.move' do + let(:fs_location) { '/path/to/source/test.mp4' } + let(:fs_file) { FileLocator.new("file://#{fs_location}") } + let(:fs_dest) { FileLocator.new('file:///path/to/dest/test.mp4') } + let(:s3_file) { FileLocator.new('s3://source_bucket/test.mp4') } + let(:s3_dest) { FileLocator.new('s3://dest_bucket/test.mp4') } + + before(:each, s3: true) do + @source_object = double(key: 'test.mp4', bucket_name: 'source_bucket', size: 1) + @dest_object = double(key: 'test.mp4', bucket_name: 'dest_bucket', exists?: true) + allow(Aws::S3::Object).to receive(:new).and_call_original + allow(Aws::S3::Object).to receive(:new).with(key: 'test.mp4', bucket_name: 'source_bucket').and_return(@source_object) + allow(Aws::S3::Object).to receive(:new).with(key: 'test.mp4', bucket_name: 'dest_bucket').and_return(@dest_object) + allow(@source_object).to receive(:delete).and_return(true) + allow(@source_object).to receive(:download_file).and_return(true) + allow(@dest_object).to receive(:copy_from).and_return(true) + end + + describe 's3 source to s3 dest', s3: true do + it 'moves file from source to dest' do + expect(@dest_object).to receive(:copy_from).with(Aws::S3::Object.new(key: 'test.mp4', bucket_name: 'source_bucket'), multipart_copy: false) + expect(@source_object).to receive(:delete) + described_class.move(s3_file, s3_dest) + end + end + + describe 's3 source to filesystem dest', s3: true do + it 'moves file from source to dest' do + expect(@source_object).to receive(:download_file).with(fs_dest.uri.path) + expect(@source_object).to receive(:delete) + described_class.move(s3_file, fs_dest) + end + end + + describe 'filesystem source to s3 dest', s3: true do + it 'moves file from source to dest' do + allow(@dest_object).to receive(:upload_file).and_return(true) + expect(@dest_object).to receive(:upload_file).with(fs_file.uri.path) + expect(FileUtils).to receive(:rm).with(fs_file.uri.path) + described_class.move(fs_file, s3_dest) + end + end + + describe 'file system source to filesystem dest' do + before :each do + FakeFS.activate! + FileUtils.mkdir_p File.dirname(fs_location) + File.open(fs_location, 'w'){} + end + + after :each do + FakeFS.deactivate! + end + + it 'moves file from source to dest' do + expect(File.exist? fs_location).to be true + described_class.move(fs_file, fs_dest) + expect(File.exist? fs_location).to be false + expect(File.exist? '/path/to/dest/test.mp4').to be true + end + end + end +end \ No newline at end of file