diff --git a/.gitignore b/.gitignore index c3f1151..8430916 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ !.env.development !.env.test *~undo-tree~ +tmp +log diff --git a/bin/build b/bin/build index 411addb..d32816a 100755 --- a/bin/build +++ b/bin/build @@ -23,6 +23,7 @@ rm -rf \ test \ tmp \ .ruby-lsp \ + log \ vendor/bundle/ruby/3.*/cache find . -iname "*~undo-tree~" -delete popd @@ -38,7 +39,8 @@ rm -rf \ README.md \ test \ tmp \ - .ruby-lsp \ + .ruby-lsp \ + log \ vendor/bundle/ruby/3.*/cache find . -iname "*~undo-tree~" -delete popd @@ -54,7 +56,8 @@ rm -rf \ README.md \ test \ tmp \ - .ruby-lsp \ + .ruby-lsp \ + log \ vendor/bundle/ruby/3.*/cache find . -iname "*~undo-tree~" -delete popd diff --git a/bin/ia_to_s3_migrator b/bin/ia_to_s3_migrator new file mode 100755 index 0000000..9946c8c --- /dev/null +++ b/bin/ia_to_s3_migrator @@ -0,0 +1,25 @@ +#!/usr/bin/env ruby + +# Usage: bin/migrate_iaids + +require 'ruby-progressbar' +require_relative '../lib/space_stone' + +puts '== Tail log/ia_to_s3_migrator.log for logs ==' + +iaids = File.read('tmp/iaids.txt').split("\n") +logger = Logger.new('log/ia_to_s3_migrator.log') +progressbar = ProgressBar.create(total: iaids.size, format: '%a %e %P% Processed: %c from %C') +s3_bucket = SpaceStone::S3Service.bucket + +iaids.each do |iaid| + # WARN: This dumbly checks for any downloads; if some files have been uploaded + # but some haven't, it will skip uploading all of them + if s3_bucket.objects(prefix: "#{iaid}/downloads").any? + logger.warn("== #{iaid} == Files have already been uploaded to S3, skipping") + else + logger.info("== #{iaid} == Downloading files...") + process_ia_id(iaid, '/store/tmp/fast-tmp') + end + progressbar.increment +end diff --git a/bin/migrate_iaids b/bin/migrate_iaids new file mode 100755 index 0000000..89f88e1 --- /dev/null +++ b/bin/migrate_iaids @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +STAGE_ENV=production docker compose run \ + --rm \ + --remove-orphans \ + --volume /store/tmp/fast-tmp:/store/tmp/fast-tmp \ + web \ + 'bundle exec bin/ia_to_s3_migrator' diff --git a/docker-compose.yml b/docker-compose.yml index aeac80c..6ab678e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,6 @@ services: - .:/var/task:delegated - /var/run/docker.sock:/var/run/docker.sock command: "sam local start-api --host '0.0.0.0' --port 3030 --docker-volume-basedir ${PWD}" - entrypoint: /bin/bash + entrypoint: /bin/bash -c ports: - 3030:3030 diff --git a/lib/space_stone/sqs_service.rb b/lib/space_stone/sqs_service.rb index 8687eea..c65cd78 100644 --- a/lib/space_stone/sqs_service.rb +++ b/lib/space_stone/sqs_service.rb @@ -6,7 +6,11 @@ module SpaceStone # Service object to add messages to either sqs queue module SqsService def client - @client ||= Aws::SQS::Client.new(region: 'us-east-2') + @client ||= if ENV.fetch('AWS_S3_ACCESS_KEY_ID', nil) + Aws::SQS::Client.new(region: 'us-east-2', credentials: Aws::Credentials.new(ENV.fetch('AWS_S3_ACCESS_KEY_ID'), ENV.fetch('AWS_S3_SECRET_ACCESS_KEY'))) + else + Aws::SQS::Client.new(region: 'us-east-2') + end end def ocr_queue_url