Merge pull request #8 from scientist-softserv/ia-to-s3-migrator-script

IA to S3 migrator script

.gitignore

@@ -5,3 +5,5 @@
 !.env.development
 !.env.test
 *~undo-tree~
+tmp
+log

bin/build

@@ -23,6 +23,7 @@ rm -rf \
   test \
   tmp \
   .ruby-lsp \
+  log \
   vendor/bundle/ruby/3.*/cache
 find . -iname "*~undo-tree~" -delete
 popd
@@ -38,7 +39,8 @@ rm -rf \
    README.md \
    test \
    tmp \
-  .ruby-lsp \
+   .ruby-lsp \
+   log \
    vendor/bundle/ruby/3.*/cache
 find . -iname "*~undo-tree~" -delete
 popd
@@ -54,7 +56,8 @@ rm -rf \
    README.md \
    test \
    tmp \
-  .ruby-lsp \
+   .ruby-lsp \
+   log \
    vendor/bundle/ruby/3.*/cache
 find . -iname "*~undo-tree~" -delete
 popd

bin/ia_to_s3_migrator

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+# Usage: bin/migrate_iaids
+
+require 'ruby-progressbar'
+require_relative '../lib/space_stone'
+
+puts '== Tail log/ia_to_s3_migrator.log for logs =='
+
+iaids = File.read('tmp/iaids.txt').split("\n")
+logger = Logger.new('log/ia_to_s3_migrator.log')
+progressbar = ProgressBar.create(total: iaids.size, format: '%a %e %P% Processed: %c from %C')
+s3_bucket = SpaceStone::S3Service.bucket
+
+iaids.each do |iaid|
+  # WARN: This dumbly checks for any downloads; if some files have been uploaded
+  # but some haven't, it will skip uploading all of them
+  if s3_bucket.objects(prefix: "#{iaid}/downloads").any?
+    logger.warn("== #{iaid} == Files have already been uploaded to S3, skipping")
+  else
+    logger.info("== #{iaid} == Downloading files...")
+    process_ia_id(iaid, '/store/tmp/fast-tmp')
+  end
+  progressbar.increment
+end

bin/migrate_iaids

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+STAGE_ENV=production docker compose run \
+  --rm \
+  --remove-orphans \
+  --volume /store/tmp/fast-tmp:/store/tmp/fast-tmp \
+  web \
+  'bundle exec bin/ia_to_s3_migrator'

docker-compose.yml

@@ -16,6 +16,6 @@ services:
       - .:/var/task:delegated
       - /var/run/docker.sock:/var/run/docker.sock
     command: "sam local start-api --host '0.0.0.0' --port 3030 --docker-volume-basedir ${PWD}"
-    entrypoint: /bin/bash
+    entrypoint: /bin/bash -c
     ports:
       - 3030:3030

lib/space_stone/sqs_service.rb

@@ -6,7 +6,11 @@ module SpaceStone
   # Service object to add messages to either sqs queue
   module SqsService
     def client
-      @client ||= Aws::SQS::Client.new(region: 'us-east-2')
+      @client ||= if ENV.fetch('AWS_S3_ACCESS_KEY_ID', nil)
+                    Aws::SQS::Client.new(region: 'us-east-2', credentials: Aws::Credentials.new(ENV.fetch('AWS_S3_ACCESS_KEY_ID'), ENV.fetch('AWS_S3_SECRET_ACCESS_KEY')))
+                  else
+                    Aws::SQS::Client.new(region: 'us-east-2')
+                  end
     end
 
     def ocr_queue_url