Merge branch 'main' into dbai-29-canister-container

.dockerignore

@@ -1,4 +1,8 @@
+# Configuration
 **/config.yml
+**/.env*
 .ssh/*
+
+# Data directories
 export/*
 prep/*

.github/workflows/tests.yml

@@ -37,4 +37,8 @@ jobs:
       - name: Run tests
         run: bundle exec rake test
         env: 
-          CONFIG_YML_PATH: "./config/config.tests.yml"
+          DATABASE_HOST: 127.0.0.1
+          DATABASE_DATABASE: test_database
+          DATABASE_PORT: 3306
+          DATABASE_USER: user
+          DATABASE_PASSWORD: password

.gitignore

@@ -1,14 +1,20 @@
+# Data directories
 export
 prep
 restore
 source
 
-.DS_Store
+# Configuration
 .ssh
+**/.env*
+# Former configuration file format; leaving here to prevent accidental inclusion
+**/config.yml
 
-config/config.yml
-
+# Testing directories
 test/bag_test_dir
 test/remote_test
 test/bag_courier_test
 test/test_bag_val_dir
+
+# Misc.
+.DS_Store

Dockerfile

@@ -22,15 +22,13 @@ COPY . /app
 
 FROM base AS development
 
-RUN bundle install
+RUN bundle config set --local without development && bundle install
 
 CMD ["tail", "-f", "/dev/null"]
 
 FROM base AS production
 
-RUN bundle config set --local without test
-
-RUN bundle install
+RUN bundle config set --local without development test && bundle install
 
 RUN groupadd -g ${GID} -o ${UNAME}
 RUN useradd -m -d /app -u ${UID} -g ${GID} -o -s /bin/bash ${UNAME}

Gemfile

@@ -16,6 +16,10 @@ gem "sftp",
   git: "https://github.com/mlibrary/sftp",
   tag: "sftp/v0.4.2"
 
+group :development do
+  gem "dotenv", "~> 3.1.0"
+end
+
 group :test do
   gem "minitest", "~> 5.20"
   gem "rake", "~> 13.1"

Gemfile.lock

@@ -33,6 +33,7 @@ GEM
     canister (0.9.2)
     concurrent-ruby (1.2.3)
     docopt (0.5.0)
+    dotenv (3.1.0)
     faraday (2.9.0)
       faraday-net_http (>= 2.0, < 3.2)
     faraday-net_http (3.1.0)
@@ -63,6 +64,7 @@ DEPENDENCIES
   aws-sdk-s3 (~> 1.136)
   bagit (~> 0.4.6)
   canister
+  dotenv (~> 3.1.0)
   faraday (~> 2.9)
   faraday-retry (~> 2.2)
   minitar (~> 0.9)

README.md

@@ -24,11 +24,11 @@ may be IP address restricted, so ensure you have access to the proper network or
 
 In all cases, you will need to set up a configuration file, which you can do by following these steps:
 
-1. Create a configuration YAML file based on the provided template.
+1. Create a `.env` file based on the provided template, [`example.env`](/example.env).
     ```sh
-    cp config/config.example.yml config/config.yml
+    cp example.env .env
     ```
-2. Open `config/config.yml` using your preferred text editor, and add values,
+2. Open `.env` using your preferred text editor, and add values,
 using the comments to guide you.
 
 For local development, you may need to set up an SSH key and add it to your `ssh-agent`
@@ -46,7 +46,7 @@ The recommended approach is to use the `database` service
 (defined in [`docker-compose.yml`](/docker-compose.yml) with the Docker approach described below.
 However, it is possible -- though not explicitly supported -- to use a MariaDB or MySQL
 database from a server running on your local machine or elsewhere.
-The following sections will assume you are not using a database with Ruby alone.
+The following sections will assume you are not using a database with Ruby outside a container.
 
 ### Ruby
 
@@ -59,28 +59,32 @@ bundle install
 
 #### Usage
 
-The primary job or process is `run_dark_blue.rb`.
-```sh
-ruby run_dark_blue.rb
-```
+*Note*: The application expects to find necessary configuration in environment variables.
+To assist with local development, it includes the [`dotenv`](https://github.com/bkeepers/dotenv) gem,
+which will populate the Ruby global `ENV` with key-value pairs from the prepared `.env` file.
+When using only Ruby to execute a file, if configuration is needed,
+precede your `ruby script.rb` with `dotenv`, as shown below.
 
-The latest deposits to APTrust can be verified using `verify_aptrust.rb`.
+The primary job or process is `run_dark_blue.rb`.
 ```sh
-ruby verify_aptrust.rb
+dotenv ruby run_dark_blue.rb
 ```
 
 If you're working on a new job or just want to try out the classes,
 you can copy `run_example.rb` and modify it as necessary.
 ```sh
 cp run_example.rb run_test.rb
 # Tweak as necessary
-ruby run_test.rb
+dotenv ruby run_test.rb
 ```
 
 ### Docker
 
 #### Installation
 
+*Note*: The provided `docker-compose.yml` file will detect your `.env` and create the appropriate
+environment variables.
+
 Build the image for the `dark-blue` service.
 ```sh
 docker compose build dark-blue
@@ -103,7 +107,8 @@ Run the `dark-blue` service to start `run_dark_blue.rb`.
 docker compose up dark-blue
 ```
 
-To use `verify_aptrust.rb`, override the entry command for the `dark-blue` service with `run`.
+The latest deposits to APTrust can be verified using `verify_aptrust.rb`.
+Override the entry command for the `dark-blue` service with `run`.
 ```sh
 docker compose run dark-blue ruby verify_aptrust.rb
 ```

Rakefile

@@ -14,6 +14,7 @@ namespace :db do
 
     Sequel.extension :migration
 
+    db_config = Config::ConfigService.database_config_from_env
     if !db_config
       raise DatabaseError, "Migration failed. A database connection is not configured."
     end

docker-compose.yml

@@ -12,7 +12,7 @@ services:
         target: /run/host-services/ssh-auth.sock
     environment:
      - SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock
-     - CONFIG_YML_PATH=./config/config.tests.yml
+    env_file: .env
     depends_on:
       database:
         condition: "service_healthy"
@@ -21,9 +21,9 @@ services:
     image: mariadb:11
     environment:
       - MARIADB_ROOT_PASSWORD=root_pw
-      - MARIADB_DATABASE=dpact_pres_service
-      - MARIADB_USER=dpact_user
-      - MARIADB_PASSWORD=dpact_pw
+      - MARIADB_DATABASE=darkblue_aptrust
+      - MARIADB_USER=darkblue_aptrust_user
+      - MARIADB_PASSWORD=darkblue_aptrust_pw
     command: [
       '--character-set-server=utf8mb4',
       '--collation-server=utf8mb4_unicode_ci'

example.env

@@ -0,0 +1,57 @@
+# Bag Courier Environment Variables Example/Template
+
+# Settings
+# One of info, debug, error, warn, trace, fatal
+SETTINGS_LOG_LEVEL=debug
+SETTINGS_WORKING_DIR=./prep
+SETTINGS_EXPORT_DIR=./export
+# Determines whether the process skips sending bag(s)
+SETTINGS_DRY_RUN=false
+# Limit for the size (in bytes) of objects to be processed (optional)
+# This is useful for development or in environments where larger files cannot be processed.
+SETTINGS_OBJECT_SIZE_LIMIT=
+
+# Repository
+# Name that will be used for the repository at the beginning of bag identifiers
+REPOSITORY_NAME=
+# Generic description that will be shared among all items in a repository
+REPOSITORY_DESCRIPTION=
+
+# Database (optional)
+# Values provided should work for local development with Docker
+# While technically optional, it is required when running all tests.
+DATABASE_HOST=database
+DATABASE_DATABASE=darkblue_aptrust
+DATABASE_PORT=3306
+DATABASE_USER=darkblue_aptrust_user
+DATABASE_PASSWORD=darkblue_aptrust_pw
+
+# Archivematica instance(s)
+# Settings for each Archivematica instance should have the same stem,
+# one of ARCHIVEMATICA_DEV, ARCHIVEMATICA_AMI, ARCHIVEMATICA_LAB, or ARCHIVEMATICA_VGA
+ARCHIVEMATICA_DEV_NAME=
+ARCHIVEMATICA_DEV_REPOSITORY_NAME=
+# REST API
+ARCHIVEMATICA_DEV_API_USERNAME=
+ARCHIVEMATICA_DEV_API_API_KEY=
+ARCHIVEMATICA_DEV_BASE_URL=
+# Remote
+ARCHIVEMATICA_DEV_REMOTE_TYPE=file_system
+ARCHIVEMATICA_DEV_REMOTE_SETTINGS_FILE_SYSTEM_REMOTE_PATH=
+# ARCHIVEMATICA_DEV_REMOTE_TYPE=sftp
+# ARCHIVEMATICA_DEV_REMOTE_SETTINGS_USER=
+# ARCHIVEMATICA_DEV_REMOTE_SETTINGS_HOST=
+# ARCHIVEMATICA_DEV_REMOTE_SETTINGS_KEY_PATH=
+
+# APTrust
+# REST API settings
+APTRUST_API_USERNAME=
+APTRUST_API_API_KEY=
+APTRUST_API_BASE_URL=
+# Remote settings
+APTRUST_REMOTE_TYPE=aptrust
+APTRUST_REMOTE_SETTINGS_RECEIVING_BUCKET=
+APTRUST_REMOTE_SETTINGS_RESTORE_BUCKET=
+APTRUST_REMOTE_SETTINGS_BUCKET_REGION=
+APTRUST_REMOTE_SETTINGS_AWS_ACCESS_KEY_ID=
+APTRUST_REMOTE_SETTINGS_AWS_SECRET_ACCESS_KEY=