Merge branch 'main' into dbai-29-canister-container

.github/workflows/tests.yml

@@ -0,0 +1,40 @@
+name: Run Tests
+
+on: push
+
+jobs:
+  # Run tests
+  test:
+    runs-on: ubuntu-latest
+    services:
+      mariadb:
+        image: mariadb:10
+        env:
+          MARIADB_USER: user
+          MARIADB_PASSWORD: password
+          MARIADB_DATABASE: test_database
+          MARIADB_ROOT_PASSWORD: password
+        ports: ["3306:3306"]
+        options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=5
+    steps:
+      - uses: actions/checkout@v3
+      - name: Wait for MariaDB
+        run: |
+          for i in {1..10}
+          do
+              if [[ $(mysqladmin ping -h"127.0.0.1" -P"3306" --silent) -eq 0 ]]; then
+                  exit 0
+              else
+                  sleep 3
+              fi
+          done
+          exit 1
+      - name: Set up Ruby 3.2
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2.3'
+          bundler-cache: true
+      - name: Run tests
+        run: bundle exec rake test
+        env: 
+          CONFIG_YML_PATH: "./config/config.tests.yml"

Rakefile

@@ -14,7 +14,6 @@ namespace :db do
 
     Sequel.extension :migration
 
-    # db_config = db_config
     if !db_config
       raise DatabaseError, "Migration failed. A database connection is not configured."
     end

config/config.tests.yml

@@ -0,0 +1,8 @@
+# Configuration for GitHub Action defined in .github/workflows/tests.yaml
+
+Database:
+  Host: 127.0.0.1
+  Database: test_database
+  Port: 3306
+  User: user
+  Password: password

docker-compose.yml

@@ -12,6 +12,7 @@ services:
         target: /run/host-services/ssh-auth.sock
     environment:
      - SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock
+     - CONFIG_YML_PATH=./config/config.tests.yml
     depends_on:
       database:
         condition: "service_healthy"

lib/bag_tag.rb

@@ -20,22 +20,28 @@ def initialize(
       identifier:,
       description:,
       bag_count: [1, 1],
-      organization: VALUE_SOURCE_ORG_DEFAULT
+      organization: VALUE_SOURCE_ORG_DEFAULT,
+      extra_data: nil
     )
       @identifier = identifier
       @description = description
       @bag_count = bag_count
       @organization = organization
+      @extra_data = extra_data
     end
 
     def data
-      {
+      data = {
         KEY_SOURCE_ORG => @organization,
         KEY_COUNT => "#{@bag_count[0]} of #{@bag_count[1]}",
         KEY_DATE => BagInfoBagTag.datetime_now,
         KEY_INTERNAL_SENDER_ID => @identifier,
         KEY_INTERNAL_SENDER_DESC => @description
       }
+      if @extra_data
+        data = data.merge(@extra_data)
+      end
+      data
     end
   end
 
@@ -124,7 +130,7 @@ def data
         Access: @access,
         "Storage-Option": @storage_option
       }
-      if !@extra_data.nil?
+      if @extra_data
         data = data.merge(@extra_data)
       end
       data

lib/config.rb

@@ -176,6 +176,16 @@ def self.create_remote_config(data)
       )
     end
 
+    def self.create_database_config(data)
+      DatabaseConfig.new(
+        host: verify_string("Host", data["Host"]),
+        database: verify_string("Database", data["Database"]),
+        port: verify_int("Port", data["Port"]),
+        user: verify_string("User", data["User"]),
+        password: verify_string("Password", data["Password"])
+      )
+    end
+
     def self.create_config(data)
       db_data = data.fetch("Database", nil)
       aptrust_data = data["APTrust"]
@@ -191,13 +201,7 @@ def self.create_config(data)
           name: verify_string("Repository", data["Repository"]),
           description: verify_string("RepositoryDescription", data["RepositoryDescription"])
         ),
-        database: db_data && DatabaseConfig.new(
-          host: verify_string("Host", db_data["Host"]),
-          database: verify_string("Database", db_data["Database"]),
-          port: verify_int("Port", db_data["Port"]),
-          user: verify_string("User", db_data["User"]),
-          password: verify_string("Password", db_data["Password"])
-        ),
+        database: db_data && create_database_config(db_data),
         dark_blue: DarkBlueConfig.new(
           archivematicas: (
             data["DarkBlue"]["ArchivematicaInstances"].map do |arch_data|
@@ -228,6 +232,12 @@ def self.create_config(data)
       )
     end
 
+    def self.database_config_from_file(yaml_path)
+      data = read_data_from_file(yaml_path)
+      db_data = data.fetch("Database", nil)
+      db_data && create_database_config(db_data)
+    end
+
     def self.from_file(yaml_path)
       create_config(read_data_from_file(yaml_path))
     end

lib/dispatcher.rb

@@ -6,7 +6,13 @@
 
 module Dispatcher
   class DispatcherBase
-    def dispatch(object_metadata:, data_transfer:, context: nil, validator: nil)
+    def dispatch(
+      object_metadata:,
+      data_transfer:,
+      context: nil,
+      validator: nil,
+      extra_bag_info_data: nil
+    )
       raise NotImplementedError
     end
   end
@@ -28,15 +34,22 @@ def initialize(
       @bag_repo = bag_repo
     end
 
-    def dispatch(object_metadata:, data_transfer:, context: nil, validator: nil)
+    def dispatch(
+      object_metadata:,
+      data_transfer:,
+      context: nil,
+      validator: nil,
+      extra_bag_info_data: nil
+    )
       bag_id = BagCourier::BagId.new(
         repository: @repository.name,
         object_id: object_metadata.id,
         context: context
       )
       bag_info = BagTag::BagInfoBagTag.new(
         identifier: object_metadata.id,
-        description: @repository.description
+        description: @repository.description,
+        extra_data: extra_bag_info_data
       )
       tags = [
         BagTag::AptrustInfoBagTag.new(

run_dark_blue.rb

@@ -20,6 +20,11 @@ class DarkBlueError < StandardError
 
 class DarkBlueJob
   include DarkBlueLogger
+  module ExtraBagInfoData
+    CONTENT_TYPE_KEY = "Dark-Blue-Content-Type"
+    LOCATION_UUID_KEY = "Archivematica-Location-UUID"
+  end
+
   def initialize(config)
     @package_repo = RepositoryPackageRepository::RepositoryPackageRepositoryFactory.for(use_db: DB)
     @dispatcher = Dispatcher::APTrustDispatcher.new(
@@ -49,15 +54,23 @@ def prepare_arch_service(name:, api_config:)
   end
   private :prepare_arch_service
 
-  def deliver_package(package_data:, remote_client:, context:)
+  def create_extra_bag_info_data(content_type:, location_uuid:)
+    {
+      ExtraBagInfoData::CONTENT_TYPE_KEY => content_type,
+      ExtraBagInfoData::LOCATION_UUID_KEY => location_uuid
+    }
+  end
+
+  def deliver_package(package_data:, remote_client:, context:, extra_bag_info_data:)
     courier = @dispatcher.dispatch(
       object_metadata: package_data.metadata,
       data_transfer: DataTransfer::RemoteClientDataTransfer.new(
         remote_client: remote_client,
         remote_path: package_data.remote_path
       ),
       context: context,
-      validator: InnerBagValidator.new(package_data.dir_name)
+      validator: InnerBagValidator.new(package_data.dir_name),
+      extra_bag_info_data: extra_bag_info_data
     )
     courier.deliver
   end
@@ -77,6 +90,9 @@ def redeliver_package(identifier)
       raise DarkBlueError, message
     end
 
+    extra_bag_info_data = create_extra_bag_info_data(
+      content_type: arch_config.name, location_uuid: api_config.location_uuid
+    )
     arch_service = prepare_arch_service(name: arch_config.name, api_config: arch_config.api)
     package_data = arch_service.get_package_data_object(package.identifier)
     unless package_data
@@ -92,7 +108,8 @@ def redeliver_package(identifier)
     deliver_package(
       package_data: package_data,
       remote_client: source_remote_client,
-      context: arch_config.name
+      context: arch_config.name,
+      extra_bag_info_data: extra_bag_info_data
     )
   end
   private :redeliver_package
@@ -102,6 +119,9 @@ def redeliver_packages(package_identifiers)
   end
 
   def process_arch_instance(arch_config)
+    extra_bag_info_data = create_extra_bag_info_data(
+      content_type: arch_config.name, location_uuid: arch_config.api.location_uuid
+    )
     arch_service = prepare_arch_service(name: arch_config.name, api_config: arch_config.api)
     source_remote_client = RemoteClient::RemoteClientFactory.from_config(
       type: arch_config.remote.type,
@@ -129,7 +149,8 @@ def process_arch_instance(arch_config)
       deliver_package(
         package_data: package_data,
         remote_client: source_remote_client,
-        context: arch_config.name
+        context: arch_config.name,
+        extra_bag_info_data: extra_bag_info_data
       )
     end
   end

run_example.rb

@@ -7,7 +7,9 @@
 
 SemanticLogger.add_appender(io: $stderr, formatter: :color)
 
-config = Config::ConfigService.from_file(File.join(".", "config", "config.yml"))
+config = Config::ConfigService.from_file(
+  ENV.fetch("CONFIG_YML_PATH", File.join(".", "config", "config.yml"))
+)
 
 SemanticLogger.default_level = config.settings.log_level
 logger = SemanticLogger["run_example"]

test/setup_db.rb

@@ -6,11 +6,11 @@
 
 Sequel.extension :migration
 
-config = Config::ConfigService.from_file(
-  File.join(__dir__, "..", "config", "config.yml")
+config_yml_path = ENV.fetch("CONFIG_YML_PATH", File.join(".", "config", "config.yml"))
+db_config = Config::ConfigService.database_config_from_file(
+  File.join(__dir__, "..", config_yml_path)
 )
 
-db_config = config.database
 if !db_config
   message = "A database connection is not configured. This is required for some tests."
   raise DatabaseError, message

test/test_bag_courier.rb

@@ -55,7 +55,7 @@ def setup
     @prep_path = File.join(@test_dir_path, "prep")
     @export_path = File.join(@test_dir_path, "export")
     @package_path = File.join(@test_dir_path, "package")
-    FileUtils.rm_r(@test_dir_path)
+    FileUtils.rm_r(@test_dir_path) if File.exist?(@test_dir_path)
     FileUtils.mkdir_p([@test_dir_path, @prep_path, @export_path, @package_path])
     innerbag = BagAdapter::BagAdapter.new(@package_path)
 

test/test_bag_tag.rb

@@ -35,14 +35,16 @@ def test_bag_info_tag_data_without_defaults
         identifier: "2124796",
         description: "Bag from repository X containing item to be preserved",
         bag_count: [2, 3],
-        organization: "Mythical University"
+        organization: "Mythical University",
+        extra_data: {"Context" => "Some important detail"}
       )
       expected = {
         "Source-Organization" => "Mythical University",
         "Bag-Count" => "2 of 3",
         "Bagging-Date" => "2023-12-07T12:00:00Z",
         "Internal-Sender-Identifier" => "2124796",
-        "Internal-Sender-Description" => "Bag from repository X containing item to be preserved"
+        "Internal-Sender-Description" => "Bag from repository X containing item to be preserved",
+        "Context" => "Some important detail"
       }
       assert_equal expected, bag_info.data
     end
@@ -76,9 +78,7 @@ def test_aptrust_tag_serialize_with_no_data_defaults
     test_data = @base_test_data.merge({
       access: "Consortia",
       storage_option: "Glacier-Deep-OR",
-      extra_data: {
-        Context: "Some important detail"
-      }
+      extra_data: {"Context" => "Some important detail"}
     })
     aptrust_info = BagTag::AptrustInfoBagTag.new(**test_data)
     expected = <<~TEXT