Remove timeout on MessageOutboxDeliveryService (#437)

* Remove timeout on MessageOutboxDeliveryService * I believe this is how multi-indexes should be specified - https://www.mongodb.com/docs/drivers/csharp/current/fundamentals/indexes/#compound-indexes * Remove unused indices. Specify useful compound indices on pretranslations (and remove redundant index on engineRef). * Update docker-compose with Atlas. --------- Co-authored-by: John Lambert <[email protected]>
sillsdev · Jul 18, 2024 · af90cbc · af90cbc
1 parent 05f074e
commit af90cbc
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 34 deletions.
diff --git a/docker-compose.withatlas.yml b/docker-compose.withatlas.yml
@@ -14,8 +14,8 @@ services:
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__Endpoints__Http2__Url=http://*:81
       - ASPNETCORE_Kestrel__Endpoints__Http2__Protocols=Http2
-      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}/${MONGO_CONNECTION_PREFIX:?mongo prefix needed}serval_jobs
-      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}serval
+      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING:?connection string needed}/${MONGO_CONNECTION_PREFIX:?mongo prefix needed}serval_jobs${MONGO_EXTENSIONS}
+      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}serval${MONGO_EXTENSIONS}
       - ASPNETCORE_Translation__Engines__0__Type=Echo
       - ASPNETCORE_Translation__Engines__0__Address=http://echo
       - ASPNETCORE_Translation__Engines__1__Type=SmtTransfer
@@ -74,22 +74,24 @@ services:
     hostname: machine-engine
     container_name: machine-engine-cntr
     build:
-      context: ${MACHINE_TESTING_DIR:-../machine}
-      dockerfile: ../machine/dockerfile.development
+      context: .
+      dockerfile: dockerfile.development
 
     environment:
       - ASPNETCORE_ENVIRONMENT=Staging
       - ASPNETCORE_Kestrel__Endpoints__Https__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
-      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs
-      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine
+      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs${MONGO_EXTENSIONS}
+      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine${MONGO_EXTENSIONS}
       - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81
       - ClearML__ApiServer=https://api.sil.hosted.allegro.ai
-      - ClearML__Queue=lambert_24gb
-      - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
       - ClearML__Project=docker-compose
       - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}"
       - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}"
+      - BuildJob__ClearML__0__Queue=lambert_24gb
+      - BuildJob__ClearML__0__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
+      - BuildJob__ClearML__1__Queue=lambert_24gb.cpu_only
+      - BuildJob__ClearML__1__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only}
       - SharedFile__Uri=s3://aqua-ml-data/docker-compose/
       - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
       - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
@@ -100,11 +102,11 @@ services:
     depends_on:
       - serval-api
     volumes:
-      - ${MACHINE_TESTING_DIR:-../machine}:/app:ro
+      - .:/app:ro
       - ~/.nuget/packages:/root/.nuget/packages:ro
       - /var/lib/machine:/var/lib/machine
       - /var/lib/serval:/var/lib/serval
-    working_dir: '/app/src/SIL.Machine.Serval.EngineServer'
+    working_dir: '/app/src/Machine/src/Serval.Machine.EngineServer'
     entrypoint:
       - dotnet
       - run
@@ -117,21 +119,23 @@ services:
     hostname: machine-job-server
     container_name: machine-job-cntr
     build:
-      context: ${MACHINE_TESTING_DIR:-../machine}
-      dockerfile: ../machine/dockerfile.development
+      context: .
+      dockerfile: dockerfile.development
     environment:
       - ASPNETCORE_ENVIRONMENT=Staging
-      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs
-      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine
+      - ASPNETCORE_ConnectionStrings__Hangfire=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine_jobs${MONGO_EXTENSIONS}
+      - ASPNETCORE_ConnectionStrings__Mongo=${MONGO_CONNECTION_STRING}/${MONGO_CONNECTION_PREFIX}machine${MONGO_EXTENSIONS}
       - ASPNETCORE_ConnectionStrings__Serval=http://serval-api:81
       - ASPNETCORE_Kestrel__Endpoints__Http__Url=http://*:80
       - ASPNETCORE_Kestrel__EndpointDefaults__Protocols=Http2
       - ClearML__ApiServer=https://api.sil.hosted.allegro.ai
-      - ClearML__Queue=lambert_24gb
-      - ClearML__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
       - ClearML__Project=docker-compose
       - "ClearML__AccessKey=${ClearML_AccessKey:?access key needed}"
       - "ClearML__SecretKey=${ClearML_SecretKey:?secret key needed}"
+      - BuildJob__ClearML__0__Queue=lambert_24gb
+      - BuildJob__ClearML__0__DockerImage=${MACHINE_PY_IMAGE:-ghcr.io/sillsdev/machine.py:latest}
+      - BuildJob__ClearML__1__Queue=lambert_24gb.cpu_only
+      - BuildJob__ClearML__1__DockerImage=${MACHINE_PY_CPU_IMAGE:-ghcr.io/sillsdev/machine.py:latest.cpu_only}
       - SharedFile__Uri=s3://aqua-ml-data/docker-compose/
       - "SharedFile__S3AccessKeyId=${AWS_ACCESS_KEY_ID:?access key needed}"
       - "SharedFile__S3SecretAccessKey=${AWS_SECRET_ACCESS_KEY:?secret key needed}"
@@ -143,11 +147,11 @@ services:
       - machine-engine
       - serval-api
     volumes:
-      - ${MACHINE_TESTING_DIR:-../machine}:/app:ro
+      - .:/app:ro
       - ~/.nuget/packages:/root/.nuget/packages:ro
       - /var/lib/machine:/var/lib/machine
       - /var/lib/serval:/var/lib/serval
-    working_dir: '/app/src/SIL.Machine.Serval.JobServer'
+    working_dir: '/app/src/Machine/src/Serval.Machine.JobServer'
     entrypoint:
       - dotnet
       - run

diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs
@@ -258,9 +258,7 @@ public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, s
                     {
                         await c.Indexes.CreateOrUpdateAsync(
                             new CreateIndexModel<TranslationEngine>(
-                                Builders<TranslationEngine>
-                                    .IndexKeys.Ascending(e => e.EngineId)
-                                    .Ascending("currentBuild._id")
+                                Builders<TranslationEngine>.IndexKeys.Ascending(e => e.EngineId)
                             )
                         );
                         await c.Indexes.CreateOrUpdateAsync(

diff --git a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs
@@ -8,8 +8,6 @@ public class MessageOutboxDeliveryService(
     ILogger<MessageOutboxDeliveryService> logger
 ) : BackgroundService
 {
-    private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10);
-
     private readonly IServiceProvider _services = services;
     private readonly Dictionary<string, IOutboxMessageHandler> _outboxMessageHandlers =
         outboxMessageHandlers.ToDictionary(o => o.OutboxId);
@@ -25,7 +23,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         using ISubscription<OutboxMessage> subscription = await messages.SubscribeAsync(e => true, stoppingToken);
         while (true)
         {
-            await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken);
+            await subscription.WaitForChangeAsync(cancellationToken: stoppingToken);
             if (stoppingToken.IsCancellationRequested)
                 break;
             await ProcessMessagesAsync(messages, stoppingToken);

diff --git a/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMongoDataAccessConfiguratorExtensions.cs
@@ -15,9 +15,6 @@ this IMongoDataAccessConfigurator configurator
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Engine>(Builders<Engine>.IndexKeys.Ascending(e => e.Owner))
                 );
-                await c.Indexes.CreateOrUpdateAsync(
-                    new CreateIndexModel<Engine>(Builders<Engine>.IndexKeys.Ascending("corpora._id"))
-                );
             }
         );
         configurator.AddRepository<Build>(
@@ -33,21 +30,32 @@ await c.Indexes.CreateOrUpdateAsync(
             {
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Pretranslation>(
-                        Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.EngineRef)
+                        Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.ModelRevision)
                     )
                 );
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Pretranslation>(
-                        Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.ModelRevision)
+                        Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.CorpusRef)
                     )
                 );
+                await c.Indexes.CreateOrUpdateAsync(
+                    new CreateIndexModel<Pretranslation>(Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.TextId))
+                );
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Pretranslation>(
-                        Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.CorpusRef)
+                        Builders<Pretranslation>
+                            .IndexKeys.Ascending(pt => pt.EngineRef)
+                            .Ascending(pt => pt.ModelRevision)
                     )
                 );
                 await c.Indexes.CreateOrUpdateAsync(
-                    new CreateIndexModel<Pretranslation>(Builders<Pretranslation>.IndexKeys.Ascending(pt => pt.TextId))
+                    new CreateIndexModel<Pretranslation>(
+                        Builders<Pretranslation>
+                            .IndexKeys.Ascending(pt => pt.EngineRef)
+                            .Ascending(pt => pt.CorpusRef)
+                            .Ascending(pt => pt.ModelRevision)
+                            .Ascending(pt => pt.TextId)
+                    )
                 );
             }
         );

diff --git a/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs b/src/Serval/src/Serval.Webhooks/Configuration/IMongoDataAccessConfiguratorExtensions.cs
@@ -10,9 +10,6 @@ public static IMongoDataAccessConfigurator AddWebhooksRepositories(this IMongoDa
             "webhooks.hooks",
             init: async c =>
             {
-                await c.Indexes.CreateOrUpdateAsync(
-                    new CreateIndexModel<Webhook>(Builders<Webhook>.IndexKeys.Ascending(h => h.Owner))
-                );
                 await c.Indexes.CreateOrUpdateAsync(
                     new CreateIndexModel<Webhook>(Builders<Webhook>.IndexKeys.Ascending(h => h.Events))
                 );