From f153cb8c753df42c3034ba950f1ae34393b9cfe2 Mon Sep 17 00:00:00 2001 From: John Lambert Date: Tue, 3 Dec 2024 07:45:18 -0500 Subject: [PATCH] Add datafile and corpus update consumers Ensure that not all filenames change in testing --- ...iatorRegistrationConfiguratorExtensions.cs | 2 + .../Consumers/CorpusUpdatedConsumer.cs | 26 ++++++++++ .../Consumers/DataFileUpdatedConsumer.cs | 15 ++++++ .../Services/EngineService.cs | 50 +++++++++++++++---- .../Services/IEngineService.cs | 12 ++++- .../TranslationEngineTests.cs | 2 + 6 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs create mode 100644 src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs diff --git a/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs b/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs index 1397ed4d..d9b91b0f 100644 --- a/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs +++ b/src/Serval/src/Serval.Translation/Configuration/IMediatorRegistrationConfiguratorExtensions.cs @@ -7,6 +7,8 @@ this IMediatorRegistrationConfigurator configurator ) { configurator.AddConsumer(); + configurator.AddConsumer(); + configurator.AddConsumer(); return configurator; } } diff --git a/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs new file mode 100644 index 00000000..366e8099 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Consumers/CorpusUpdatedConsumer.cs @@ -0,0 +1,26 @@ +namespace Serval.Translation.Consumers; + +public class CorpusUpdatedConsumer(IEngineService engineService) : IConsumer +{ + private readonly IEngineService _engineService = engineService; + + public async Task Consume(ConsumeContext context) + { + await _engineService.UpdateCorpusFilesAsync( + context.Message.CorpusId, + context.Message.Files.Select(Map).ToList(), + context.CancellationToken + ); + } + + private static CorpusFile Map(CorpusFileResult corpusFile) + { + return new CorpusFile + { + Id = corpusFile.File.DataFileId, + TextId = corpusFile.TextId, + Filename = corpusFile.File.Filename, + Format = corpusFile.File.Format, + }; + } +} diff --git a/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs b/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs new file mode 100644 index 00000000..b75572f7 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Consumers/DataFileUpdatedConsumer.cs @@ -0,0 +1,15 @@ +namespace Serval.Translation.Consumers; + +public class DataFileUpdatedConsumer(IEngineService engineService) : IConsumer +{ + private readonly IEngineService _engineService = engineService; + + public async Task Consume(ConsumeContext context) + { + await _engineService.UpdateDataFileFilenameFilesAsync( + context.Message.DataFileId, + context.Message.Filename, + context.CancellationToken + ); + } +} diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index 5a370397..b9f1adeb 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -545,30 +545,62 @@ public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cance ); } - public Task UpdateAllCorpusFilesAsync( - DataFileUpdated dataFileUpdated, + public Task UpdateDataFileFilenameFilesAsync( + string dataFileId, + string filename, CancellationToken cancellationToken = default ) { return Entities.UpdateAllAsync( e => e.Corpora.Any(c => - c.SourceFiles.Any(f => f.Id == dataFileUpdated.DataFileId) - || c.TargetFiles.Any(f => f.Id == dataFileUpdated.DataFileId) + c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId) + ) + || e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId)) + || c.TargetCorpora.Any(mc => mc.Files.Any(f => f.Id == dataFileId)) ), u => - u.Set( - e => e.Corpora[ArrayPosition.All].SourceFiles[ArrayPosition.All].Filename, - dataFileUpdated.Filename + u.Set(e => e.Corpora[ArrayPosition.All].SourceFiles[ArrayPosition.All].Filename, filename) + .Set(e => e.Corpora[ArrayPosition.All].TargetFiles[ArrayPosition.All].Filename, filename) + .Set( + e => + e.ParallelCorpora[ArrayPosition.All] + .SourceCorpora[ArrayPosition.All] + .Files[ArrayPosition.All] + .Filename, + filename ) .Set( - e => e.Corpora[ArrayPosition.All].TargetFiles[ArrayPosition.All].Filename, - dataFileUpdated.Filename + e => + e.ParallelCorpora[ArrayPosition.All] + .TargetCorpora[ArrayPosition.All] + .Files[ArrayPosition.All] + .Filename, + filename ), cancellationToken ); } + public Task UpdateCorpusFilesAsync( + string corpusId, + IReadOnlyList files, + CancellationToken cancellationToken = default + ) + { + return Entities.UpdateAllAsync( + e => + e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(mc => mc.Id == corpusId) || c.TargetCorpora.Any(mc => mc.Id == corpusId) + ), + u => + u.Set(e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files, files) + .Set(e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files, files), + cancellationToken + ); + } + public async Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default) { TranslationEngineApi.TranslationEngineApiClient client = diff --git a/src/Serval/src/Serval.Translation/Services/IEngineService.cs b/src/Serval/src/Serval.Translation/Services/IEngineService.cs index 82648184..c25d522c 100644 --- a/src/Serval/src/Serval.Translation/Services/IEngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/IEngineService.cs @@ -68,7 +68,17 @@ Task DeleteParallelCorpusAsync( Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default); - Task UpdateAllCorpusFilesAsync(DataFileUpdated dataFileUpdated, CancellationToken cancellationToken = default); + Task UpdateDataFileFilenameFilesAsync( + string dataFileId, + string filename, + CancellationToken cancellationToken = default + ); + + Task UpdateCorpusFilesAsync( + string corpusId, + IReadOnlyList files, + CancellationToken cancellationToken = default + ); Task GetQueueAsync(string engineType, CancellationToken cancellationToken = default); diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index 127d4659..877e905b 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -2063,12 +2063,14 @@ await corporaClient.UpdateAsync( Engine newEngine1 = (await _env.Engines.GetAsync(ECHO_ENGINE1_ID))!; Assert.That(newEngine1.Corpora[0].SourceFiles[0].Filename, Is.EqualTo(newFileFromRepo.Filename)); + Assert.That(newEngine1.Corpora[0].TargetFiles[0].Filename, Is.EqualTo(FILE2_FILENAME)); Engine newEngine2 = (await _env.Engines.GetAsync(ECHO_ENGINE2_ID))!; Assert.That( newEngine2.ParallelCorpora[0].SourceCorpora[0].Files[0].Filename, Is.EqualTo(newFileFromRepo.Filename) ); + Assert.That(newEngine2.ParallelCorpora[0].TargetCorpora[0].Files[0].Filename, Is.EqualTo(FILE4_FILENAME)); DataFiles.Models.Corpus newCorpusFromRepo = (await _env.Corpora.GetAsync(TARGET_CORPUS_ID))!; Assert.That(newCorpusFromRepo.Files[0].FileId, Is.EqualTo(FILE4_TRG_ZIP_ID));