From a57443981e3f09bd0e398dc00969a933b91ad61b Mon Sep 17 00:00:00 2001 From: John Lambert Date: Mon, 14 Oct 2024 12:08:19 -0400 Subject: [PATCH] Fix merge errors --- .../WordAlignmentEngineServiceV1.cs | 15 +- src/Serval/src/Serval.Client/Client.g.cs | 289 ++++++-------- .../Serval.Shared/Controllers/Endpoints.cs | 2 +- .../Models/MonolingualCorpus.cs | 2 +- .../Models/TrainingCorpus.cs | 2 +- .../Services/EngineService.cs | 6 +- .../ParallelCorpusFilterConfigDto.cs | 8 + .../Contracts/ParallelCorpusFilterDto.cs | 8 + .../Contracts/TrainingCorpusConfigDto.cs | 6 +- .../Contracts/TrainingCorpusDto.cs | 8 +- .../Contracts/WordAlignOnCorpusConfigDto.cs | 6 +- .../Contracts/WordAlignOnCorpusDto.cs | 8 +- .../Contracts/WordAlignmentCorpusConfigDto.cs | 17 - .../Contracts/WordAlignmentCorpusDto.cs | 13 - .../WordAlignmentCorpusFileConfigDto.cs | 8 - .../Contracts/WordAlignmentCorpusFileDto.cs | 7 - .../WordAlignmentCorpusUpdateConfigDto.cs | 23 -- .../WordAlignmentParallelCorpusConfigDto.cs | 12 + .../WordAlignmentParallelCorpusDto.cs | 10 + .../WordAlignmentParallelCorpusUpdateDto.cs | 23 ++ .../WordAlignmentEnginesController.cs | 359 +++++++++--------- .../src/Serval.WordAlignment/Models/Build.cs | 2 +- .../src/Serval.WordAlignment/Models/Corpus.cs | 11 - .../src/Serval.WordAlignment/Models/Engine.cs | 2 +- .../Models/ParallelCorpus.cs | 8 + .../Models/TrainingCorpus.cs | 8 + .../Models/WordAlignmentCorpus.cs | 2 +- .../Services/EngineService.cs | 320 ++++++++-------- .../Services/IEngineService.cs | 15 +- .../Services/EngineServiceTests.cs | 8 +- 30 files changed, 595 insertions(+), 613 deletions(-) rename src/Serval/src/{Serval.Translation => Serval.Shared}/Models/MonolingualCorpus.cs (86%) rename src/Serval/src/{Serval.Shared => Serval.Translation}/Models/TrainingCorpus.cs (91%) create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs delete mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs delete mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs delete mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileConfigDto.cs delete mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileDto.cs delete mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusUpdateConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs delete mode 100644 src/Serval/src/Serval.WordAlignment/Models/Corpus.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs create mode 100644 src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs diff --git a/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs b/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs index a9b3128f..461938ff 100644 --- a/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs +++ b/src/Echo/src/EchoEngine/WordAlignmentEngineServiceV1.cs @@ -76,9 +76,6 @@ await client.BuildStartedAsync( { foreach (ParallelCorpus corpus in request.Corpora) { - if (!corpus.WordAlignOnAll && corpus.WordAlignOnTextIds.Count == 0) - continue; - var sourceFiles = corpus .SourceCorpora.SelectMany(sc => sc.Files.Where(f => @@ -92,9 +89,15 @@ await client.BuildStartedAsync( ) .ToDictionary(f => f.TextId, f => f.Location); var targetFiles = corpus - .TargetFiles.Where(f => - (corpus.WordAlignOnAll || corpus.WordAlignOnTextIds.Contains(f.TextId)) - && f.Format == FileFormat.Text + .TargetCorpora.SelectMany(tc => + tc.Files.Where(f => + ( + tc.WordAlignOnAll + || tc.WordAlignOnTextIds is null + || tc.WordAlignOnTextIds.Contains(f.TextId) + ) + && f.Format == FileFormat.Text + ) ) .ToDictionary(f => f.TextId, f => f.Location); diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 1c41cc49..7b45ddc1 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -9008,77 +9008,64 @@ public partial interface IWordAlignmentEnginesClient /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Add a corpus to a engine + /// Add a parallel corpus to an engine /// /// /// ## Parameters - ///
* **name**: A name to help identify and distinguish the corpus from other corpora - ///
* The name does not have to be unique since the corpus is uniquely identified by an auto-generated id - ///
* **sourceLanguage**: The source language code (See documentation on endpoint /word-alignment/engines/ - "Create a new engine" for details on language codes). - ///
* Normally, this is the same as the engine sourceLanguage. This may change for future engines as a means of transfer learning. - ///
* **targetLanguage**: The target language code (See documentation on endpoint /word-alignment/engines/ - "Create a new engine" for details on language codes). - ///
* **SourceFiles**: The source files associated with the corpus - ///
* **FileId**: The unique id referencing the uploaded file - ///
* **TextId**: The client-defined name to associate source and target files. - ///
* If the TextIds in the SourceFiles and TargetFiles match, they will be used to train the engine. - ///
* If a TextId is used more than once in SourceFiles, the sources will be randomly and evenly mixed for training. - ///
* For Paratext projects, TextId will be ignored - multiple Paratext source projects will always be mixed (as if they have the same TextId). - ///
* **TargetFiles**: The target files associated with the corpus - ///
* Same as SourceFiles, except only a single instance of a TextID or a single paratext project is supported. There is no mixing or combining of multiple targets. + ///
* **SourceCorpusIds**: The source corpora associated with the parallel corpus + ///
* **TargetCorpusIds**: The target corpora associated with the parallel corpus /// /// The engine id /// The corpus configuration (see remarks) /// The added corpus /// A server side error occurred. - System.Threading.Tasks.Task AddCorpusAsync(string id, WordAlignmentCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task AddParallelCorpusAsync(string id, WordAlignmentParallelCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all corpora for a engine + /// Get all parallel corpora for a engine /// /// The engine id - /// The files + /// The parallel corpora /// A server side error occurred. - System.Threading.Tasks.Task> GetAllCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task> GetAllParallelCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Update a corpus with a new set of files + /// Update a parallel corpus with a new set of corpora /// /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - ///
Will not affect jobs already queued or running. Will not affect existing word alignments until new build is complete. + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing word graphs until new build is complete. /// /// The engine id - /// The corpus id + /// The parallel corpus id /// The corpus configuration /// The corpus was updated successfully /// A server side error occurred. - System.Threading.Tasks.Task UpdateCorpusAsync(string id, string corpusId, WordAlignmentCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task UpdateParallelCorpusAsync(string id, string parallelCorpusId, WordAlignmentParallelCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the configuration of a corpus for a engine + /// Get the configuration of a parallel corpus for a engine /// /// The engine id - /// The corpus id - /// The corpus configuration + /// The parallel corpus id + /// The parallel corpus configuration /// A server side error occurred. - System.Threading.Tasks.Task GetCorpusAsync(string id, string corpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task GetParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Remove a corpus from a engine + /// Remove a parallel corpus from a engine /// /// - /// Removing a corpus will remove all word alignments associated with that corpus. + /// Removing a parallel corpus will remove all word alignments associated with that corpus. /// /// The engine id - /// The corpus id - /// If true, all files associated with the corpus will be deleted as well (even if they are associated with other corpora). If false, no files will be deleted. - /// The corpus was deleted successfully. + /// The parallel corpus id + /// The parallel corpus was deleted successfully. /// A server side error occurred. - System.Threading.Tasks.Task DeleteCorpusAsync(string id, string corpusId, bool? deleteFiles = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + System.Threading.Tasks.Task DeleteParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// @@ -9783,29 +9770,18 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Add a corpus to a engine + /// Add a parallel corpus to an engine /// /// /// ## Parameters - ///
* **name**: A name to help identify and distinguish the corpus from other corpora - ///
* The name does not have to be unique since the corpus is uniquely identified by an auto-generated id - ///
* **sourceLanguage**: The source language code (See documentation on endpoint /word-alignment/engines/ - "Create a new engine" for details on language codes). - ///
* Normally, this is the same as the engine sourceLanguage. This may change for future engines as a means of transfer learning. - ///
* **targetLanguage**: The target language code (See documentation on endpoint /word-alignment/engines/ - "Create a new engine" for details on language codes). - ///
* **SourceFiles**: The source files associated with the corpus - ///
* **FileId**: The unique id referencing the uploaded file - ///
* **TextId**: The client-defined name to associate source and target files. - ///
* If the TextIds in the SourceFiles and TargetFiles match, they will be used to train the engine. - ///
* If a TextId is used more than once in SourceFiles, the sources will be randomly and evenly mixed for training. - ///
* For Paratext projects, TextId will be ignored - multiple Paratext source projects will always be mixed (as if they have the same TextId). - ///
* **TargetFiles**: The target files associated with the corpus - ///
* Same as SourceFiles, except only a single instance of a TextID or a single paratext project is supported. There is no mixing or combining of multiple targets. + ///
* **SourceCorpusIds**: The source corpora associated with the parallel corpus + ///
* **TargetCorpusIds**: The target corpora associated with the parallel corpus /// /// The engine id /// The corpus configuration (see remarks) /// The added corpus /// A server side error occurred. - public virtual async System.Threading.Tasks.Task AddCorpusAsync(string id, WordAlignmentCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task AddParallelCorpusAsync(string id, WordAlignmentParallelCorpusConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -9828,10 +9804,10 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "word-alignment/engines/{id}/corpora" + // Operation Path: "word-alignment/engines/{id}/parallel-corpora" urlBuilder_.Append("word-alignment/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora"); + urlBuilder_.Append("/parallel-corpora"); PrepareRequest(client_, request_, urlBuilder_); @@ -9858,7 +9834,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 201) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -9917,12 +9893,12 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get all corpora for a engine + /// Get all parallel corpora for a engine /// /// The engine id - /// The files + /// The parallel corpora /// A server side error occurred. - public virtual async System.Threading.Tasks.Task> GetAllCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task> GetAllParallelCorporaAsync(string id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); @@ -9938,10 +9914,10 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "word-alignment/engines/{id}/corpora" + // Operation Path: "word-alignment/engines/{id}/parallel-corpora" urlBuilder_.Append("word-alignment/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora"); + urlBuilder_.Append("/parallel-corpora"); PrepareRequest(client_, request_, urlBuilder_); @@ -9968,7 +9944,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync>(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -10021,24 +9997,23 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Update a corpus with a new set of files + /// Update a parallel corpus with a new set of corpora /// /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - ///
Will not affect jobs already queued or running. Will not affect existing word alignments until new build is complete. + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing word graphs until new build is complete. /// /// The engine id - /// The corpus id + /// The parallel corpus id /// The corpus configuration /// The corpus was updated successfully /// A server side error occurred. - public virtual async System.Threading.Tasks.Task UpdateCorpusAsync(string id, string corpusId, WordAlignmentCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task UpdateParallelCorpusAsync(string id, string parallelCorpusId, WordAlignmentParallelCorpusUpdateConfig corpusConfig, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); if (corpusConfig == null) throw new System.ArgumentNullException("corpusConfig"); @@ -10058,11 +10033,11 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "word-alignment/engines/{id}/corpora/{corpusId}" + // Operation Path: "word-alignment/engines/{id}/parallel-corpora/{parallelCorpusId}" urlBuilder_.Append("word-alignment/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -10089,7 +10064,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -10148,19 +10123,19 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Get the configuration of a corpus for a engine + /// Get the configuration of a parallel corpus for a engine /// /// The engine id - /// The corpus id - /// The corpus configuration + /// The parallel corpus id + /// The parallel corpus configuration /// A server side error occurred. - public virtual async System.Threading.Tasks.Task GetCorpusAsync(string id, string corpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task GetParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); var client_ = _httpClient; var disposeClient_ = false; @@ -10173,11 +10148,11 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "word-alignment/engines/{id}/corpora/{corpusId}" + // Operation Path: "word-alignment/engines/{id}/parallel-corpora/{parallelCorpusId}" urlBuilder_.Append("word-alignment/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -10204,7 +10179,7 @@ public string BaseUrl var status_ = (int)response_.StatusCode; if (status_ == 200) { - var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); + var objectResponse_ = await ReadObjectResponseAsync(response_, headers_, cancellationToken).ConfigureAwait(false); if (objectResponse_.Object == null) { throw new ServalApiException("Response was null which was not expected.", status_, objectResponse_.Text, headers_, null); @@ -10227,7 +10202,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -10257,23 +10232,22 @@ public string BaseUrl /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. /// - /// Remove a corpus from a engine + /// Remove a parallel corpus from a engine /// /// - /// Removing a corpus will remove all word alignments associated with that corpus. + /// Removing a parallel corpus will remove all word alignments associated with that corpus. /// /// The engine id - /// The corpus id - /// If true, all files associated with the corpus will be deleted as well (even if they are associated with other corpora). If false, no files will be deleted. - /// The corpus was deleted successfully. + /// The parallel corpus id + /// The parallel corpus was deleted successfully. /// A server side error occurred. - public virtual async System.Threading.Tasks.Task DeleteCorpusAsync(string id, string corpusId, bool? deleteFiles = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) + public virtual async System.Threading.Tasks.Task DeleteParallelCorpusAsync(string id, string parallelCorpusId, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { if (id == null) throw new System.ArgumentNullException("id"); - if (corpusId == null) - throw new System.ArgumentNullException("corpusId"); + if (parallelCorpusId == null) + throw new System.ArgumentNullException("parallelCorpusId"); var client_ = _httpClient; var disposeClient_ = false; @@ -10285,17 +10259,11 @@ public string BaseUrl var urlBuilder_ = new System.Text.StringBuilder(); if (!string.IsNullOrEmpty(_baseUrl)) urlBuilder_.Append(_baseUrl); - // Operation Path: "word-alignment/engines/{id}/corpora/{corpusId}" + // Operation Path: "word-alignment/engines/{id}/parallel-corpora/{parallelCorpusId}" urlBuilder_.Append("word-alignment/engines/"); urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(id, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append("/corpora/"); - urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(corpusId, System.Globalization.CultureInfo.InvariantCulture))); - urlBuilder_.Append('?'); - if (deleteFiles != null) - { - urlBuilder_.Append(System.Uri.EscapeDataString("delete-files")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(deleteFiles, System.Globalization.CultureInfo.InvariantCulture))).Append('&'); - } - urlBuilder_.Length--; + urlBuilder_.Append("/parallel-corpora/"); + urlBuilder_.Append(System.Uri.EscapeDataString(ConvertToString(parallelCorpusId, System.Globalization.CultureInfo.InvariantCulture))); PrepareRequest(client_, request_, urlBuilder_); @@ -10340,7 +10308,7 @@ public string BaseUrl if (status_ == 404) { string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false); - throw new ServalApiException("The engine or corpus does not exist.", status_, responseText_, headers_, null); + throw new ServalApiException("The engine or parallel corpus does not exist.", status_, responseText_, headers_, null); } else if (status_ == 503) @@ -12745,7 +12713,7 @@ public partial class WordAlignmentRequest } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordAlignmentCorpus + public partial class WordAlignmentParallelCorpus { [Newtonsoft.Json.JsonProperty("id", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -12759,41 +12727,18 @@ public partial class WordAlignmentCorpus [System.ComponentModel.DataAnnotations.Required] public ResourceLink Engine { get; set; } = new ResourceLink(); - [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? Name { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string SourceLanguage { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetLanguage { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceFiles { get; set; } = new System.Collections.ObjectModel.Collection(); - - [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("sourceCorpora", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList TargetFiles { get; set; } = new System.Collections.ObjectModel.Collection(); - - } + public System.Collections.Generic.IList SourceCorpora { get; set; } = new System.Collections.ObjectModel.Collection(); - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordAlignmentCorpusFile - { - [Newtonsoft.Json.JsonProperty("file", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetCorpora", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public ResourceLink File { get; set; } = new ResourceLink(); - - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; + public System.Collections.Generic.IList TargetCorpora { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordAlignmentCorpusConfig + public partial class WordAlignmentParallelCorpusConfig { /// /// The corpus name. @@ -12801,44 +12746,24 @@ public partial class WordAlignmentCorpusConfig [Newtonsoft.Json.JsonProperty("name", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? Name { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("sourceLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string SourceLanguage { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("targetLanguage", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string TargetLanguage { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("sourceCorpusIds", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList SourceFiles { get; set; } = new System.Collections.ObjectModel.Collection(); + public System.Collections.Generic.IList SourceCorpusIds { get; set; } = new System.Collections.ObjectModel.Collection(); - [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Always)] + [Newtonsoft.Json.JsonProperty("targetCorpusIds", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] - public System.Collections.Generic.IList TargetFiles { get; set; } = new System.Collections.ObjectModel.Collection(); - - } - - [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordAlignmentCorpusFileConfig - { - [Newtonsoft.Json.JsonProperty("fileId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string FileId { get; set; } = default!; - - [Newtonsoft.Json.JsonProperty("textId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? TextId { get; set; } = default!; + public System.Collections.Generic.IList TargetCorpusIds { get; set; } = new System.Collections.ObjectModel.Collection(); } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] - public partial class WordAlignmentCorpusUpdateConfig + public partial class WordAlignmentParallelCorpusUpdateConfig { - [Newtonsoft.Json.JsonProperty("sourceFiles", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? SourceFiles { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceCorpusIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceCorpusIds { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("targetFiles", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TargetFiles { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetCorpusIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetCorpusIds { get; set; } = default!; } @@ -12928,6 +12853,20 @@ public partial class WordAlignmentBuild [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TrainingCorpus2 + { + [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public ResourceLink? ParallelCorpus { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class ParallelCorpusFilter2 { [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] @@ -12944,15 +12883,14 @@ public partial class TrainingCorpus2 [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class WordAlignOnCorpus { - [Newtonsoft.Json.JsonProperty("corpus", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required] - public ResourceLink Corpus { get; set; } = new ResourceLink(); + [Newtonsoft.Json.JsonProperty("parallelCorpus", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public ResourceLink? ParallelCorpus { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } @@ -12975,6 +12913,20 @@ public partial class WordAlignmentBuildConfig [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TrainingCorpusConfig2 + { + [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ParallelCorpusId { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class ParallelCorpusFilterConfig2 { [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] @@ -12991,15 +12943,14 @@ public partial class TrainingCorpusConfig2 [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class WordAlignOnCorpusConfig { - [Newtonsoft.Json.JsonProperty("corpusId", Required = Newtonsoft.Json.Required.Always)] - [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] - public string CorpusId { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("parallelCorpusId", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? ParallelCorpusId { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("textIds", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public System.Collections.Generic.IList? TextIds { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("sourceFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? SourceFilters { get; set; } = default!; - [Newtonsoft.Json.JsonProperty("scriptureRange", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] - public string? ScriptureRange { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("targetFilters", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? TargetFilters { get; set; } = default!; } diff --git a/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs b/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs index 6ae1bf28..c58e4d44 100644 --- a/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs +++ b/src/Serval/src/Serval.Shared/Controllers/Endpoints.cs @@ -14,7 +14,7 @@ public static class Endpoints public const string GetAssessmentReferenceCorpus = "GetAssessmentReferenceCorpus"; public const string GetAssessmentJob = "GetAssessmentJob"; public const string GetWordAlignmentEngine = "GetWordAlignmentEngine"; - public const string GetWordAlignmentCorpus = "GetWordAlignmentCorpus"; + public const string GetParallelWordAlignmentCorpus = "GetParallelWordAlignmentCorpus"; public const string GetWordAlignmentBuild = "GetWordAlignmentBuild"; public const string GetWebhook = "GetWebhook"; diff --git a/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs similarity index 86% rename from src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs rename to src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs index 0762e878..f9c58fb4 100644 --- a/src/Serval/src/Serval.Translation/Models/MonolingualCorpus.cs +++ b/src/Serval/src/Serval.Shared/Models/MonolingualCorpus.cs @@ -1,4 +1,4 @@ -namespace Serval.Translation.Models; +namespace Serval.Shared.Models; public record MonolingualCorpus { diff --git a/src/Serval/src/Serval.Shared/Models/TrainingCorpus.cs b/src/Serval/src/Serval.Translation/Models/TrainingCorpus.cs similarity index 91% rename from src/Serval/src/Serval.Shared/Models/TrainingCorpus.cs rename to src/Serval/src/Serval.Translation/Models/TrainingCorpus.cs index 54db7d14..fc927406 100644 --- a/src/Serval/src/Serval.Shared/Models/TrainingCorpus.cs +++ b/src/Serval/src/Serval.Translation/Models/TrainingCorpus.cs @@ -1,4 +1,4 @@ -namespace Serval.Shared.Models; +namespace Serval.Translation.Models; public record TrainingCorpus { diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index 06f6dfcc..c13101f1 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -432,8 +432,8 @@ public Task AddParallelCorpusAsync( public async Task UpdateParallelCorpusAsync( string engineId, string parallelCorpusId, - IReadOnlyList? sourceCorpora, - IReadOnlyList? targetCorpora, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ) { @@ -716,7 +716,7 @@ private V1.ParallelCorpus Map( } private V1.MonolingualCorpus Map( - Models.MonolingualCorpus source, + Shared.Models.MonolingualCorpus source, ParallelCorpusFilter? trainingFilter, ParallelCorpusFilter? pretranslateFilter, string? referenceFileLocation diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs new file mode 100644 index 00000000..c1286abf --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterConfigDto.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record ParallelCorpusFilterConfigDto +{ + public required string CorpusId { get; init; } + public IReadOnlyList? TextIds { get; init; } + public string? ScriptureRange { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs new file mode 100644 index 00000000..0824fe95 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/ParallelCorpusFilterDto.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Contracts; + +public record ParallelCorpusFilterDto +{ + public required ResourceLinkDto Corpus { get; init; } + public IReadOnlyList? TextIds { get; init; } + public string? ScriptureRange { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs index 2634676d..bfe1c3e5 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusConfigDto.cs @@ -2,7 +2,7 @@ namespace Serval.WordAlignment.Contracts; public record TrainingCorpusConfigDto { - public required string CorpusId { get; init; } - public IReadOnlyList? TextIds { get; init; } - public string? ScriptureRange { get; init; } + public string? ParallelCorpusId { get; init; } + public IReadOnlyList? SourceFilters { get; init; } + public IReadOnlyList? TargetFilters { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs index 48f20e0c..de110ceb 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Contracts/TrainingCorpusDto.cs @@ -2,9 +2,7 @@ namespace Serval.WordAlignment.Contracts; public record TrainingCorpusDto { - public required ResourceLinkDto Corpus { get; init; } - - public IReadOnlyList? TextIds { get; init; } - - public string? ScriptureRange { get; init; } + public ResourceLinkDto? ParallelCorpus { get; init; } + public IReadOnlyList? SourceFilters { get; init; } + public IReadOnlyList? TargetFilters { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusConfigDto.cs index a3712f74..21511f7f 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusConfigDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusConfigDto.cs @@ -2,7 +2,7 @@ namespace Serval.WordAlignment.Contracts; public record WordAlignOnCorpusConfigDto { - public required string CorpusId { get; init; } - public IReadOnlyList? TextIds { get; init; } - public string? ScriptureRange { get; init; } + public string? ParallelCorpusId { get; init; } + public IReadOnlyList? SourceFilters { get; init; } + public IReadOnlyList? TargetFilters { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusDto.cs index 9fdb23cd..32cd354b 100644 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignOnCorpusDto.cs @@ -2,9 +2,7 @@ namespace Serval.WordAlignment.Contracts; public record WordAlignOnCorpusDto { - public required ResourceLinkDto Corpus { get; init; } - - public IReadOnlyList? TextIds { get; init; } - - public string? ScriptureRange { get; init; } + public ResourceLinkDto? ParallelCorpus { get; init; } + public IReadOnlyList? SourceFilters { get; init; } + public IReadOnlyList? TargetFilters { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs deleted file mode 100644 index a44c2a3e..00000000 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusConfigDto.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace Serval.WordAlignment.Contracts; - -public record WordAlignmentCorpusConfigDto -{ - /// - /// The corpus name. - /// - public string? Name { get; init; } - - public required string SourceLanguage { get; init; } - - public required string TargetLanguage { get; init; } - - public required IReadOnlyList SourceFiles { get; init; } - - public required IReadOnlyList TargetFiles { get; init; } -} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs deleted file mode 100644 index 6397f665..00000000 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusDto.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Serval.WordAlignment.Contracts; - -public record WordAlignmentCorpusDto -{ - public required string Id { get; init; } - public required string Url { get; init; } - public required ResourceLinkDto Engine { get; init; } - public string? Name { get; init; } - public required string SourceLanguage { get; init; } - public required string TargetLanguage { get; init; } - public required IReadOnlyList SourceFiles { get; init; } - public required IReadOnlyList TargetFiles { get; init; } -} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileConfigDto.cs deleted file mode 100644 index 9cf5e9ea..00000000 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileConfigDto.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.WordAlignment.Contracts; - -public record WordAlignmentCorpusFileConfigDto -{ - public required string FileId { get; init; } - - public string? TextId { get; init; } -} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileDto.cs deleted file mode 100644 index 416fc363..00000000 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusFileDto.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.WordAlignment.Contracts; - -public record WordAlignmentCorpusFileDto -{ - public required ResourceLinkDto File { get; init; } - public string? TextId { get; init; } -} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusUpdateConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusUpdateConfigDto.cs deleted file mode 100644 index a8f6a608..00000000 --- a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentCorpusUpdateConfigDto.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace Serval.WordAlignment.Contracts; - -public record WordAlignmentCorpusUpdateConfigDto : IValidatableObject -{ - public IReadOnlyList? SourceFiles { get; init; } - - public IReadOnlyList? TargetFiles { get; init; } - - public IEnumerable Validate( - ValidationContext validationContext - ) - { - if (SourceFiles is null && TargetFiles is null) - { - yield return new System.ComponentModel.DataAnnotations.ValidationResult( - "At least one field must be specified.", - [nameof(SourceFiles), nameof(TargetFiles)] - ); - } - } -} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs new file mode 100644 index 00000000..6aa341c4 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusConfigDto.cs @@ -0,0 +1,12 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentParallelCorpusConfigDto +{ + /// + /// The corpus name. + /// + public string? Name { get; init; } + + public required IReadOnlyList SourceCorpusIds { get; init; } = new List(); + public required IReadOnlyList TargetCorpusIds { get; init; } = new List(); +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs new file mode 100644 index 00000000..53f00302 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusDto.cs @@ -0,0 +1,10 @@ +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentParallelCorpusDto +{ + public required string Id { get; init; } + public required string Url { get; init; } + public required ResourceLinkDto Engine { get; init; } + public required IReadOnlyList SourceCorpora { get; init; } + public required IReadOnlyList TargetCorpora { get; init; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs new file mode 100644 index 00000000..5a966947 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Contracts/WordAlignmentParallelCorpusUpdateDto.cs @@ -0,0 +1,23 @@ +using System.ComponentModel.DataAnnotations; + +namespace Serval.WordAlignment.Contracts; + +public record WordAlignmentParallelCorpusUpdateConfigDto : IValidatableObject +{ + public IReadOnlyList? SourceCorpusIds { get; init; } + + public IReadOnlyList? TargetCorpusIds { get; init; } + + public IEnumerable Validate( + ValidationContext validationContext + ) + { + if (SourceCorpusIds is null && TargetCorpusIds is null) + { + yield return new System.ComponentModel.DataAnnotations.ValidationResult( + "At least one field must be specified.", + [nameof(SourceCorpusIds), nameof(TargetCorpusIds)] + ); + } + } +} diff --git a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs index 716ad111..dce606fe 100644 --- a/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs +++ b/src/Serval/src/Serval.WordAlignment/Controllers/WordAlignmentEnginesController.cs @@ -188,27 +188,16 @@ CancellationToken cancellationToken } /// - /// Add a corpus to a engine + /// Add a parallel corpus to an engine /// /// /// ## Parameters - /// * **name**: A name to help identify and distinguish the corpus from other corpora - /// * The name does not have to be unique since the corpus is uniquely identified by an auto-generated id - /// * **sourceLanguage**: The source language code (See documentation on endpoint /word-alignment/engines/ - "Create a new engine" for details on language codes). - /// * Normally, this is the same as the engine sourceLanguage. This may change for future engines as a means of transfer learning. - /// * **targetLanguage**: The target language code (See documentation on endpoint /word-alignment/engines/ - "Create a new engine" for details on language codes). - /// * **SourceFiles**: The source files associated with the corpus - /// * **FileId**: The unique id referencing the uploaded file - /// * **TextId**: The client-defined name to associate source and target files. - /// * If the TextIds in the SourceFiles and TargetFiles match, they will be used to train the engine. - /// * If a TextId is used more than once in SourceFiles, the sources will be randomly and evenly mixed for training. - /// * For Paratext projects, TextId will be ignored - multiple Paratext source projects will always be mixed (as if they have the same TextId). - /// * **TargetFiles**: The target files associated with the corpus - /// * Same as SourceFiles, except only a single instance of a TextID or a single paratext project is supported. There is no mixing or combining of multiple targets. + /// * **SourceCorpusIds**: The source corpora associated with the parallel corpus + /// * **TargetCorpusIds**: The target corpora associated with the parallel corpus /// /// The engine id /// The corpus configuration (see remarks) - /// + /// /// /// /// The added corpus @@ -218,40 +207,44 @@ CancellationToken cancellationToken /// The engine does not exist. /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateWordAlignmentEngines)] - [HttpPost("{id}/corpora")] + [HttpPost("{id}/parallel-corpora")] [ProducesResponseType(StatusCodes.Status201Created)] [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> AddCorpusAsync( + public async Task> AddParallelCorpusAsync( [NotNull] string id, - [FromBody] WordAlignmentCorpusConfigDto corpusConfig, - [FromServices] IRequestClient getDataFileClient, + [FromBody] WordAlignmentParallelCorpusConfigDto corpusConfig, + [FromServices] IRequestClient getCorpusClient, [FromServices] IIdGenerator idGenerator, CancellationToken cancellationToken ) { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - Corpus corpus = await MapAsync(getDataFileClient, idGenerator.GenerateId(), corpusConfig, cancellationToken); - await _engineService.AddCorpusAsync(id, corpus, cancellationToken); - WordAlignmentCorpusDto dto = Map(id, corpus); + ParallelCorpus corpus = await MapAsync( + getCorpusClient, + idGenerator.GenerateId(), + corpusConfig, + cancellationToken + ); + await _engineService.AddParallelCorpusAsync(id, corpus, cancellationToken); + WordAlignmentParallelCorpusDto dto = Map(id, corpus); return Created(dto.Url, dto); } /// - /// Update a corpus with a new set of files + /// Update a parallel corpus with a new set of corpora /// /// - /// See posting a new corpus for details of use. Will completely replace corpus' file associations. - /// Will not affect jobs already queued or running. Will not affect existing word alignments until new build is complete. + /// Will completely replace the parallel corpus' file associations. Will not affect jobs already queued or running. Will not affect existing word graphs until new build is complete. /// /// The engine id - /// The corpus id + /// The parallel corpus id /// The corpus configuration - /// The data file client + /// The data file client /// /// The corpus was updated successfully /// Bad request @@ -260,126 +253,124 @@ CancellationToken cancellationToken /// The engine or corpus does not exist. /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateWordAlignmentEngines)] - [HttpPatch("{id}/corpora/{corpusId}")] + [HttpPatch("{id}/parallel-corpora/{parallelCorpusId}")] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> UpdateCorpusAsync( + public async Task> UpdateParallelCorpusAsync( [NotNull] string id, - [NotNull] string corpusId, - [FromBody] WordAlignmentCorpusUpdateConfigDto corpusConfig, - [FromServices] IRequestClient getDataFileClient, + [NotNull] string parallelCorpusId, + [FromBody] WordAlignmentParallelCorpusUpdateConfigDto corpusConfig, + [FromServices] IRequestClient getCorpusClient, CancellationToken cancellationToken ) { await AuthorizeAsync(id, cancellationToken); - Corpus corpus = await _engineService.UpdateCorpusAsync( + ParallelCorpus parallelCorpus = await _engineService.UpdateParallelCorpusAsync( id, - corpusId, - corpusConfig.SourceFiles is null + parallelCorpusId, + corpusConfig.SourceCorpusIds is null ? null - : await MapAsync(getDataFileClient, corpusConfig.SourceFiles, cancellationToken), - corpusConfig.TargetFiles is null + : await MapAsync(getCorpusClient, corpusConfig.SourceCorpusIds, cancellationToken), + corpusConfig.TargetCorpusIds is null ? null - : await MapAsync(getDataFileClient, corpusConfig.TargetFiles, cancellationToken), + : await MapAsync(getCorpusClient, corpusConfig.TargetCorpusIds, cancellationToken), cancellationToken ); - return Ok(Map(id, corpus)); + return Ok(Map(id, parallelCorpus)); } /// - /// Get all corpora for a engine + /// Get all parallel corpora for a engine /// /// The engine id /// - /// The files + /// The parallel corpora /// The client is not authenticated /// The authenticated client cannot perform the operation or does not own the engine /// The engine does not exist /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.ReadWordAlignmentEngines)] - [HttpGet("{id}/corpora")] + [HttpGet("{id}/parallel-corpora")] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task>> GetAllCorporaAsync( + public async Task>> GetAllParallelCorporaAsync( [NotNull] string id, CancellationToken cancellationToken ) { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - return Ok(engine.Corpora.Select(c => Map(id, c))); + return Ok(engine.ParallelCorpora.Select(c => Map(id, c))); } /// - /// Get the configuration of a corpus for a engine + /// Get the configuration of a parallel corpus for a engine /// /// The engine id - /// The corpus id + /// The parallel corpus id /// - /// The corpus configuration + /// The parallel corpus configuration /// The client is not authenticated. /// The authenticated client cannot perform the operation or does not own the engine. - /// The engine or corpus does not exist. + /// The engine or parallel corpus does not exist. /// A necessary service is currently unavailable. Check `/health` for more details. - [Authorize(Scopes.ReadWordAlignmentEngines)] - [HttpGet("{id}/corpora/{corpusId}", Name = Endpoints.GetWordAlignmentCorpus)] + [Authorize(Scopes.ReadTranslationEngines)] + [HttpGet("{id}/parallel-corpora/{parallelCorpusId}", Name = Endpoints.GetParallelWordAlignmentCorpus)] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task> GetCorpusAsync( + public async Task> GetParallelCorpusAsync( [NotNull] string id, - [NotNull] string corpusId, + [NotNull] string parallelCorpusId, CancellationToken cancellationToken ) { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - Corpus? corpus = engine.Corpora.FirstOrDefault(f => f.Id == corpusId); + ParallelCorpus? corpus = engine.ParallelCorpora.FirstOrDefault(f => f.Id == parallelCorpusId); if (corpus == null) return NotFound(); return Ok(Map(id, corpus)); } /// - /// Remove a corpus from a engine + /// Remove a parallel corpus from a engine /// /// - /// Removing a corpus will remove all word alignments associated with that corpus. + /// Removing a parallel corpus will remove all word alignments associated with that corpus. /// /// The engine id - /// The corpus id - /// If true, all files associated with the corpus will be deleted as well (even if they are associated with other corpora). If false, no files will be deleted. + /// The parallel corpus id /// - /// The corpus was deleted successfully. + /// The parallel corpus was deleted successfully. /// The client is not authenticated. /// The authenticated client cannot perform the operation or does not own the engine. - /// The engine or corpus does not exist. + /// The engine or parallel corpus does not exist. /// A necessary service is currently unavailable. Check `/health` for more details. [Authorize(Scopes.UpdateWordAlignmentEngines)] - [HttpDelete("{id}/corpora/{corpusId}")] + [HttpDelete("{id}/parallel-corpora/{parallelCorpusId}")] [ProducesResponseType(typeof(void), StatusCodes.Status200OK)] [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)] [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)] [ProducesResponseType(typeof(void), StatusCodes.Status404NotFound)] [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)] - public async Task DeleteCorpusAsync( + public async Task DeleteParallelCorpusAsync( [NotNull] string id, - [NotNull] string corpusId, - [FromQuery(Name = "delete-files")] bool? deleteFiles, + [NotNull] string parallelCorpusId, CancellationToken cancellationToken ) { await AuthorizeAsync(id, cancellationToken); - await _engineService.DeleteCorpusAsync(id, corpusId, deleteFiles ?? false, cancellationToken); + await _engineService.DeleteParallelCorpusAsync(id, parallelCorpusId, cancellationToken); return Ok(); } @@ -427,7 +418,7 @@ CancellationToken cancellationToken { Engine engine = await _engineService.GetAsync(id, cancellationToken); await AuthorizeAsync(engine); - if (!engine.Corpora.Any(c => c.Id == corpusId)) + if (!engine.ParallelCorpora.Any(c => c.Id == corpusId)) return NotFound(); if (engine.ModelRevision == 0) return Conflict(); @@ -681,67 +672,87 @@ private async Task AuthorizeAsync(string id, CancellationToken cancellationToken await AuthorizeAsync(engine); } - private async Task MapAsync( - IRequestClient getDataFileClient, + private async Task MapAsync( + IRequestClient getDataFileClient, string corpusId, - WordAlignmentCorpusConfigDto source, + WordAlignmentParallelCorpusConfigDto source, CancellationToken cancellationToken ) { - return new Corpus + return new ParallelCorpus { Id = corpusId, - Name = source.Name, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - SourceFiles = await MapAsync(getDataFileClient, source.SourceFiles, cancellationToken), - TargetFiles = await MapAsync(getDataFileClient, source.TargetFiles, cancellationToken) + SourceCorpora = await MapAsync(getDataFileClient, source.SourceCorpusIds, cancellationToken), + TargetCorpora = await MapAsync(getDataFileClient, source.TargetCorpusIds, cancellationToken) }; } - private async Task> MapAsync( - IRequestClient getDataFileClient, - IEnumerable fileConfigs, + private async Task> MapAsync( + IRequestClient getCorpusClient, + IEnumerable corpusIds, CancellationToken cancellationToken ) { - var files = new List(); - foreach (WordAlignmentCorpusFileConfigDto fileConfig in fileConfigs) + var corpora = new List(); + foreach (string corpusId in corpusIds) { - Response response = await getDataFileClient.GetResponse< - DataFileResult, - DataFileNotFound - >(new GetDataFile { DataFileId = fileConfig.FileId, Owner = Owner }, cancellationToken); - if (response.Is(out Response? result)) + Response response = await getCorpusClient.GetResponse< + CorpusResult, + CorpusNotFound + >(new GetCorpus { CorpusId = corpusId, Owner = Owner }, cancellationToken); + if (response.Is(out Response? result)) { - files.Add( - new CorpusFile + corpora.Add( + new MonolingualCorpus { - Id = fileConfig.FileId, - Filename = result.Message.Filename, - TextId = fileConfig.TextId ?? result.Message.Name, - Format = result.Message.Format + Id = corpusId, + Name = result.Message.Name ?? "", + Language = result.Message.Language, + Files = result + .Message.Files.Select(f => new CorpusFile + { + Id = f.File.DataFileId, + Filename = f.File.Filename, + Format = f.File.Format, + TextId = f.TextId + }) + .ToList(), } ); } - else if (response.Is(out Response? _)) + else if (response.Is(out Response? _)) { - throw new InvalidOperationException($"The data file {fileConfig.FileId} cannot be found."); + throw new InvalidOperationException($"The corpus {corpusId} cannot be found."); } } - return files; + return corpora; } - private Engine Map(WordAlignmentEngineConfigDto source) + private WordAlignmentParallelCorpusDto Map(string engineId, ParallelCorpus source) { - return new Engine + return new WordAlignmentParallelCorpusDto { - Name = source.Name, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - Type = source.Type.ToPascalCase(), - Owner = Owner, - Corpora = [], + Id = source.Id, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { id = engineId, corpusId = source.Id }), + Engine = new ResourceLinkDto + { + Id = engineId, + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = engineId }) + }, + SourceCorpora = source + .SourceCorpora.Select(c => new ResourceLinkDto + { + Id = c.Id, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { Id = c.Id }) + }) + .ToList(), + TargetCorpora = source + .TargetCorpora.Select(c => new ResourceLinkDto + { + Id = c.Id, + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { Id = c.Id }) + }) + .ToList() }; } @@ -762,28 +773,26 @@ private static Build Map(Engine engine, WordAlignmentBuildConfigDto source) if (source is null) return null; - var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); + var corpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); var wordAlignmentCorpora = new List(); - foreach (WordAlignOnCorpusConfigDto ptcc in source) + foreach (WordAlignOnCorpusConfigDto cc in source) { - if (!corpusIds.Contains(ptcc.CorpusId)) + if (cc.ParallelCorpusId == null) { - throw new InvalidOperationException( - $"The corpus {ptcc.CorpusId} is not valid: This corpus does not exist for engine {engine.Id}." - ); + throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); } - if (ptcc.TextIds != null && ptcc.ScriptureRange != null) + if (!corpusIds.Contains(cc.ParallelCorpusId)) { throw new InvalidOperationException( - $"The corpus {ptcc.CorpusId} is not valid: Set at most one of TextIds and ScriptureRange." + $"The parallel corpus {cc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." ); } wordAlignmentCorpora.Add( new WordAlignmentCorpus { - CorpusRef = ptcc.CorpusId, - TextIds = ptcc.TextIds?.ToList(), - ScriptureRange = ptcc.ScriptureRange + ParallelCorpusRef = cc.ParallelCorpusId, + SourceFilters = cc.SourceFilters?.Select(Map).ToList(), + TargetFilters = cc.TargetFilters?.Select(Map).ToList() } ); } @@ -795,32 +804,46 @@ private static Build Map(Engine engine, WordAlignmentBuildConfigDto source) if (source is null) return null; - var corpusIds = new HashSet(engine.Corpora.Select(c => c.Id)); - var trainOnCorpora = new List(); - foreach (TrainingCorpusConfigDto tcc in source) + var corpusIds = new HashSet(engine.ParallelCorpora.Select(c => c.Id)); + var trainingCorpora = new List(); + foreach (TrainingCorpusConfigDto cc in source) { - if (!corpusIds.Contains(tcc.CorpusId)) + if (cc.ParallelCorpusId == null) { - throw new InvalidOperationException( - $"The corpus {tcc.CorpusId} is not valid: This corpus does not exist for engine {engine.Id}." - ); + throw new InvalidOperationException($"One of ParallelCorpusId and CorpusId must be set."); } - if (tcc.TextIds != null && tcc.ScriptureRange != null) + if (!corpusIds.Contains(cc.ParallelCorpusId)) { throw new InvalidOperationException( - $"The corpus {tcc.CorpusId} is not valid: Set at most one of TextIds and ScriptureRange." + $"The parallel corpus {cc.ParallelCorpusId} is not valid: This parallel corpus does not exist for engine {engine.Id}." ); } - trainOnCorpora.Add( + trainingCorpora.Add( new TrainingCorpus { - CorpusRef = tcc.CorpusId, - TextIds = tcc.TextIds?.ToList(), - ScriptureRange = tcc.ScriptureRange + ParallelCorpusRef = cc.ParallelCorpusId, + SourceFilters = cc.SourceFilters?.Select(Map).ToList(), + TargetFilters = cc.TargetFilters?.Select(Map).ToList() } ); } - return trainOnCorpora; + return trainingCorpora; + } + + private static ParallelCorpusFilter Map(ParallelCorpusFilterConfigDto source) + { + if (source.TextIds != null && source.ScriptureRange != null) + { + throw new InvalidOperationException( + $"The parallel corpus filter for corpus {source.CorpusId} is not valid: At most, one of TextIds and ScriptureRange can be set." + ); + } + return new ParallelCorpusFilter + { + CorpusRef = source.CorpusId, + TextIds = source.TextIds, + ScriptureRange = source.ScriptureRange + }; } private static Dictionary? Map(object? source) @@ -860,16 +883,13 @@ private WordAlignmentBuildDto Map(Build source) return new WordAlignmentBuildDto { Id = source.Id, - Url = _urlService.GetUrl( - Endpoints.GetWordAlignmentBuild, - new { id = source.EngineRef, buildId = source.Id } - ), + Url = _urlService.GetUrl(Endpoints.GetTranslationBuild, new { id = source.EngineRef, buildId = source.Id }), Revision = source.Revision, Name = source.Name, Engine = new ResourceLinkDto { Id = source.EngineRef, - Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = source.EngineRef }) + Url = _urlService.GetUrl(Endpoints.GetTranslationEngine, new { id = source.EngineRef }) }, TrainOn = source.TrainOn?.Select(s => Map(source.EngineRef, s)).ToList(), WordAlignOn = source.WordAlignOn?.Select(s => Map(source.EngineRef, s)).ToList(), @@ -887,30 +907,50 @@ private WordAlignOnCorpusDto Map(string engineId, WordAlignmentCorpus source) { return new WordAlignOnCorpusDto { - Corpus = new ResourceLinkDto - { - Id = source.CorpusRef, - Url = _urlService.GetUrl( - Endpoints.GetWordAlignmentCorpus, - new { id = engineId, corpusId = source.CorpusRef } - ) - }, - TextIds = source.TextIds, - ScriptureRange = source.ScriptureRange + ParallelCorpus = + source.ParallelCorpusRef != null + ? new ResourceLinkDto + { + Id = source.ParallelCorpusRef, + Url = _urlService.GetUrl( + Endpoints.GetParallelTranslationCorpus, + new { id = engineId, parallelCorpusId = source.ParallelCorpusRef } + ) + } + : null, + SourceFilters = source.SourceFilters?.Select(Map).ToList(), + TargetFilters = source.TargetFilters?.Select(Map).ToList() }; } private TrainingCorpusDto Map(string engineId, TrainingCorpus source) { return new TrainingCorpusDto + { + ParallelCorpus = + source.ParallelCorpusRef != null + ? new ResourceLinkDto + { + Id = source.ParallelCorpusRef, + Url = _urlService.GetUrl( + Endpoints.GetParallelTranslationCorpus, + new { id = engineId, parallelCorpusId = source.ParallelCorpusRef } + ) + } + : null, + SourceFilters = source.SourceFilters?.Select(Map).ToList(), + TargetFilters = source.TargetFilters?.Select(Map).ToList() + }; + } + + private ParallelCorpusFilterDto Map(ParallelCorpusFilter source) + { + return new ParallelCorpusFilterDto { Corpus = new ResourceLinkDto { Id = source.CorpusRef, - Url = _urlService.GetUrl( - Endpoints.GetWordAlignmentCorpus, - new { id = engineId, corpusId = source.CorpusRef } - ) + Url = _urlService.GetUrl(Endpoints.GetCorpus, new { id = source.CorpusRef }) }, TextIds = source.TextIds, ScriptureRange = source.ScriptureRange @@ -952,35 +992,16 @@ private static WordAlignmentDto Map(Models.WordAlignment source) }; } - private WordAlignmentCorpusDto Map(string engineId, Corpus source) + private Engine Map(WordAlignmentEngineConfigDto source) { - return new WordAlignmentCorpusDto + return new Engine { - Id = source.Id, - Url = _urlService.GetUrl(Endpoints.GetWordAlignmentCorpus, new { id = engineId, corpusId = source.Id }), - Engine = new ResourceLinkDto - { - Id = engineId, - Url = _urlService.GetUrl(Endpoints.GetWordAlignmentEngine, new { id = engineId }) - }, Name = source.Name, SourceLanguage = source.SourceLanguage, TargetLanguage = source.TargetLanguage, - SourceFiles = source.SourceFiles.Select(Map).ToList(), - TargetFiles = source.TargetFiles.Select(Map).ToList() - }; - } - - private WordAlignmentCorpusFileDto Map(CorpusFile source) - { - return new WordAlignmentCorpusFileDto - { - File = new ResourceLinkDto - { - Id = source.Id, - Url = _urlService.GetUrl(Endpoints.GetDataFile, new { id = source.Id }) - }, - TextId = source.TextId + Type = source.Type.ToPascalCase(), + Owner = Owner, + ParallelCorpora = [], }; } } diff --git a/src/Serval/src/Serval.WordAlignment/Models/Build.cs b/src/Serval/src/Serval.WordAlignment/Models/Build.cs index 974633a3..54731b7f 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/Build.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/Build.cs @@ -6,7 +6,7 @@ public record Build : IEntity public int Revision { get; set; } = 1; public string? Name { get; init; } public required string EngineRef { get; init; } - public IReadOnlyList? TrainOn { get; init; } + public IReadOnlyList? TrainOn { get; init; } public IReadOnlyList? WordAlignOn { get; init; } public int Step { get; init; } public double? PercentCompleted { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Models/Corpus.cs b/src/Serval/src/Serval.WordAlignment/Models/Corpus.cs deleted file mode 100644 index fccd722f..00000000 --- a/src/Serval/src/Serval.WordAlignment/Models/Corpus.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.WordAlignment.Models; - -public record Corpus -{ - public string Id { get; set; } = ""; - public string? Name { get; set; } - public required string SourceLanguage { get; set; } - public required string TargetLanguage { get; set; } - public required IReadOnlyList SourceFiles { get; set; } - public required IReadOnlyList TargetFiles { get; set; } -} diff --git a/src/Serval/src/Serval.WordAlignment/Models/Engine.cs b/src/Serval/src/Serval.WordAlignment/Models/Engine.cs index 9860a3e1..fb1774ae 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/Engine.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/Engine.cs @@ -9,7 +9,7 @@ public record Engine : IOwnedEntity public required string TargetLanguage { get; init; } public required string Type { get; init; } public required string Owner { get; init; } - public required IReadOnlyList Corpora { get; init; } + public required IReadOnlyList ParallelCorpora { get; init; } public bool? IsModelPersisted { get; init; } public bool IsBuilding { get; init; } public int ModelRevision { get; init; } diff --git a/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs new file mode 100644 index 00000000..a870eba9 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/ParallelCorpus.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Models; + +public record ParallelCorpus +{ + public required string Id { get; set; } + public IReadOnlyList SourceCorpora { get; set; } = new List(); + public IReadOnlyList TargetCorpora { get; set; } = new List(); +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs new file mode 100644 index 00000000..fd9c06a6 --- /dev/null +++ b/src/Serval/src/Serval.WordAlignment/Models/TrainingCorpus.cs @@ -0,0 +1,8 @@ +namespace Serval.WordAlignment.Models; + +public record TrainingCorpus +{ + public string? ParallelCorpusRef { get; set; } + public IReadOnlyList? SourceFilters { get; set; } + public IReadOnlyList? TargetFilters { get; set; } +} diff --git a/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentCorpus.cs b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentCorpus.cs index 3e726573..4c20d2e2 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentCorpus.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/WordAlignmentCorpus.cs @@ -2,7 +2,7 @@ public record WordAlignmentCorpus { - public required string ParallelCorpusRef { get; set; } + public string? ParallelCorpusRef { get; set; } public IReadOnlyList? SourceFilters { get; set; } public IReadOnlyList? TargetFilters { get; set; } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs index ffbaba6e..aae367f8 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/EngineService.cs @@ -1,5 +1,4 @@ -using MassTransit.Mediator; -using Serval.WordAlignment.V1; +using Serval.WordAlignment.V1; namespace Serval.WordAlignment.Services; @@ -7,7 +6,6 @@ public class EngineService( IRepository engines, IRepository builds, IRepository wordAlignments, - IScopedMediator mediator, GrpcClientFactory grpcClientFactory, IOptionsMonitor dataFileOptions, IDataAccessContext dataAccessContext, @@ -17,7 +15,6 @@ IScriptureDataFileService scriptureDataFileService { private readonly IRepository _builds = builds; private readonly IRepository _wordAlignments = wordAlignments; - private readonly IScopedMediator _mediator = mediator; private readonly GrpcClientFactory _grpcClientFactory = grpcClientFactory; private readonly IOptionsMonitor _dataFileOptions = dataFileOptions; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; @@ -122,129 +119,47 @@ await _dataAccessContext.WithTransactionAsync( ); } + private Dictionary> GetChapters(string fileLocation, string scriptureRange) + { + try + { + return ScriptureRangeParser.GetChapters( + scriptureRange, + _scriptureDataFileService.GetParatextProjectSettings(fileLocation).Versification + ); + } + catch (ArgumentException ae) + { + throw new InvalidOperationException($"The scripture range {scriptureRange} is not valid: {ae.Message}"); + } + } + public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default) { Engine engine = await GetAsync(build.EngineRef, cancellationToken); await _builds.InsertAsync(build, cancellationToken); + WordAlignmentEngineApi.WordAlignmentEngineApiClient client = + _grpcClientFactory.CreateClient(engine.Type); + try { - var trainOn = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef); - var wordAlignOn = build.WordAlignOn?.ToDictionary(c => c.ParallelCorpusRef); - WordAlignmentEngineApi.WordAlignmentEngineApiClient client = - _grpcClientFactory.CreateClient(engine.Type); - Dictionary> GetChapters(V1.Corpus corpus, string scriptureRange) - { - try - { - return ScriptureRangeParser.GetChapters( - scriptureRange, - _scriptureDataFileService - .GetParatextProjectSettings(corpus.TargetFiles.First().Location) - .Versification - ); - } - catch (ArgumentException ae) - { - throw new InvalidOperationException( - $"The scripture range {scriptureRange} is not valid: {ae.Message}" - ); - } - } - var request = new StartBuildRequest + StartBuildRequest request; + var trainOn = build.TrainOn?.ToDictionary(c => c.ParallelCorpusRef!); + var wordAlignOn = build.WordAlignOn?.ToDictionary(c => c.ParallelCorpusRef!); + request = new StartBuildRequest { EngineType = engine.Type, EngineId = engine.Id, BuildId = build.Id, Corpora = { - engine.Corpora.Select(c => - { - V1.Corpus corpus = Map(c); - if (wordAlignOn?.TryGetValue(c.Id, out WordAlignmentCorpus? wordAlignmentCorpus) ?? false) - { - corpus.WordAlignOnAll = - wordAlignmentCorpus.TextIds is null && wordAlignmentCorpus.ScriptureRange is null; - if ( - wordAlignmentCorpus.TextIds is not null - && wordAlignmentCorpus.ScriptureRange is not null - ) - { - throw new InvalidOperationException( - $"The corpus {c.Id} cannot specify both 'textIds' and 'scriptureRange' for 'pretranslate'." - ); - } - if (wordAlignmentCorpus.TextIds is not null) - corpus.WordAlignOnTextIds.Add(wordAlignmentCorpus.TextIds); - if (!string.IsNullOrEmpty(wordAlignmentCorpus.ScriptureRange)) - { - if ( - c.TargetFiles.Count > 1 - || c.TargetFiles[0].Format != Shared.Contracts.FileFormat.Paratext - ) - { - throw new InvalidOperationException( - $"The corpus {c.Id} is not compatible with using a scripture range" - ); - } - corpus.WordAlignOnChapters.Add( - GetChapters(corpus, wordAlignmentCorpus.ScriptureRange) - .Select( - (kvp) => - { - var scriptureChapters = new ScriptureChapters(); - scriptureChapters.Chapters.Add(kvp.Value); - return (kvp.Key, scriptureChapters); - } - ) - .ToDictionary() - ); - } - } - if (trainOn?.TryGetValue(c.Id, out TrainingCorpus? trainingCorpus) ?? false) - { - corpus.TrainOnAll = trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null; - if (trainingCorpus.TextIds is not null && trainingCorpus.ScriptureRange is not null) - { - throw new InvalidOperationException( - $"The corpus {c.Id} cannot specify both 'textIds' and 'scriptureRange' for trainOn" - ); - } - if (trainingCorpus.TextIds is not null) - corpus.TrainOnTextIds.Add(trainingCorpus.TextIds); - if (!string.IsNullOrEmpty(trainingCorpus.ScriptureRange)) - { - if ( - c.TargetFiles.Count > 1 - || c.TargetFiles[0].Format != Shared.Contracts.FileFormat.Paratext - ) - { - throw new InvalidOperationException( - $"The corpus {c.Id} is not compatible with using a scripture range" - ); - } - corpus.TrainOnChapters.Add( - GetChapters(corpus, trainingCorpus.ScriptureRange) - .Select( - (kvp) => - { - var scriptureChapters = new ScriptureChapters(); - scriptureChapters.Chapters.Add(kvp.Value); - return (kvp.Key, scriptureChapters); - } - ) - .ToDictionary() - ); - } - } - else if (trainOn is null) - { - corpus.TrainOnAll = true; - } - return corpus; - }) + engine.ParallelCorpora.Select(c => + Map(c, trainOn?.GetValueOrDefault(c.Id), wordAlignOn?.GetValueOrDefault(c.Id)) + ) } }; + if (build.Options is not null) request.Options = JsonSerializer.Serialize(build.Options); @@ -309,39 +224,51 @@ await client.CancelBuildAsync( return true; } - public Task AddCorpusAsync(string engineId, Models.Corpus corpus, CancellationToken cancellationToken = default) + public Task AddParallelCorpusAsync( + string engineId, + Models.ParallelCorpus corpus, + CancellationToken cancellationToken = default + ) { - return Entities.UpdateAsync(engineId, u => u.Add(e => e.Corpora, corpus), cancellationToken: cancellationToken); + return Entities.UpdateAsync( + engineId, + u => u.Add(e => e.ParallelCorpora, corpus), + cancellationToken: cancellationToken + ); } - public async Task UpdateCorpusAsync( + public async Task UpdateParallelCorpusAsync( string engineId, - string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, + string parallelCorpusId, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ) { Engine? engine = await Entities.UpdateAsync( - e => e.Id == engineId && e.Corpora.Any(c => c.Id == corpusId), + e => e.Id == engineId && e.ParallelCorpora.Any(c => c.Id == parallelCorpusId), u => { - if (sourceFiles is not null) - u.Set(c => c.Corpora[ArrayPosition.FirstMatching].SourceFiles, sourceFiles); - if (targetFiles is not null) - u.Set(c => c.Corpora[ArrayPosition.FirstMatching].TargetFiles, targetFiles); + if (sourceCorpora is not null) + u.Set(c => c.ParallelCorpora[ArrayPosition.FirstMatching].SourceCorpora, sourceCorpora); + if (targetCorpora is not null) + u.Set(c => c.ParallelCorpora[ArrayPosition.FirstMatching].TargetCorpora, targetCorpora); }, cancellationToken: cancellationToken ); if (engine is null) - throw new EntityNotFoundException($"Could not find the Corpus '{corpusId}' in Engine '{engineId}'."); - return engine.Corpora.First(c => c.Id == corpusId); + { + throw new EntityNotFoundException( + $"Could not find the Corpus '{parallelCorpusId}' in Engine '{engineId}'." + ); + } + + return engine.ParallelCorpora.First(c => c.Id == parallelCorpusId); } - public async Task DeleteCorpusAsync( + public async Task DeleteParallelCorpusAsync( string engineId, - string corpusId, - bool deleteFiles, + string parallelCorpusId, CancellationToken cancellationToken = default ) { @@ -351,43 +278,39 @@ await _dataAccessContext.WithTransactionAsync( { originalEngine = await Entities.UpdateAsync( engineId, - u => u.RemoveAll(e => e.Corpora, c => c.Id == corpusId), + u => u.RemoveAll(e => e.ParallelCorpora, c => c.Id == parallelCorpusId), returnOriginal: true, cancellationToken: ct ); - if (originalEngine is null || !originalEngine.Corpora.Any(c => c.Id == corpusId)) + if (originalEngine is null || !originalEngine.ParallelCorpora.Any(c => c.Id == parallelCorpusId)) { throw new EntityNotFoundException( - $"Could not find the Corpus '{corpusId}' in Engine '{engineId}'." + $"Could not find the Corpus '{parallelCorpusId}' in Engine '{engineId}'." ); } - await _wordAlignments.DeleteAllAsync(pt => pt.CorpusRef == corpusId, ct); + await _wordAlignments.DeleteAllAsync(pt => pt.CorpusRef == parallelCorpusId, ct); }, cancellationToken: cancellationToken ); - if (deleteFiles && originalEngine != null) - { - foreach ( - string id in originalEngine.Corpora.SelectMany(c => - c.TargetFiles.Select(f => f.Id).Concat(c.SourceFiles.Select(f => f.Id).Distinct()) - ) - ) - { - await _mediator.Send(new { DataFileId = id }, cancellationToken); - } - } } public Task DeleteAllCorpusFilesAsync(string dataFileId, CancellationToken cancellationToken = default) { return Entities.UpdateAllAsync( e => - e.Corpora.Any(c => - c.SourceFiles.Any(f => f.Id == dataFileId) || c.TargetFiles.Any(f => f.Id == dataFileId) + e.ParallelCorpora.Any(c => + c.SourceCorpora.Any(sc => sc.Files.Any(f => f.Id == dataFileId)) + || c.TargetCorpora.Any(tc => tc.Files.Any(f => f.Id == dataFileId)) ), u => - u.RemoveAll(e => e.Corpora[ArrayPosition.All].SourceFiles, f => f.Id == dataFileId) - .RemoveAll(e => e.Corpora[ArrayPosition.All].TargetFiles, f => f.Id == dataFileId), + u.RemoveAll( + e => e.ParallelCorpora[ArrayPosition.All].SourceCorpora[ArrayPosition.All].Files, + f => f.Id == dataFileId + ) + .RemoveAll( + e => e.ParallelCorpora[ArrayPosition.All].TargetCorpora[ArrayPosition.All].Files, + f => f.Id == dataFileId + ), cancellationToken ); } @@ -419,16 +342,107 @@ private Shared.Models.AlignedWordPair Map(V1.AlignedWordPair source) return new Shared.Models.AlignedWordPair { SourceIndex = source.SourceIndex, TargetIndex = source.TargetIndex }; } - private V1.Corpus Map(Models.Corpus source) + private V1.ParallelCorpus Map( + Models.ParallelCorpus source, + TrainingCorpus? trainOn, + WordAlignmentCorpus? wordAlignOn + ) + { + string? referenceFileLocation = + source.TargetCorpora.Count > 0 && source.TargetCorpora[0].Files.Count > 0 + ? Map(source.TargetCorpora[0].Files[0]).Location + : null; + + return new V1.ParallelCorpus + { + Id = source.Id, + SourceCorpora = + { + source.SourceCorpora.Select(sc => + Map( + sc, + trainOn?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(), + wordAlignOn?.SourceFilters?.Where(sf => sf.CorpusRef == sc.Id).FirstOrDefault(), + referenceFileLocation + ) + ) + }, + TargetCorpora = + { + source.TargetCorpora.Select(tc => + Map( + tc, + trainOn?.TargetFilters?.Where(sf => sf.CorpusRef == tc.Id).FirstOrDefault(), + null, + referenceFileLocation + ) + ) + } + }; + } + + private V1.MonolingualCorpus Map( + Shared.Models.MonolingualCorpus source, + ParallelCorpusFilter? trainingFilter, + ParallelCorpusFilter? wordAlignmentFilter, + string? referenceFileLocation + ) { - return new V1.Corpus + Dictionary? trainOnChapters = null; + if ( + trainingFilter is not null + && trainingFilter.ScriptureRange is not null + && referenceFileLocation is not null + ) + { + trainOnChapters = GetChapters(referenceFileLocation, trainingFilter.ScriptureRange) + .Select( + (kvp) => + { + var scriptureChapters = new ScriptureChapters(); + scriptureChapters.Chapters.Add(kvp.Value); + return (kvp.Key, scriptureChapters); + } + ) + .ToDictionary(); + } + + Dictionary? wordAlignmentChapters = null; + if ( + wordAlignmentFilter is not null + && wordAlignmentFilter.ScriptureRange is not null + && referenceFileLocation is not null + ) + { + GetChapters(referenceFileLocation, wordAlignmentFilter.ScriptureRange) + .Select( + (kvp) => + { + var scriptureChapters = new ScriptureChapters(); + scriptureChapters.Chapters.Add(kvp.Value); + return (kvp.Key, scriptureChapters); + } + ) + .ToDictionary(); + } + + var corpus = new V1.MonolingualCorpus { Id = source.Id, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - SourceFiles = { source.SourceFiles.Select(Map) }, - TargetFiles = { source.TargetFiles.Select(Map) } + Language = source.Language, + Files = { source.Files.Select(Map) } }; + + if (trainOnChapters is not null) + corpus.TrainOnChapters.Add(trainOnChapters); + if (trainingFilter?.TextIds is not null) + corpus.TrainOnTextIds.Add(trainingFilter.TextIds); + if (wordAlignmentChapters is not null) + corpus.WordAlignOnChapters.Add(wordAlignmentChapters); + if (wordAlignmentFilter?.TextIds is not null) + corpus.WordAlignOnTextIds.Add(wordAlignmentFilter.TextIds); + + return corpus; } private V1.CorpusFile Map(Shared.Models.CorpusFile source) diff --git a/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs b/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs index f4be871d..17a59f6c 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/IEngineService.cs @@ -19,18 +19,17 @@ Task GetWordAlignmentAsync( Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - Task AddCorpusAsync(string engineId, Corpus corpus, CancellationToken cancellationToken = default); - Task UpdateCorpusAsync( + Task AddParallelCorpusAsync(string engineId, ParallelCorpus corpus, CancellationToken cancellationToken = default); + Task UpdateParallelCorpusAsync( string engineId, - string corpusId, - IReadOnlyList? sourceFiles, - IReadOnlyList? targetFiles, + string parallelCorpusId, + IReadOnlyList? sourceCorpora, + IReadOnlyList? targetCorpora, CancellationToken cancellationToken = default ); - Task DeleteCorpusAsync( + Task DeleteParallelCorpusAsync( string engineId, - string corpusId, - bool deleteFiles, + string parallelCorpusId, CancellationToken cancellationToken = default ); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index 0253b00e..64fb32ec 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -1683,7 +1683,7 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -1718,7 +1718,7 @@ public async Task CreateParallelCorpusEngineWithTextFilesAsync() ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() { @@ -1774,7 +1774,7 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() new() { Id = "parallel-corpus1", - SourceCorpora = new List() + SourceCorpora = new List() { new() { @@ -1809,7 +1809,7 @@ public async Task CreateParallelCorpusEngineWithParatextProjectAsync() ] } }, - TargetCorpora = new List() + TargetCorpora = new List() { new() {