From 99875d15185cc7de1d9581ca84b98febf8a88948 Mon Sep 17 00:00:00 2001 From: "Eli C. Lowry" <83078660+Enkidu93@users.noreply.github.com> Date: Fri, 6 Oct 2023 16:17:05 -0400 Subject: [PATCH] Config passing (#152) * Basic config passing as string --ECL * MInor changes to options field --ECL * Store buildOptions as object in MongoDB rather than string * (Oops, incomplete commit) * Fix without using Json.net * No json.net * Fixed usings * Fixes as per PR review --- src/Serval.Client/Client.g.cs | 6 ++++++ .../Protos/serval/translation/v1/engine.proto | 3 ++- src/Serval.Shared/Usings.cs | 2 ++ .../Utils/ObjectToInferredTypesConverter.cs | 19 ++++++++++++++++++ .../Contracts/TranslationBuildConfigDto.cs | 2 ++ .../Contracts/TranslationBuildDto.cs | 1 + .../TranslationEnginesController.cs | 20 ++++++++++++++++++- src/Serval.Translation/Models/Build.cs | 1 + .../Services/EngineService.cs | 1 + src/Serval.Translation/Usings.cs | 7 ++++--- 10 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 src/Serval.Shared/Utils/ObjectToInferredTypesConverter.cs diff --git a/src/Serval.Client/Client.g.cs b/src/Serval.Client/Client.g.cs index 6482fcda..e02ca400 100644 --- a/src/Serval.Client/Client.g.cs +++ b/src/Serval.Client/Client.g.cs @@ -4320,6 +4320,9 @@ public partial class TranslationBuild [Newtonsoft.Json.JsonProperty("dateFinished", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public System.DateTimeOffset? DateFinished { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Options { get; set; } = default!; + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "13.18.2.0 (NJsonSchema v10.8.0.0 (Newtonsoft.Json v13.0.0.0))")] @@ -4364,6 +4367,9 @@ public partial class TranslationBuildConfig [Newtonsoft.Json.JsonProperty("pretranslate", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public System.Collections.Generic.IList? Pretranslate { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Options { get; set; } = default!; + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "13.18.2.0 (NJsonSchema v10.8.0.0 (Newtonsoft.Json v13.0.0.0))")] diff --git a/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto b/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto index b5d37d37..08f258fb 100644 --- a/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto +++ b/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto @@ -60,7 +60,8 @@ message StartBuildRequest { string engine_type = 1; string engine_id = 2; string build_id = 3; - repeated Corpus corpora = 4; + optional string options = 4; + repeated Corpus corpora = 5; } message CancelBuildRequest { diff --git a/src/Serval.Shared/Usings.cs b/src/Serval.Shared/Usings.cs index a0ea2a99..f7de1ea8 100644 --- a/src/Serval.Shared/Usings.cs +++ b/src/Serval.Shared/Usings.cs @@ -1,5 +1,7 @@ global using Grpc.Core; global using Grpc.Net.ClientFactory; +global using System.Text.Json; +global using System.Text.Json.Serialization; global using Microsoft.AspNetCore.Authorization; global using Microsoft.AspNetCore.Http; global using Microsoft.AspNetCore.Mvc; diff --git a/src/Serval.Shared/Utils/ObjectToInferredTypesConverter.cs b/src/Serval.Shared/Utils/ObjectToInferredTypesConverter.cs new file mode 100644 index 00000000..875f1484 --- /dev/null +++ b/src/Serval.Shared/Utils/ObjectToInferredTypesConverter.cs @@ -0,0 +1,19 @@ +namespace Serval.Shared.Utils; + +public class ObjectToInferredTypesConverter : JsonConverter +{ + public override object Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + reader.TokenType switch + { + JsonTokenType.True => true, + JsonTokenType.False => false, + JsonTokenType.Number when reader.TryGetInt64(out long l) => l, + JsonTokenType.Number => reader.GetDouble(), + JsonTokenType.String when reader.TryGetDateTime(out DateTime datetime) => datetime, + JsonTokenType.String => reader.GetString()!, + _ => JsonDocument.ParseValue(ref reader).RootElement.Clone() + }; + + public override void Write(Utf8JsonWriter writer, object objectToWrite, JsonSerializerOptions options) => + JsonSerializer.Serialize(writer, objectToWrite, objectToWrite.GetType(), options); +} diff --git a/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs b/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs index 7ad0a141..14e1d2c4 100644 --- a/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs +++ b/src/Serval.Translation/Contracts/TranslationBuildConfigDto.cs @@ -4,4 +4,6 @@ public class TranslationBuildConfigDto { public string? Name { get; set; } public IList? Pretranslate { get; set; } + + public string? Options { get; set; } } diff --git a/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval.Translation/Contracts/TranslationBuildDto.cs index 561caa95..3e914b44 100644 --- a/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval.Translation/Contracts/TranslationBuildDto.cs @@ -17,4 +17,5 @@ public class TranslationBuildDto /// public JobState State { get; set; } public DateTime? DateFinished { get; set; } + public string? Options { get; set; } } diff --git a/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 836d264d..eaa55d75 100644 --- a/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -784,6 +784,10 @@ CancellationToken cancellationToken { return BadRequest(ioe.Message); } + catch (ArgumentException ae) + { + return BadRequest(ae.Message); + } if (!await _engineService.StartBuildAsync(build, cancellationToken)) return NotFound(); @@ -976,6 +980,19 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source) } build.Pretranslate = pretranslateCorpora; } + try + { + var jsonSerializerOptions = new JsonSerializerOptions(); + jsonSerializerOptions.Converters.Add(new ObjectToInferredTypesConverter()); + build.Options = JsonSerializer.Deserialize>( + source.Options ?? "{}", + jsonSerializerOptions + ); + } + catch (Exception e) + { + throw new ArgumentException($"Unable to parse field 'options' : {e.Message}"); + } return build; } @@ -1014,7 +1031,8 @@ private TranslationBuildDto Map(Build source) PercentCompleted = source.PercentCompleted, Message = source.Message, State = source.State, - DateFinished = source.DateFinished + DateFinished = source.DateFinished, + Options = JsonSerializer.Serialize(source.Options) }; } diff --git a/src/Serval.Translation/Models/Build.cs b/src/Serval.Translation/Models/Build.cs index 8791c252..d934a35c 100644 --- a/src/Serval.Translation/Models/Build.cs +++ b/src/Serval.Translation/Models/Build.cs @@ -12,4 +12,5 @@ public class Build : IEntity public string? Message { get; set; } public JobState State { get; set; } = JobState.Pending; public DateTime? DateFinished { get; set; } + public IDictionary? Options { get; set; } } diff --git a/src/Serval.Translation/Services/EngineService.cs b/src/Serval.Translation/Services/EngineService.cs index 8c55dfab..408cb9f3 100644 --- a/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval.Translation/Services/EngineService.cs @@ -197,6 +197,7 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellat EngineType = engine.Type, EngineId = engine.Id, BuildId = build.Id, + Options = JsonSerializer.Serialize(build.Options), Corpora = { engine.Corpora.Select(c => diff --git a/src/Serval.Translation/Usings.cs b/src/Serval.Translation/Usings.cs index 41550d03..534b5ddb 100644 --- a/src/Serval.Translation/Usings.cs +++ b/src/Serval.Translation/Usings.cs @@ -1,6 +1,4 @@ -global using System.Diagnostics.CodeAnalysis; -global using System.Linq.Expressions; -global using Asp.Versioning; +global using Asp.Versioning; global using Grpc.Core; global using Grpc.Net.ClientFactory; global using MassTransit; @@ -23,3 +21,6 @@ global using Serval.Translation.Models; global using Serval.Translation.Services; global using SIL.DataAccess; +global using System.Diagnostics.CodeAnalysis; +global using System.Linq.Expressions; +global using System.Text.Json;