From c15c2490e05827216cf5b7b2055df4decfd7ce8d Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Thu, 22 Aug 2024 07:20:27 +1200 Subject: [PATCH 1/6] SF-2926 Allow viewing history if Paratext sync directory is missing (#2674) --- .../Services/ParatextService.cs | 16 +++++++++++-- .../Services/ParatextServiceTests.cs | 23 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/SIL.XForge.Scripture/Services/ParatextService.cs b/src/SIL.XForge.Scripture/Services/ParatextService.cs index 8b97eaae41..16f69506e2 100644 --- a/src/SIL.XForge.Scripture/Services/ParatextService.cs +++ b/src/SIL.XForge.Scripture/Services/ParatextService.cs @@ -1979,8 +1979,17 @@ int chapter milestonePeriod = ops.FirstOrDefault()?.Metadata.Timestamp ?? DateTime.UtcNow; // Load the Paratext project - string ptProjectId = projectDoc.Data.ParatextId; - using ScrText scrText = GetScrText(userSecret, ptProjectId); + ScrText scrText; + try + { + string ptProjectId = projectDoc.Data.ParatextId; + scrText = GetScrText(userSecret, ptProjectId); + } + catch (DataNotFoundException) + { + // If an error occurs loading the project from disk, just return the revisions from Mongo + yield break; + } // Get the Paratext users Dictionary paratextUsers = projectDoc.Data.ParatextUsers.ToDictionary( @@ -2019,6 +2028,9 @@ HgRevision revision in revisionCollection.MutableCollection.Where(r => r.CommitT }; } } + + // Clean up the scripture text + scrText.Dispose(); } /// diff --git a/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs b/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs index 6525ead3c5..8849be3666 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs @@ -15,6 +15,7 @@ using Newtonsoft.Json.Linq; using NSubstitute; using NSubstitute.ExceptionExtensions; +using NSubstitute.ReturnsExtensions; using NUnit.Framework; using Paratext.Data; using Paratext.Data.Languages; @@ -5027,6 +5028,28 @@ public void GetRevisionHistoryAsync_MissingUser() }); } + [Test] + public async Task GetRevisionHistoryAsync_MissingParatextDirectory() + { + var env = new TestEnvironment(); + UserSecret userSecret = TestEnvironment.MakeUserSecret(env.User01, env.Username01, env.ParatextUserId01); + var associatedPtUser = new SFParatextUser(env.Username01); + env.SetupProject(env.Project01, associatedPtUser); + SFProject project = env.NewSFProject(); + project.UserRoles = new Dictionary { { env.User01, SFProjectRole.PTObserver } }; + env.AddProjectRepository(project); + env.MockScrTextCollection.FindById(Arg.Any(), Arg.Any()).ReturnsNull(); + + // SUT + bool historyExists = false; + await foreach (DocumentRevision _ in env.Service.GetRevisionHistoryAsync(userSecret, project.Id, "MAT", 1)) + { + historyExists = true; + } + + Assert.IsTrue(historyExists); + } + [Test] public async Task GetRevisionHistoryAsync_Success() { From 27c7419f4ea3d8aa324f04fd6b7bd476cdb8d0e9 Mon Sep 17 00:00:00 2001 From: Nathaniel Paulus Date: Wed, 21 Aug 2024 18:14:50 -0400 Subject: [PATCH 2/6] SF-2927 Fix incorrect escaping of XML characters in notes (#2675) Co-authored-by: MarkS Co-authored-by: Peter Chapman --- .../biblical-terms.component.spec.ts | 5 +- .../biblical-terms.component.ts | 9 +- .../Services/ParatextService.cs | 10 +- .../Services/ParatextServiceTests.cs | 120 +++++++++--------- 4 files changed, 76 insertions(+), 68 deletions(-) diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.spec.ts index ef9a3397b4..2cdd055b4a 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.spec.ts @@ -38,6 +38,7 @@ import { SFProjectProfileDoc } from '../../core/models/sf-project-profile-doc'; import { SFProjectUserConfigDoc } from '../../core/models/sf-project-user-config-doc'; import { SF_TYPE_REGISTRY } from '../../core/models/sf-type-registry'; import { SFProjectService } from '../../core/sf-project.service'; +import { XmlUtils } from '../../shared/utils'; import { MockNoteDialogRef } from '../editor/editor.component.spec'; import { NoteDialogComponent } from '../editor/note-dialog/note-dialog.component'; import { BiblicalTermDialogIcon, BiblicalTermNoteIcon, BiblicalTermsComponent } from './biblical-terms.component'; @@ -215,7 +216,7 @@ describe('BiblicalTermsComponent', () => { env.biblicalTermsNotesButton.click(); env.wait(); - const noteContent: string = 'content in the thread'; + const noteContent: string = 'content in the thread & with an ampersand'; env.mockNoteDialogRef.close({ noteContent }); env.wait(); @@ -232,7 +233,7 @@ describe('BiblicalTermsComponent', () => { expect(noteThread.originalSelectedText).toEqual(''); expect(noteThread.publishedToSF).toBe(true); expect(noteThread.notes[0].ownerRef).toEqual('user01'); - expect(noteThread.notes[0].content).toEqual(noteContent); + expect(noteThread.notes[0].content).toEqual(XmlUtils.encodeForXml(noteContent)); expect(noteThread.notes[0].tagId).toEqual(BIBLICAL_TERM_TAG_ID); const projectUserConfigDoc: SFProjectUserConfigDoc = env.getProjectUserConfigDoc('project01', 'user01'); expect(projectUserConfigDoc.data!.noteRefsRead).not.toContain(noteThread.notes[0].dataId); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.ts index b418f66250..b341d9697e 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/biblical-terms/biblical-terms.component.ts @@ -12,7 +12,7 @@ import { NoteThread, NoteType } from 'realtime-server/lib/esm/scriptureforge/models/note-thread'; -import { SFProjectDomain, SF_PROJECT_RIGHTS } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-rights'; +import { SF_PROJECT_RIGHTS, SFProjectDomain } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-rights'; import { fromVerseRef } from 'realtime-server/lib/esm/scriptureforge/models/verse-ref-data'; import { BehaviorSubject, firstValueFrom, merge, Subscription } from 'rxjs'; import { filter } from 'rxjs/operators'; @@ -29,7 +29,7 @@ import { SFProjectProfileDoc } from '../../core/models/sf-project-profile-doc'; import { SFProjectUserConfigDoc } from '../../core/models/sf-project-user-config-doc'; import { TextDocId } from '../../core/models/text-doc'; import { SFProjectService } from '../../core/sf-project.service'; -import { getVerseNumbers } from '../../shared/utils'; +import { getVerseNumbers, XmlUtils } from '../../shared/utils'; import { SaveNoteParameters } from '../editor/editor.component'; import { NoteDialogComponent, NoteDialogData, NoteDialogResult } from '../editor/note-dialog/note-dialog.component'; import { BiblicalTermDialogComponent, BiblicalTermDialogData } from './biblical-term-dialog.component'; @@ -468,6 +468,7 @@ export class BiblicalTermsComponent extends DataLoadingComponent implements OnDe const currentDate: string = new Date().toJSON(); const threadId = `BT_${biblicalTermDoc.data.termId}`; + const noteContent: string | undefined = params.content == null ? undefined : XmlUtils.encodeForXml(params.content); // Configure the note const note: Note = { dateCreated: currentDate, @@ -476,7 +477,7 @@ export class BiblicalTermsComponent extends DataLoadingComponent implements OnDe threadId, tagId: BIBLICAL_TERM_TAG_ID, ownerRef: this.userService.currentUserId, - content: params.content, + content: noteContent, conflictType: NoteConflictType.DefaultValue, type: NoteType.Normal, status: params.status ?? NoteStatus.Todo, @@ -516,7 +517,7 @@ export class BiblicalTermsComponent extends DataLoadingComponent implements OnDe const noteIndex: number = threadDoc.data!.notes.findIndex(n => n.dataId === params.dataId); if (noteIndex >= 0) { await threadDoc!.submitJson0Op(op => { - op.set(t => t.notes[noteIndex].content, params.content); + op.set(t => t.notes[noteIndex].content, noteContent); op.set(t => t.notes[noteIndex].dateModified, currentDate); }); } else { diff --git a/src/SIL.XForge.Scripture/Services/ParatextService.cs b/src/SIL.XForge.Scripture/Services/ParatextService.cs index 16f69506e2..fca4114a9a 100644 --- a/src/SIL.XForge.Scripture/Services/ParatextService.cs +++ b/src/SIL.XForge.Scripture/Services/ParatextService.cs @@ -2920,15 +2920,17 @@ Dictionary ptProjectUsers string label = $"[{displayName} - {_siteOptions.Value.Name}]"; var labelElement = new XElement("p", label, new XAttribute("sf-user-label", "true")); contentElem.Add(labelElement); + + XDocument commentDoc = XDocument.Parse($"{note.Content}"); if (!note.Content.StartsWith("

")) { // add the note content in a paragraph tag - XElement commentNode = new XElement("p", note.Content); - contentElem.Add(commentNode); + XElement paragraph = new XElement("p"); + paragraph.Add(commentDoc.Root.Nodes()); + contentElem.Add(paragraph); } else { - XDocument commentDoc = XDocument.Parse($"{note.Content}"); // add the note content paragraph by paragraph foreach (XElement paragraphElems in commentDoc.Root.Descendants("p")) contentElem.Add(paragraphElems); @@ -2981,7 +2983,7 @@ private static string GetNoteContentFromComment(Paratext.Data.ProjectComments.Co continue; // If there is only one paragraph node other than the SF user label then omit the paragraph tags if (isReviewer && paragraphNodeCount <= 2) - sb.Append(element.Value); + sb.AppendJoin(string.Empty, element.Nodes()); else sb.Append(node); } diff --git a/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs b/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs index 8849be3666..fe74c6f007 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs @@ -5,6 +5,7 @@ using System.Net; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Xml; @@ -1160,31 +1161,34 @@ public void GetNoteThreadChanges_AddsNoteWithXmlFormatting() string paratextId = env.SetupProject(env.Project01, associatedPtUser); UserSecret userSecret = TestEnvironment.MakeUserSecret(env.User01, env.Username01, env.ParatextUserId01); - string formattedContent = "Text with bold and italics styles."; - string nonFormattedContent = "Text without formatting"; - string formattedContentInParagraph = + const string formattedContent = "Text with bold and italics styles."; + const string nonFormattedContent = "Text without formatting"; + const string formattedContentInParagraph = "

Text with bold style.

Text withitalics style.

"; - string nonFormattedContentInParagraph = "

Text without formatting.

Second paragraph.

"; - string whitespaceInContent = "

First paragraph.

\n

Second paragraph.

"; + const string nonFormattedContentInParagraph = "

Text without formatting.

Second paragraph.

"; + const string whitespaceInContent = "

First paragraph.

\n

Second paragraph.

"; + const string reviewerAuthorParagraph = "

[testuser - Scripture Forge]

"; + const string reviewerContentParagraph = "

Test entity parsing with 1&2 John!

"; + const string reviewerContent = reviewerAuthorParagraph + reviewerContentParagraph; var note1 = new ThreadNoteComponents { content = formattedContent }; var note2 = new ThreadNoteComponents { content = nonFormattedContent }; var note3 = new ThreadNoteComponents { content = formattedContentInParagraph }; var note4 = new ThreadNoteComponents { content = nonFormattedContentInParagraph }; var note5 = new ThreadNoteComponents { content = whitespaceInContent }; + var note6 = new ThreadNoteComponents { content = reviewerContent }; env.AddParatextComments( - new[] - { + [ new ThreadComponents { threadNum = 1, - noteCount = 5, + noteCount = 6, username = env.Username01, - notes = new[] { note1, note2, note3, note4, note5 } - } - } + notes = [note1, note2, note3, note4, note5, note6], + }, + ] ); - var noteThreadDocs = Array.Empty>(); + IDocument[] noteThreadDocs = []; Dictionary chapterDeltas = env.GetChapterDeltasByBook(1, env.ContextBefore, "Text selected"); Dictionary ptProjectUsers = new Dictionary { @@ -1194,9 +1198,9 @@ public void GetNoteThreadChanges_AddsNoteWithXmlFormatting() { SFUserId = env.User01, OpaqueUserId = "syncuser01", - Username = env.Username01 + Username = env.Username01, } - } + }, }; IEnumerable changes = env.Service.GetNoteThreadChanges( userSecret, @@ -1208,17 +1212,19 @@ public void GetNoteThreadChanges_AddsNoteWithXmlFormatting() ); Assert.That(changes.Count, Is.EqualTo(1)); NoteThreadChange change1 = changes.Single(); - string expected1 = $"thread1-syncuser01-{formattedContent}"; + const string expected1 = $"thread1-syncuser01-{formattedContent}"; Assert.That(change1.NotesAdded[0].NoteToString(), Is.EqualTo(expected1)); - string expected2 = $"thread1-syncuser01-{nonFormattedContent}"; + const string expected2 = $"thread1-syncuser01-{nonFormattedContent}"; Assert.That(change1.NotesAdded[1].NoteToString(), Is.EqualTo(expected2)); - string expected3 = $"thread1-syncuser01-{formattedContentInParagraph}"; + const string expected3 = $"thread1-syncuser01-{formattedContentInParagraph}"; Assert.That(change1.NotesAdded[2].NoteToString(), Is.EqualTo(expected3)); - string expected4 = $"thread1-syncuser01-{nonFormattedContentInParagraph}"; + const string expected4 = $"thread1-syncuser01-{nonFormattedContentInParagraph}"; Assert.That(change1.NotesAdded[3].NoteToString(), Is.EqualTo(expected4)); // whitespace does not get processed as a node in xml, so it gets omitted from note content - string expected5 = $"thread1-syncuser01-{whitespaceInContent}".Replace("\n", ""); + string expected5 = $"thread1-syncuser01-{whitespaceInContent.Replace("\n", string.Empty)}"; Assert.That(change1.NotesAdded[4].NoteToString(), Is.EqualTo(expected5)); + string expected6 = $"thread1-syncuser01-{Regex.Replace(reviewerContentParagraph, "", string.Empty)}"; + Assert.That(change1.NotesAdded[5].NoteToString(), Is.EqualTo(expected6)); } [Test] @@ -2625,80 +2631,78 @@ public async Task UpdateParatextComments_AddsComment() string ptProjectId = env.SetupProject(env.Project01, associatedPtUser); UserSecret userSecret = TestEnvironment.MakeUserSecret(env.User01, env.Username01, env.ParatextUserId01); - string dataId1 = "dataId1"; - string dataId2 = "dataId2"; - string dataId3 = "dataId3"; - string thread1 = "thread1"; - string thread2 = "thread2"; - string thread3 = "thread3"; - var thread1Notes = new[] - { + const string dataId1 = "dataId1"; + const string dataId2 = "dataId2"; + const string dataId3 = "dataId3"; + const string thread1 = "thread1"; + const string thread2 = "thread2"; + const string thread3 = "thread3"; + ThreadNoteComponents[] thread1Notes = + [ new ThreadNoteComponents { ownerRef = env.User01, - tagsAdded = new[] { "1" }, + tagsAdded = ["1"], editable = true, versionNumber = 1, status = NoteStatus.Todo, - } - }; - var thread2Notes = new[] - { + }, + ]; + ThreadNoteComponents[] thread2Notes = + [ new ThreadNoteComponents { ownerRef = env.User05, - tagsAdded = new[] { "1" }, + tagsAdded = ["1"], editable = true, versionNumber = 1, - } - }; + content = "See 1&2 John", + }, + ]; env.AddNoteThreadData( - new[] - { + [ new ThreadComponents { threadNum = 1, noteCount = 1, isNew = true, - notes = thread1Notes + notes = thread1Notes, }, new ThreadComponents { threadNum = 2, noteCount = 1, isNew = true, - notes = thread2Notes + notes = thread2Notes, }, new ThreadComponents { threadNum = 3, noteCount = 2, - deletedNotes = new[] { false, true }, - versionNumber = 1 - } - } + deletedNotes = [false, true], + versionNumber = 1, + }, + ] ); env.AddParatextComments( - new[] - { + [ new ThreadComponents { threadNum = 3, noteCount = 1, username = env.Username01, - deletedNotes = new[] { false }, - versionNumber = 1 - } - } + deletedNotes = [false], + versionNumber = 1, + }, + ] ); await using IConnection conn = await env.RealtimeService.ConnectAsync(); CommentThread thread = env.ProjectCommentManager.FindThread(thread1); Assert.That(thread, Is.Null); - string[] noteThreadDataIds = new[] { dataId1, dataId2, dataId3 }; - IEnumerable> noteThreadDocs = await TestEnvironment.GetNoteThreadDocsAsync( - conn, - noteThreadDataIds - ); + string[] noteThreadDataIds = [dataId1, dataId2, dataId3]; + List> noteThreadDocs = ( + await TestEnvironment.GetNoteThreadDocsAsync(conn, noteThreadDataIds) + ).ToList(); Dictionary ptProjectUsers = new Dictionary { { @@ -2707,9 +2711,9 @@ public async Task UpdateParatextComments_AddsComment() { Username = env.Username01, OpaqueUserId = "syncuser01", - SFUserId = env.User01 + SFUserId = env.User01, } - } + }, }; SyncMetricInfo syncMetricInfo = await env.Service.UpdateParatextCommentsAsync( userSecret, @@ -2739,7 +2743,7 @@ public async Task UpdateParatextComments_AddsComment() expected = "thread2/User 01/2019-01-01T08:00:00.0000000+00:00-" + "MAT 1:2-" - + "

[User 05 - xForge]

thread2 note 1.

-" + + $"

[User 05 - xForge]

{thread2Notes[0].content}

-" + "Start:0-" + "user05-" + "Tag:1-" @@ -2757,7 +2761,7 @@ public async Task UpdateParatextComments_AddsComment() Assert.That(syncMetricInfo, Is.EqualTo(new SyncMetricInfo(added: 2, deleted: 0, updated: 0))); // PT username is not written to server logs - env.MockLogger.AssertNoEvent((LogEvent logEvent) => logEvent.Message.Contains(env.Username02)); + env.MockLogger.AssertNoEvent(logEvent => logEvent.Message!.Contains(env.Username02)); } [Test] From b3380677ad23db619c7137a67832203e942fb1a6 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Thu, 22 Aug 2024 11:55:54 +1200 Subject: [PATCH 3/6] SF-2918 Create extract project script for Windows and Linux (#2673) --- scripts/extract-project.bat | 51 +++++++++++++++++++++++++++++++++++++ scripts/extract-project.sh | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 scripts/extract-project.bat create mode 100644 scripts/extract-project.sh diff --git a/scripts/extract-project.bat b/scripts/extract-project.bat new file mode 100644 index 0000000000..b2a8f2eb28 --- /dev/null +++ b/scripts/extract-project.bat @@ -0,0 +1,51 @@ +@ECHO OFF +REM Extract Scripture Forge Project Data +REM +REM Usage: +REM extract-project {server_name}:{server_port} {project_id} +REM +REM Example: +REM extract-project localhost:27017 66bb989d961308bb2652ac15 +REM +REM Notes: +REM - User associations to projects will not be extracted. This must be done manually. +REM - Flat files need to be restored manually. +REM - To restore the project to a MongoDB server, run the following command: +REM mongorestore -h {server_name}:{server_port} {project_id} + +SETLOCAL +IF %1.==. GOTO NO_SERVER +IF %2.==. GOTO NO_PROJECTID +SET SERVER=%1 +SET PROJECTID=%2 +SET OUTPUT_PATH=%PROJECTID% + +ECHO Extracting %PROJECTID% from %SERVER%... +mongodump -h %SERVER% -d xforge -c sf_projects -o %OUTPUT_PATH% -q "{\"_id\":\"%PROJECTID%\"}" +mongodump -h %SERVER% -d xforge -c o_sf_projects -o %OUTPUT_PATH% -q "{\"d\":\"%PROJECTID%\"}" +mongodump -h %SERVER% -d xforge -c sf_project_secrets -o %OUTPUT_PATH% -q "{\"_id\":\"%PROJECTID%\"}" +mongodump -h %SERVER% -d xforge -c sf_project_user_configs -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_sf_project_user_configs -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c texts -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_texts -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c m_texts -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c questions -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_questions -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c note_threads -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_note_threads -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c text_audio -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_text_audio -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c biblical_terms -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_biblical_terms -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c training_data -o %OUTPUT_PATH% -q "{\"_id\":{\"$regex\":\"^%PROJECTID%:\"}}" +mongodump -h %SERVER% -d xforge -c o_training_data -o %OUTPUT_PATH% -q "{\"d\":{\"$regex\":\"^%PROJECTID%:\"}}" +GOTO END + +:NO_SERVER +ECHO Please specify the server and port as the first argument +GOTO END + +:NO_PROJECTID +ECHO Please specify a project id as the second argument +GOTO END +:END diff --git a/scripts/extract-project.sh b/scripts/extract-project.sh new file mode 100644 index 0000000000..daa87df3a1 --- /dev/null +++ b/scripts/extract-project.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Extract Scripture Forge Project Data +# +# Usage: +# extract-project.sh {server_name}:{server_port} {project_id} +# +# Example: +# ./extract-project.sh localhost:27017 66bb989d961308bb2652ac15 +# +# Notes: +# - User associations to projects will not be extracted. This must be done manually. +# - Flat files need to be restored manually. +# - To restore the project to a MongoDB server, run the following command: +# mongorestore -h {server_name}:{server_port} {project_id} + +if [ -z "$1" ]; then + echo "Please specify the server and port as the first argument" + exit 1 +fi + +if [ -z "$2" ]; then + echo "Please specify a project id as the second argument" + exit 1 +fi + +SERVER=$1 +PROJECTID=$2 +OUTPUT_PATH=$PROJECTID + +echo "Extracting $PROJECTID from $SERVER..." +mongodump -h "$SERVER" -d xforge -c sf_projects -o "$OUTPUT_PATH" -q "{\"_id\":\"$PROJECTID\"}" +mongodump -h "$SERVER" -d xforge -c o_sf_projects -o "$OUTPUT_PATH" -q "{\"d\":\"$PROJECTID\"}" +mongodump -h "$SERVER" -d xforge -c sf_project_secrets -o "$OUTPUT_PATH" -q "{\"_id\":\"$PROJECTID\"}" +mongodump -h "$SERVER" -d xforge -c sf_project_user_configs -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_sf_project_user_configs -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c texts -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_texts -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c m_texts -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c questions -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_questions -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c note_threads -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_note_threads -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c text_audio -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_text_audio -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c biblical_terms -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_biblical_terms -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c training_data -o "$OUTPUT_PATH" -q "{\"_id\":{\"\$regex\":\"^$PROJECTID:\"}}" +mongodump -h "$SERVER" -d xforge -c o_training_data -o "$OUTPUT_PATH" -q "{\"d\":{\"\$regex\":\"^$PROJECTID:\"}}" From dbbbcfbc630b85d8ad94c20ee300ce237cb9e0c2 Mon Sep 17 00:00:00 2001 From: josephmyers <43733878+josephmyers@users.noreply.github.com> Date: Thu, 22 Aug 2024 08:12:08 +0700 Subject: [PATCH 4/6] SF-2901 Display completion status via new ProgressService (#2670) --- .../book-multi-select.component.html | 1 + .../book-multi-select.component.scss | 15 ++ .../book-multi-select.component.spec.ts | 61 ++++--- .../book-multi-select.component.ts | 42 +++-- .../progress-service/progress-service.spec.ts | 151 ++++++++++++++++ .../progress-service/progress-service.ts | 168 ++++++++++++++++++ .../draft-generation-steps.component.spec.ts | 17 +- .../draft-generation-steps.component.ts | 6 +- .../translate-overview.component.html | 10 +- .../translate-overview.component.spec.ts | 55 +----- .../translate-overview.component.ts | 115 +----------- .../xforge-common/data-loading-component.ts | 17 +- 12 files changed, 456 insertions(+), 202 deletions(-) create mode 100644 src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.spec.ts create mode 100644 src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.ts diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.html b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.html index d620b6a127..6eccb64117 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.html +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.html @@ -41,6 +41,7 @@ > {{ "canon.book_names." + book.bookId | transloco }} +
} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.scss b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.scss index b8a413e540..9a422f8908 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.scss +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.scss @@ -1,12 +1,27 @@ @use 'src/variables'; .book-multi-select { + .mat-mdc-standard-chip { + position: relative; + display: inline-block; + overflow: hidden; + border-radius: 4px; + } + .mat-mdc-standard-chip { &.mat-mdc-chip-selected, &.mat-mdc-chip-highlighted { --mdc-chip-elevated-selected-container-color: #4a79c9; } } + + .border-fill { + position: absolute; + bottom: 0; + left: 0; + height: 3px; + background: black; + } } .bulk-select { diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.spec.ts index d1c06a5ba9..087a3a03bc 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.spec.ts @@ -1,7 +1,14 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; import { MatChipsModule } from '@angular/material/chips'; +import { ActivatedRoute } from '@angular/router'; +import { of } from 'rxjs'; +import { mock, when } from 'ts-mockito'; import { configureTestingModule, TestTranslocoModule } from 'xforge-common/test-utils'; -import { BookMultiSelectComponent, BookOption } from './book-multi-select.component'; +import { ProgressService, TextProgress } from '../progress-service/progress-service'; +import { BookMultiSelectComponent } from './book-multi-select.component'; + +const mockedActivatedRoute = mock(ActivatedRoute); +const mockedProgressService = mock(ProgressService); describe('BookMultiSelectComponent', () => { let component: BookMultiSelectComponent; @@ -11,12 +18,26 @@ describe('BookMultiSelectComponent', () => { let mockSelectedBooks: number[]; configureTestingModule(() => ({ - imports: [MatChipsModule, TestTranslocoModule] + imports: [MatChipsModule, TestTranslocoModule], + providers: [ + { provide: ActivatedRoute, useMock: mockedActivatedRoute }, + { provide: ProgressService, useMock: mockedProgressService } + ] })); beforeEach(() => { mockBooks = [1, 2, 3, 42, 70]; mockSelectedBooks = [1, 3]; + when(mockedActivatedRoute.params).thenReturn(of({ projectId: 'project01' })); + when(mockedProgressService.isLoaded$).thenReturn(of(true)); + when(mockedProgressService.texts).thenReturn([ + { text: { bookNum: 1 }, percentage: 0 } as TextProgress, + { text: { bookNum: 2 }, percentage: 20 } as TextProgress, + { text: { bookNum: 3 }, percentage: 40 } as TextProgress, + { text: { bookNum: 42 }, percentage: 70 } as TextProgress, + { text: { bookNum: 70 }, percentage: 100 } as TextProgress + ]); + fixture = TestBed.createComponent(BookMultiSelectComponent); component = fixture.componentInstance; component.availableBooks = mockBooks; @@ -24,48 +45,46 @@ describe('BookMultiSelectComponent', () => { fixture.detectChanges(); }); - it('should initialize book options on ngOnChanges', () => { - const mockBookOptions: BookOption[] = [ - { bookNum: 1, bookId: 'GEN', selected: true }, - { bookNum: 2, bookId: 'EXO', selected: false }, - { bookNum: 3, bookId: 'LEV', selected: true }, - { bookNum: 42, bookId: 'LUK', selected: false }, - { bookNum: 70, bookId: 'WIS', selected: false } - ]; - - component.ngOnChanges(); + it('should initialize book options on ngOnChanges', async () => { + await component.ngOnChanges(); - expect(component.bookOptions).toEqual(mockBookOptions); + expect(component.bookOptions).toEqual([ + { bookNum: 1, bookId: 'GEN', selected: true, progressPercentage: 0 }, + { bookNum: 2, bookId: 'EXO', selected: false, progressPercentage: 20 }, + { bookNum: 3, bookId: 'LEV', selected: true, progressPercentage: 40 }, + { bookNum: 42, bookId: 'LUK', selected: false, progressPercentage: 70 }, + { bookNum: 70, bookId: 'WIS', selected: false, progressPercentage: 100 } + ]); }); - it('can select all OT books', () => { + it('can select all OT books', async () => { expect(component.selectedBooks.length).toEqual(2); - component.select('OT'); + await component.select('OT'); expect(component.selectedBooks.length).toEqual(3); }); - it('can select all NT books', () => { + it('can select all NT books', async () => { expect(component.selectedBooks.length).toEqual(2); - component.select('NT'); + await component.select('NT'); expect(component.selectedBooks.length).toEqual(3); }); - it('can select all DC books', () => { + it('can select all DC books', async () => { expect(component.selectedBooks.length).toEqual(2); - component.select('DC'); + await component.select('DC'); expect(component.selectedBooks.length).toEqual(3); }); - it('can reset book selection', () => { + it('can reset book selection', async () => { expect(component.selectedBooks.length).toEqual(2); - component.clear(); + await component.clear(); expect(component.selectedBooks.length).toEqual(0); }); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.ts index 34e0e2bd21..8b7f4e537e 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/book-multi-select/book-multi-select.component.ts @@ -1,13 +1,18 @@ -import { Component, EventEmitter, Input, OnChanges, Output } from '@angular/core'; +import { Component, EventEmitter, Input, OnChanges, OnInit, Output } from '@angular/core'; import { MatChipsModule } from '@angular/material/chips'; +import { ActivatedRoute } from '@angular/router'; import { TranslocoModule } from '@ngneat/transloco'; import { Canon } from '@sillsdev/scripture'; +import { filter, firstValueFrom, map } from 'rxjs'; +import { SubscriptionDisposable } from 'xforge-common/subscription-disposable'; import { UICommonModule } from 'xforge-common/ui-common.module'; +import { ProgressService } from '../progress-service/progress-service'; export interface BookOption { bookNum: number; bookId: string; selected: boolean; + progressPercentage: number; } @Component({ @@ -17,7 +22,7 @@ export interface BookOption { imports: [UICommonModule, MatChipsModule, TranslocoModule], styleUrls: ['./book-multi-select.component.scss'] }) -export class BookMultiSelectComponent implements OnChanges { +export class BookMultiSelectComponent extends SubscriptionDisposable implements OnInit, OnChanges { @Input() availableBooks: number[] = []; @Input() selectedBooks: number[] = []; @Input() readonly: boolean = false; @@ -27,17 +32,34 @@ export class BookMultiSelectComponent implements OnChanges { selection: any = ''; - ngOnChanges(): void { - this.initBookOptions(); + constructor( + private readonly activatedRoute: ActivatedRoute, + private readonly progressService: ProgressService + ) { + super(); } - initBookOptions(): void { + ngOnInit(): void { + this.subscribe(this.activatedRoute.params.pipe(map(params => params['projectId'])), async projectId => { + this.progressService.initialize(projectId); + }); + } + + async ngOnChanges(): Promise { + await this.initBookOptions(); + } + + async initBookOptions(): Promise { const selectedSet = new Set(this.selectedBooks); + await firstValueFrom(this.progressService.isLoaded$.pipe(filter(loaded => loaded))); + const progress = this.progressService.texts; + this.bookOptions = this.availableBooks.map((bookNum: number) => ({ bookNum, bookId: Canon.bookNumberToId(bookNum), - selected: selectedSet.has(bookNum) + selected: selectedSet.has(bookNum), + progressPercentage: progress.find(p => p.text.bookNum === bookNum)!.percentage })); } @@ -50,7 +72,7 @@ export class BookMultiSelectComponent implements OnChanges { this.bookSelect.emit(this.selectedBooks); } - select(eventValue: string): void { + async select(eventValue: string): Promise { if (eventValue === 'OT') { this.selectedBooks.push( ...this.availableBooks.filter(n => Canon.isBookOT(n) && this.selectedBooks.indexOf(n) === -1) @@ -64,14 +86,14 @@ export class BookMultiSelectComponent implements OnChanges { ...this.availableBooks.filter(n => Canon.isBookDC(n) && this.selectedBooks.indexOf(n) === -1) ); } - this.initBookOptions(); + await this.initBookOptions(); this.bookSelect.emit(this.selectedBooks); } - clear(): void { + async clear(): Promise { this.selectedBooks.length = 0; this.selection = undefined; - this.initBookOptions(); + await this.initBookOptions(); this.bookSelect.emit(this.selectedBooks); } diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.spec.ts new file mode 100644 index 0000000000..9d428f7374 --- /dev/null +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.spec.ts @@ -0,0 +1,151 @@ +import { NgZone } from '@angular/core'; +import { fakeAsync, TestBed, tick } from '@angular/core/testing'; +import { createTestProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-test-data'; +import { TextInfo } from 'realtime-server/lib/esm/scriptureforge/models/text-info'; +import { BehaviorSubject } from 'rxjs'; +import { anything, deepEqual, mock, when } from 'ts-mockito'; +import { NoticeService } from 'xforge-common/notice.service'; +import { OnlineStatusService } from 'xforge-common/online-status.service'; +import { TestOnlineStatusModule } from 'xforge-common/test-online-status.module'; +import { TestOnlineStatusService } from 'xforge-common/test-online-status.service'; +import { TestRealtimeModule } from 'xforge-common/test-realtime.module'; +import { configureTestingModule } from 'xforge-common/test-utils'; +import { SFProjectProfileDoc } from '../../core/models/sf-project-profile-doc'; +import { SF_TYPE_REGISTRY } from '../../core/models/sf-type-registry'; +import { TextDocId } from '../../core/models/text-doc'; +import { PermissionsService } from '../../core/permissions.service'; +import { SFProjectService } from '../../core/sf-project.service'; +import { ProgressService } from '../../shared/progress-service/progress-service'; + +const mockSFProjectService = mock(SFProjectService); +const mockNoticeService = mock(NoticeService); +const mockPermissionService = mock(PermissionsService); + +describe('progress service', () => { + configureTestingModule(() => ({ + declarations: [], + imports: [TestOnlineStatusModule.forRoot(), TestRealtimeModule.forRoot(SF_TYPE_REGISTRY)], + providers: [ + { provide: NoticeService, useMock: mockNoticeService }, + { provide: PermissionsService, useMock: mockPermissionService }, + { provide: SFProjectService, useMock: mockSFProjectService }, + { provide: OnlineStatusService, useClass: TestOnlineStatusService } + ] + })); + + it('populates progress and texts on init', fakeAsync(async () => { + const env = new TestEnvironment(100, 50); + await env.service.initialize('project01'); + tick(); + + expect(env.service.overallProgress.translated).toEqual(100); + expect(env.service.overallProgress.blank).toEqual(50); + expect(env.service.overallProgress.total).toEqual(150); + expect(env.service.overallProgress.percentage).toEqual(67); + expect(env.service.texts.length).toBeGreaterThan(0); + let i = 0; + for (const book of env.service.texts) { + expect(book.text.bookNum).toEqual(i++); + expect(book.text.chapters.length).toBeGreaterThan(0); + let j = 0; + for (const chapter of book.text.chapters) { + expect(chapter.number).toEqual(j++); + } + } + })); + + it('can train suggestions', fakeAsync(async () => { + const env = new TestEnvironment(); + await env.service.initialize('project01'); + tick(); + + expect(env.service.canTrainSuggestions).toBeTruthy(); + })); + + it('cannot train suggestions if too few segments', fakeAsync(async () => { + const env = new TestEnvironment(9); + await env.service.initialize('project01'); + tick(); + + expect(env.service.canTrainSuggestions).toBeFalsy(); + })); + + it('cannot train suggestions if no source permission', fakeAsync(async () => { + const env = new TestEnvironment(); + when( + mockPermissionService.canAccessText(deepEqual(new TextDocId('sourceId', anything(), anything(), 'target'))) + ).thenResolve(false); + await env.service.initialize('project01'); + tick(); + + expect(env.service.canTrainSuggestions).toBeFalsy(); + })); +}); + +class TestEnvironment { + readonly ngZone: NgZone = TestBed.inject(NgZone); + readonly service: ProgressService; + + constructor( + private readonly translatedSegments: number = 1000, + private readonly blankSegments: number = 500 + ) { + this.service = TestBed.inject(ProgressService); + + const data = createTestProjectProfile({ + texts: this.createTexts(), + translateConfig: { + translationSuggestionsEnabled: true, + source: { + projectRef: 'sourceId' + } + } + }); + + when(mockPermissionService.canAccessText(anything())).thenResolve(true); + when(mockSFProjectService.getProfile('project01')).thenResolve({ + data, + id: 'project01', + remoteChanges$: new BehaviorSubject([]) + } as unknown as SFProjectProfileDoc); + + this.setUpGetText('sourceId'); + this.setUpGetText('project01'); + } + + private setUpGetText(projectId: string): void { + let translatedSegments = this.translatedSegments; + let blankSegments = this.blankSegments; + when(mockSFProjectService.getText(deepEqual(new TextDocId(projectId, anything(), anything(), 'target')))).thenCall( + () => { + const translated = translatedSegments >= 9 ? 9 : translatedSegments; + translatedSegments -= translated; + const blank = blankSegments >= 5 ? 5 : blankSegments; + blankSegments -= blank; + return { + getSegmentCount: () => { + return { translated, blank }; + }, + getNonEmptyVerses: () => this.createVerses(translated) + }; + } + ); + } + + private createVerses(num: number): string[] { + let count = 0; + return Array.from({ length: num }, () => 'verse' + ++count); + } + + private createTexts(): TextInfo[] { + const texts: TextInfo[] = []; + for (let book = 0; book < 20; book++) { + const chapters = []; + for (let chapter = 0; chapter < 20; chapter++) { + chapters.push({ isValid: true, lastVerse: 1, number: chapter, permissions: {}, hasAudio: false }); + } + texts.push({ bookNum: book, chapters: chapters, hasSource: true, permissions: {} }); + } + return texts; + } +} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.ts new file mode 100644 index 0000000000..06a38d54b9 --- /dev/null +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/shared/progress-service/progress-service.ts @@ -0,0 +1,168 @@ +import { Injectable, OnDestroy } from '@angular/core'; +import { ANY_INDEX, obj } from 'realtime-server/lib/esm/common/utils/obj-path'; +import { SFProject } from 'realtime-server/lib/esm/scriptureforge/models/sf-project'; +import { TextInfo } from 'realtime-server/lib/esm/scriptureforge/models/text-info'; +import { asyncScheduler, filter, Subscription, throttleTime } from 'rxjs'; +import { DataLoadingComponent } from 'xforge-common/data-loading-component'; +import { NoticeService } from 'xforge-common/notice.service'; +import { OnlineStatusService } from 'xforge-common/online-status.service'; +import { SFProjectProfileDoc } from '../../core/models/sf-project-profile-doc'; +import { TextDoc, TextDocId } from '../../core/models/text-doc'; +import { PermissionsService } from '../../core/permissions.service'; +import { SFProjectService } from '../../core/sf-project.service'; + +const TEXT_PATH_TEMPLATE = obj().pathTemplate(p => p.texts[ANY_INDEX]); + +export class Progress { + translated: number = 0; + blank: number = 0; + + get total(): number { + return this.translated + this.blank; + } + + get percentage(): number { + return Math.round((this.translated / this.total) * 100); + } + + reset(): void { + this.translated = 0; + this.blank = 0; + } +} + +export class TextProgress extends Progress { + constructor(public readonly text: TextInfo) { + super(); + } +} + +@Injectable({ + providedIn: 'root' +}) +export class ProgressService extends DataLoadingComponent implements OnDestroy { + readonly overallProgress = new Progress(); + + private _texts?: TextProgress[]; + private projectDoc?: SFProjectProfileDoc; + private projectDataChangesSub?: Subscription; + private _canTrainSuggestions: boolean = false; + + constructor( + readonly noticeService: NoticeService, + private readonly onlineStatusService: OnlineStatusService, + private readonly projectService: SFProjectService, + private readonly permissionsService: PermissionsService + ) { + super(noticeService); + } + + async initialize(projectId: string): Promise { + if (this.projectDoc?.id !== projectId) { + this.projectDoc = await this.projectService.getProfile(projectId); + + // If we are offline, just update the progress with what we have + if (!this.onlineStatusService.isOnline) { + await this.calculateProgress(); + } + + // Update the overview now if we are online, or when we are next online + this.onlineStatusService.online.then(async () => { + await this.calculateProgress(); + }); + + if (this.projectDataChangesSub != null) { + this.projectDataChangesSub.unsubscribe(); + } + this.projectDataChangesSub = this.projectDoc.remoteChanges$ + .pipe( + filter(ops => ops.some(op => TEXT_PATH_TEMPLATE.matches(op.p))), + throttleTime(1000, asyncScheduler, { leading: true, trailing: true }) + ) + .subscribe(async () => { + await this.calculateProgress(); + }); + } + } + + ngOnDestroy(): void { + super.ngOnDestroy(); + if (this.projectDataChangesSub != null) { + this.projectDataChangesSub.unsubscribe(); + } + } + + get texts(): TextProgress[] { + return this._texts ?? []; + } + + // Whether or not we have the minimum number of segment pairs + get canTrainSuggestions(): boolean { + return this._canTrainSuggestions; + } + + private async calculateProgress(): Promise { + this.loadingStarted(); + try { + if (this.projectDoc == null || this.projectDoc.data == null) { + return; + } + this._texts = this.projectDoc.data.texts + .map(t => new TextProgress(t)) + .sort((a, b) => a.text.bookNum - b.text.bookNum); + this.overallProgress.reset(); + const updateTextProgressPromises: Promise[] = []; + for (const book of this.texts) { + updateTextProgressPromises.push(this.updateTextProgress(this.projectDoc, book)); + } + await Promise.all(updateTextProgressPromises); + } finally { + this.loadingFinished(); + } + } + + private async updateTextProgress(project: SFProjectProfileDoc, book: TextProgress): Promise { + // NOTE: This will stop being incremented when the minimum required number of pairs for training is reached + let numTranslatedSegments: number = 0; + for (const chapter of book.text.chapters) { + const textDocId = new TextDocId(project.id, book.text.bookNum, chapter.number, 'target'); + const chapterText: TextDoc = await this.projectService.getText(textDocId); + + // Calculate Segment Count + const { translated, blank } = chapterText.getSegmentCount(); + book.translated += translated; + book.blank += blank; + this.overallProgress.translated += translated; + this.overallProgress.blank += blank; + + // If translation suggestions are enabled, collect the number of segment pairs up to the minimum required + // We don't go any further so we don't load all of the source texts while this is running + if ( + project.data?.translateConfig.translationSuggestionsEnabled && + project.data.translateConfig.source != null && + book.text.hasSource && + !this.canTrainSuggestions + ) { + const sourceId: string = project.data.translateConfig.source.projectRef; + const sourceTextDocId = new TextDocId(sourceId, book.text.bookNum, chapter.number, 'target'); + + // Only retrieve the source text if the user has permission + let sourceNonEmptyVerses: string[] = []; + if (await this.permissionsService.canAccessText(sourceTextDocId)) { + const sourceChapterText: TextDoc = await this.projectService.getText(sourceTextDocId); + sourceNonEmptyVerses = sourceChapterText.getNonEmptyVerses(); + } + + // Get the intersect of the source and target arrays of non-empty verses + // i.e. The verses with data that both texts have in common + const targetNonEmptyVerses: string[] = chapterText.getNonEmptyVerses(); + numTranslatedSegments += targetNonEmptyVerses.filter(item => sourceNonEmptyVerses.includes(item)).length; + // 9 is the minimum number found in testing, but we will use 10 to be safe + if (numTranslatedSegments >= 10) { + this._canTrainSuggestions = true; + return; + } + } + } + } +} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.spec.ts index 33e21ebb84..966cdf7b56 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.spec.ts @@ -1,6 +1,7 @@ import { ComponentFixture, fakeAsync, TestBed, tick } from '@angular/core/testing'; import { By } from '@angular/platform-browser'; import { NoopAnimationsModule } from '@angular/platform-browser/animations'; +import { ActivatedRoute } from '@angular/router'; import { createTestUser } from 'realtime-server/lib/esm/common/models/user-test-data'; import { createTestProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-test-data'; import { BehaviorSubject, of } from 'rxjs'; @@ -16,6 +17,7 @@ import { environment } from '../../../../environments/environment'; import { SFProjectProfileDoc } from '../../../core/models/sf-project-profile-doc'; import { TrainingDataDoc } from '../../../core/models/training-data-doc'; import { SFProjectService } from '../../../core/sf-project.service'; +import { ProgressService, TextProgress } from '../../../shared/progress-service/progress-service'; import { NllbLanguageService } from '../../nllb-language.service'; import { TrainingDataService } from '../training-data/training-data.service'; import { DraftGenerationStepsComponent, DraftGenerationStepsResult } from './draft-generation-steps.component'; @@ -25,11 +27,13 @@ describe('DraftGenerationStepsComponent', () => { let fixture: ComponentFixture; const mockActivatedProjectService = mock(ActivatedProjectService); + const mockActivatedRoute = mock(ActivatedRoute); const mockFeatureFlagService = mock(FeatureFlagService); const mockProjectService = mock(SFProjectService); const mockNllbLanguageService = mock(NllbLanguageService); const mockTrainingDataService = mock(TrainingDataService); const mockUserService = mock(UserService); + const mockProgressService = mock(ProgressService); const mockTargetProjectDoc = { data: createTestProjectProfile({ @@ -84,12 +88,23 @@ describe('DraftGenerationStepsComponent', () => { { provide: NllbLanguageService, useMock: mockNllbLanguageService }, { provide: SFProjectService, useMock: mockProjectService }, { provide: TrainingDataService, useMock: mockTrainingDataService }, - { provide: UserService, useMock: mockUserService } + { provide: UserService, useMock: mockUserService }, + { provide: ProgressService, useMock: mockProgressService }, + { provide: ActivatedRoute, useMock: mockActivatedRoute } ] })); beforeEach(fakeAsync(() => { when(mockUserService.getCurrentUser()).thenResolve(mockUserDoc); + when(mockActivatedRoute.params).thenReturn(of({ projectId: 'project01' })); + when(mockProgressService.isLoaded$).thenReturn(of(true)); + when(mockProgressService.texts).thenReturn([ + { text: { bookNum: 1 } } as TextProgress, + { text: { bookNum: 2 } } as TextProgress, + { text: { bookNum: 3 } } as TextProgress, + { text: { bookNum: 6 } } as TextProgress, + { text: { bookNum: 7 } } as TextProgress + ]); })); describe('alternate training source project', () => { diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.ts index 4f75438c7d..3d7add52bf 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/draft-generation/draft-generation-steps/draft-generation-steps.component.ts @@ -14,6 +14,7 @@ import { UICommonModule } from 'xforge-common/ui-common.module'; import { filterNullish } from 'xforge-common/util/rxjs-util'; import { TrainingDataDoc } from '../../../core/models/training-data-doc'; import { BookMultiSelectComponent } from '../../../shared/book-multi-select/book-multi-select.component'; +import { ProgressService } from '../../../shared/progress-service/progress-service'; import { SharedModule } from '../../../shared/shared.module'; import { NllbLanguageService } from '../../nllb-language.service'; import { DraftSource, DraftSourcesService } from '../draft-sources.service'; @@ -87,7 +88,8 @@ export class DraftGenerationStepsComponent extends SubscriptionDisposable implem private readonly draftSourcesService: DraftSourcesService, readonly featureFlags: FeatureFlagService, private readonly nllbLanguageService: NllbLanguageService, - private readonly trainingDataService: TrainingDataService + private readonly trainingDataService: TrainingDataService, + private readonly progressService: ProgressService ) { super(); } @@ -179,6 +181,8 @@ export class DraftGenerationStepsComponent extends SubscriptionDisposable implem this.activatedProject.projectDoc$.pipe( filterNullish(), tap(async projectDoc => { + this.progressService.initialize(projectDoc.id); + // Query for all training data files in the project this.trainingDataQuery?.dispose(); this.trainingDataQuery = await this.trainingDataService.queryTrainingDataAsync(projectDoc.id); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.html b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.html index 63d1817e18..e75406c177 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.html +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.html @@ -10,10 +10,10 @@ {{ t("progress") }} -
+
- @if (texts != null) { + @if (progressService.texts != null) { - @for (textProgress of texts; track trackTextByBookNum($index, textProgress)) { + @for (textProgress of progressService.texts; track trackTextByBookNum($index, textProgress)) { book {{ getBookName(textProgress.text) }} @@ -72,7 +72,7 @@ {{ trainedSegmentCount }}{{ t("trained_segments") }}
- @if (showCannotTrainEngineMessage || !canTrainSuggestions) { + @if (showCannotTrainEngineMessage || !progressService.canTrainSuggestions) {
{{ showCannotTrainEngineMessage ? t("cannot_train_suggestion_engine") : t("not_enough_verses_translated") diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.spec.ts index 95080160d8..98f75e7305 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.spec.ts @@ -81,7 +81,7 @@ describe('TranslateOverviewComponent', () => { env.wait(); expect(env.progressTitle.textContent).toContain('Progress'); - expect(env.component.texts!.length).toEqual(4); + expect(env.component.progressService.texts!.length).toEqual(4); env.expectContainsTextProgress(0, 'Matthew', '10 of 20 segments'); env.expectContainsTextProgress(1, 'Mark', '10 of 20 segments'); env.expectContainsTextProgress(2, 'Luke', '10 of 20 segments'); @@ -93,7 +93,7 @@ describe('TranslateOverviewComponent', () => { env.wait(); expect(env.progressTitle.textContent).toContain('Progress'); - expect(env.component.texts!.length).toEqual(4); + expect(env.component.progressService.texts!.length).toEqual(4); env.expectContainsTextProgress(0, 'Matthew', '10 of 20 segments'); env.deleteText(40, 1); @@ -219,57 +219,6 @@ describe('TranslateOverviewComponent', () => { expect(env.translationSuggestionsInfoMessage).toBeTruthy(); })); - it('can train suggestions', fakeAsync(() => { - const env = new TestEnvironment(); - env.setupProjectData({ - projectId: 'project01', - sourceProjectId: 'project02', - translationSuggestionsEnabled: true, - allSegmentsBlank: false - }); - env.setupProjectData({ projectId: 'project02', translationSuggestionsEnabled: false, allSegmentsBlank: false }); - env.setupUserData('user01', ['project01', 'project02']); - env.wait(); - - expect(env.component.canTrainSuggestions).toBeTruthy(); - })); - - it('cannot train suggestions', fakeAsync(() => { - const env = new TestEnvironment(); - env.setupProjectData({ - projectId: 'project01', - sourceProjectId: 'project02', - translationSuggestionsEnabled: true, - allSegmentsBlank: true - }); - env.setupProjectData({ projectId: 'project02', translationSuggestionsEnabled: true, allSegmentsBlank: true }); - env.setupUserData('user01', ['project01', 'project02']); - env.wait(); - - expect(env.component.canTrainSuggestions).toBeFalsy(); - })); - - it('cannot train suggestions if no source permission', fakeAsync(() => { - const env = new TestEnvironment(); - env.setupProjectData({ - projectId: 'project01', - sourceProjectId: 'project02', - translationSuggestionsEnabled: true, - allSegmentsBlank: false, - textPermission: TextInfoPermission.Write - }); - env.setupProjectData({ - projectId: 'project02', - translationSuggestionsEnabled: true, - allSegmentsBlank: false, - textPermission: TextInfoPermission.None - }); - env.setupUserData('user01', ['project01']); - env.wait(); - - expect(env.component.canTrainSuggestions).toBeFalsy(); - })); - it('retrain should be disabled if offline', fakeAsync(() => { const env = new TestEnvironment(); env.wait(); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.ts index 84c527ea17..e949584c73 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/translate/translate-overview/translate-overview.component.ts @@ -9,7 +9,7 @@ import { SFProject } from 'realtime-server/lib/esm/scriptureforge/models/sf-proj import { SF_PROJECT_RIGHTS, SFProjectDomain } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-rights'; import { isParatextRole } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-role'; import { TextInfo } from 'realtime-server/lib/esm/scriptureforge/models/text-info'; -import { asyncScheduler, Subscription, timer } from 'rxjs'; +import { asyncScheduler, firstValueFrom, Subscription, timer } from 'rxjs'; import { delayWhen, filter, map, repeat, retryWhen, tap, throttleTime } from 'rxjs/operators'; import { AuthService } from 'xforge-common/auth.service'; import { DataLoadingComponent } from 'xforge-common/data-loading-component'; @@ -18,42 +18,20 @@ import { NoticeService } from 'xforge-common/notice.service'; import { OnlineStatusService } from 'xforge-common/online-status.service'; import { UserService } from 'xforge-common/user.service'; import { SFProjectProfileDoc } from '../../core/models/sf-project-profile-doc'; -import { TextDoc, TextDocId } from '../../core/models/text-doc'; -import { PermissionsService } from '../../core/permissions.service'; import { SFProjectService } from '../../core/sf-project.service'; import { TranslationEngineService } from '../../core/translation-engine.service'; import { RemoteTranslationEngine } from '../../machine-api/remote-translation-engine'; +import { ProgressService, TextProgress } from '../../shared/progress-service/progress-service'; const ENGINE_QUALITY_STAR_COUNT = 3; const TEXT_PATH_TEMPLATE = obj().pathTemplate(p => p.texts[ANY_INDEX]); -class Progress { - translated: number = 0; - blank: number = 0; - - get total(): number { - return this.translated + this.blank; - } - - get percentage(): number { - return Math.round((this.translated / this.total) * 100); - } -} - -class TextProgress extends Progress { - constructor(public readonly text: TextInfo) { - super(); - } -} - @Component({ selector: 'app-translate-overview', templateUrl: './translate-overview.component.html', styleUrls: ['./translate-overview.component.scss'] }) export class TranslateOverviewComponent extends DataLoadingComponent implements OnInit, OnDestroy { - texts?: TextProgress[]; - overallProgress = new Progress(); trainingPercentage: number = 0; isTraining: boolean = false; readonly engineQualityStars: number[]; @@ -65,8 +43,6 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements private translationEngine?: RemoteTranslationEngine; private projectDoc?: SFProjectProfileDoc; private projectDataChangesSub?: Subscription; - // NOTE: This will stop being incremented when the minimum required number of pairs for training is reached - private segmentPairs: number = 0; constructor( private readonly activatedRoute: ActivatedRoute, @@ -76,7 +52,7 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements private readonly projectService: SFProjectService, private readonly translationEngineService: TranslationEngineService, private readonly userService: UserService, - private readonly permissionsService: PermissionsService, + public readonly progressService: ProgressService, readonly i18n: I18nService ) { super(noticeService); @@ -87,17 +63,7 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements } get translationSuggestionsEnabled(): boolean { - return ( - this.projectDoc != null && - this.projectDoc.data != null && - this.projectDoc.data.translateConfig.translationSuggestionsEnabled - ); - } - - // Whether or not we have the minimum number of segment pairs - get canTrainSuggestions(): boolean { - // 9 is the minimum number found in testing, but we will use 10 to be safe - return this.segmentPairs >= 10; + return this.projectDoc?.data?.translateConfig.translationSuggestionsEnabled ?? false; } get canEditTexts(): boolean { @@ -127,16 +93,7 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements ngOnInit(): void { this.subscribe(this.activatedRoute.params.pipe(map(params => params['projectId'])), async projectId => { this.projectDoc = await this.projectService.getProfile(projectId); - - // If we are offline, just update the progress with what we have - if (!this.isOnline) { - this.loadingStarted(); - try { - this.calculateProgress(); - } finally { - this.loadingFinished(); - } - } + this.progressService.initialize(projectId); // Update the overview now if we are online, or when we are next online this.onlineStatusService.online.then(async () => { @@ -145,7 +102,8 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements if (this.translationEngine == null) { this.setupTranslationEngine(); } - await Promise.all([this.calculateProgress(), this.updateEngineStats()]); + await this.updateEngineStats(); + await firstValueFrom(this.progressService.isLoaded$.pipe(filter(loaded => loaded))); } finally { this.loadingFinished(); } @@ -168,7 +126,8 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements .subscribe(async () => { this.loadingStarted(); try { - await Promise.all([this.calculateProgress(), this.updateEngineStats()]); + await this.updateEngineStats(); + await firstValueFrom(this.progressService.isLoaded$.pipe(filter(loaded => loaded))); } finally { this.loadingFinished(); } @@ -178,9 +137,6 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements ngOnDestroy(): void { super.ngOnDestroy(); - if (this.projectDataChangesSub != null) { - this.projectDataChangesSub.unsubscribe(); - } if (this.trainingSub != null) { this.trainingSub.unsubscribe(); } @@ -221,59 +177,6 @@ export class TranslateOverviewComponent extends DataLoadingComponent implements return isParatextRole(this.projectDoc?.data?.userRoles[this.userService.currentUserId]); } - private async calculateProgress(): Promise { - if (this.projectDoc == null || this.projectDoc.data == null) { - return; - } - this.texts = this.projectDoc.data.texts - .map(t => new TextProgress(t)) - .sort((a, b) => a.text.bookNum - b.text.bookNum); - this.overallProgress = new Progress(); - const updateTextProgressPromises: Promise[] = []; - for (const textProgress of this.texts) { - updateTextProgressPromises.push(this.updateTextProgress(this.projectDoc, textProgress)); - } - await Promise.all(updateTextProgressPromises); - } - - private async updateTextProgress(project: SFProjectProfileDoc, textProgress: TextProgress): Promise { - for (const chapter of textProgress.text.chapters) { - const textDocId = new TextDocId(project.id, textProgress.text.bookNum, chapter.number, 'target'); - const chapterText: TextDoc = await this.projectService.getText(textDocId); - - // Calculate Segment Count - const { translated, blank } = chapterText.getSegmentCount(); - textProgress.translated += translated; - textProgress.blank += blank; - this.overallProgress.translated += translated; - this.overallProgress.blank += blank; - - // If translation suggestions are enabled, collect the number of segment pairs up to the minimum required - // We don't go any further so we don't load all of the source texts while this is running - if ( - project.data?.translateConfig.translationSuggestionsEnabled && - project.data.translateConfig.source != null && - textProgress.text.hasSource && - !this.canTrainSuggestions - ) { - const sourceId: string = project.data.translateConfig.source.projectRef; - const sourceTextDocId = new TextDocId(sourceId, textProgress.text.bookNum, chapter.number, 'target'); - - // Only retrieve the source text if the user has permission - let sourceNonEmptyVerses: string[] = []; - if (await this.permissionsService.canAccessText(sourceTextDocId)) { - const sourceChapterText: TextDoc = await this.projectService.getText(sourceTextDocId); - sourceNonEmptyVerses = sourceChapterText.getNonEmptyVerses(); - } - - // Get the intersect of the source and target arrays of non-empty verses - // i.e. The verses with data that both texts have in common - const targetNonEmptyVerses: string[] = chapterText.getNonEmptyVerses(); - this.segmentPairs += targetNonEmptyVerses.filter(item => sourceNonEmptyVerses.includes(item)).length; - } - } - } - private setupTranslationEngine(): void { if (this.trainingSub != null) { this.trainingSub.unsubscribe(); diff --git a/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/data-loading-component.ts b/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/data-loading-component.ts index 44875ee172..d371e5a904 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/data-loading-component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/data-loading-component.ts @@ -13,18 +13,23 @@ import { SubscriptionDisposable } from './subscription-disposable'; @Directive() // eslint-disable-next-line @angular-eslint/directive-class-suffix export abstract class DataLoadingComponent extends SubscriptionDisposable implements OnDestroy { - private isLoadingSource$ = new BehaviorSubject(false); + private _isLoading$ = new BehaviorSubject(false); + private _isLoaded$ = new BehaviorSubject(false); constructor(protected readonly noticeService: NoticeService) { super(); } get isLoading$(): Observable { - return this.isLoadingSource$.asObservable(); + return this._isLoading$.asObservable(); } get isLoading(): boolean { - return this.isLoadingSource$.value; + return this._isLoading$.value; + } + + get isLoaded$(): Observable { + return this._isLoaded$.asObservable(); } ngOnDestroy(): void { @@ -35,14 +40,16 @@ export abstract class DataLoadingComponent extends SubscriptionDisposable implem protected loadingStarted(): void { if (!this.isLoading) { this.noticeService.loadingStarted(); - this.isLoadingSource$.next(true); + this._isLoading$.next(true); + this._isLoaded$.next(false); } } protected loadingFinished(): void { if (this.isLoading) { this.noticeService.loadingFinished(); - this.isLoadingSource$.next(false); + this._isLoading$.next(false); + this._isLoaded$.next(true); } } } From 39b110f24339210334df700a51392b8e8295e7a9 Mon Sep 17 00:00:00 2001 From: Nathaniel Paulus Date: Wed, 21 Aug 2024 22:13:11 -0400 Subject: [PATCH 5/6] Add draft configuration information to Serval admin page (#2681) --- .../serval-project.component.html | 14 ++++++++ .../serval-project.component.scss | 4 +++ .../serval-project.component.ts | 33 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html index 000ec1dc1c..8102789f54 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html @@ -55,3 +55,17 @@

Downloads

+ +

Last Draft settings

+

Training books

+{{ trainingBooks.join(", ") || "None" }} +

Training data files

+{{ trainingFiles.join(", ") || "None" }} +

Translation books

+{{ translationBooks.join(", ") || "None" }} + +

Raw draft config

+
+@for (key of keys(draftConfig ?? {}); track key) {{{ key }}: {{ stringify(draftConfig![key]) }}
+}
+
diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss index d184769f22..e8d084c7ea 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss @@ -9,3 +9,7 @@ column-gap: 10px; } } + +.raw-draft-config { + font-family: monospace; +} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts index 876bcc3966..0233763e60 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts @@ -1,5 +1,6 @@ import { CommonModule } from '@angular/common'; import { Component, OnInit } from '@angular/core'; +import { Canon } from '@sillsdev/scripture'; import { saveAs } from 'file-saver'; import { SFProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project'; import { catchError, lastValueFrom, of, tap, throwError } from 'rxjs'; @@ -37,6 +38,12 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn columnsToDisplay = ['category', 'name', 'id']; rows: Row[] = []; + trainingBooks: string[] = []; + trainingFiles: string[] = []; + translationBooks: string[] = []; + + draftConfig: Object | undefined; + constructor( private readonly activatedProjectService: ActivatedProjectService, noticeService: NoticeService, @@ -119,6 +126,17 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn // We have to set the rows this way to trigger the update this.rows = rows; + + // Setup the books + this.trainingBooks = project.translateConfig.draftConfig.lastSelectedTrainingBooks.map(bookNum => + Canon.bookNumberToEnglishName(bookNum) + ); + this.trainingFiles = project.translateConfig.draftConfig.lastSelectedTrainingDataFiles; + this.translationBooks = project.translateConfig.draftConfig.lastSelectedTranslationBooks.map(bookNum => + Canon.bookNumberToEnglishName(bookNum) + ); + + this.draftConfig = project.translateConfig.draftConfig; }) ) ); @@ -162,4 +180,19 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn await this.servalAdministrationService.onlineRetrievePreTranslationStatus(this.activatedProjectService.projectId!); await this.noticeService.show('Webhook job started.'); } + + keys(obj: Object): string[] { + return Object.keys(obj); + } + + stringify(value: any): string { + if (Array.isArray(value)) { + return this.arrayToString(value); + } + return JSON.stringify(value, (_key, value) => (Array.isArray(value) ? this.arrayToString(value) : value), 2); + } + + arrayToString(value: any): string { + return '[' + value.join(', ') + ']'; + } } From 7e91fbc4a6aa52fed8a97c07862cae51d18f7d0e Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Fri, 23 Aug 2024 01:59:44 +1200 Subject: [PATCH 6/6] SF-2908 Fix crash when syncing specific Paratext Resources (#2669) --- .../Services/ParatextService.cs | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/src/SIL.XForge.Scripture/Services/ParatextService.cs b/src/SIL.XForge.Scripture/Services/ParatextService.cs index fca4114a9a..d57eb0c0c5 100644 --- a/src/SIL.XForge.Scripture/Services/ParatextService.cs +++ b/src/SIL.XForge.Scripture/Services/ParatextService.cs @@ -27,6 +27,7 @@ using Paratext.Data; using Paratext.Data.Languages; using Paratext.Data.ProjectComments; +using Paratext.Data.ProjectFileAccess; using Paratext.Data.ProjectSettingsAccess; using Paratext.Data.RegistryServerAccess; using Paratext.Data.Repository; @@ -2355,6 +2356,13 @@ bool needsToBeCloned { if (resource.InstallableResource != null) { + // Correct the language code for old resources + LanguageId? overrideLanguageId = null; + if (DetermineBestLanguageForResource(resource.InstallableResource.ExistingScrText)) + { + overrideLanguageId = resource.InstallableResource.ExistingScrText?.Settings.LanguageID; + } + // Install the resource if it is missing or out of date if ( !resource.IsInstalled @@ -2364,6 +2372,12 @@ bool needsToBeCloned { resource.InstallableResource.Install(); needsToBeCloned = true; + + // On first install, we will now have an existing ScrText, so check the language is OK + if (DetermineBestLanguageForResource(resource.InstallableResource.ExistingScrText)) + { + overrideLanguageId = resource.InstallableResource.ExistingScrText?.Settings.LanguageID; + } } // Extract the resource to the source directory @@ -2372,7 +2386,7 @@ bool needsToBeCloned string path = LocalProjectDir(targetParatextId); _fileSystemService.CreateDirectory(path); resource.InstallableResource.ExtractToDirectory(path); - MigrateResourceIfRequired(username, targetParatextId); + MigrateResourceIfRequired(username, targetParatextId, overrideLanguageId); } } else @@ -2381,13 +2395,58 @@ bool needsToBeCloned } } + /// + /// Determines the best language for a resource project + /// + /// The scripture text for the resource. + /// true if the project language was overridden by the DBL; otherwise, false. + /// + /// + /// This is reimplemented from Paratext.Migration.MigrateLanguage.DetermineBestLangIdToUseForResource(). + /// + /// + /// Because resources are not written to (as they are readonly), this should be run before using the LanguageID. + /// + /// + private static bool DetermineBestLanguageForResource(ScrText? scrText) + { + // If we do not have a ScrText, or this is not a resource, do not determine the language + if (scrText is null || !scrText.IsResourceProject) + { + return false; + } + + // Get the language identifier from the .SSF file + string? languageIdLDML = scrText.Settings.LanguageID?.Id; + + // Get the language identifier embedded in the .P8Z folder structure: .dbl\language\iso + string languageIdDBL = ((ZippedProjectFileManagerBase)scrText.FileManager).DBLResourceSettings.LanguageIso639_3; + LanguageId langIdDBL = LanguageId.FromEthnologueCode(languageIdDBL); + if (string.IsNullOrEmpty(languageIdLDML)) + { + scrText.Settings.LanguageID = langIdDBL; + return true; + } + + LanguageId langIdLDML = LanguageId.FromEthnologueCode(languageIdLDML); + if (langIdLDML.Code == langIdDBL.Code) + { + scrText.Settings.LanguageID = langIdLDML; + return false; + } + + scrText.Settings.LanguageID = langIdDBL; + return true; + } + /// /// Migrates a Paratext Resource, if required. /// /// The username. /// The paratext project identifier. + /// The language to override, if the project's language is incorrect. /// This only performs one basic migration. Full migration can only be performed by Paratext. - private void MigrateResourceIfRequired(string username, string paratextId) + private void MigrateResourceIfRequired(string username, string paratextId, LanguageId? overrideLanguage) { // Ensure that we have the ScrText to migrate using ScrText scrText = ScrTextCollection.FindById(username, paratextId); @@ -2396,6 +2455,15 @@ private void MigrateResourceIfRequired(string username, string paratextId) return; } + // Migrate the language id if it is missing. It will be missing as the project has changed from a resource (p8z) + // to a project (directory based), and we did not write to the p8z file as Paratext does in its migrators. + // The ScrText created above will not have the values defined in DetermineBestLanguageForResource() above, so + // we will need to override them again before migrating the LDML (an action which requires the LanguageID). + if (overrideLanguage is not null) + { + scrText.Settings.LanguageID = overrideLanguage; + } + // Perform a simple migration of the Paratext 7 LDML file to the new Paratext 8+ location. // Paratext performs a much more complex migration, but we do not need that level of detail. // If the publisher updates this resource, this file will be overwritten with the fully migrated language file,