From 0ef6640c6a9e8b423ecd4cb6e442905a981344ee Mon Sep 17 00:00:00 2001 From: tombogle Date: Mon, 1 Apr 2024 15:06:25 -0400 Subject: [PATCH] SP-2337: Added post-save verification to ensure that metadata file has contents and is valid/readable. WIP - TODO (SP-2326, SP-2336): Need similar verification for EAF files. --- SayMore.sln.DotSettings | 1 + src/SayMore/Model/Files/XmlFileSerializer.cs | 71 +++++++++++++------ src/SayMore/ProjectContext.cs | 8 +-- .../Transcription/Model/TierCollection.cs | 2 + src/SayMore/Utilities/ExtensionMethods.cs | 16 +++-- .../model/Files/FileSerializerTests.cs | 4 +- 6 files changed, 67 insertions(+), 35 deletions(-) diff --git a/SayMore.sln.DotSettings b/SayMore.sln.DotSettings index 9fc41e298..47eb8546c 100644 --- a/SayMore.sln.DotSettings +++ b/SayMore.sln.DotSettings @@ -13,6 +13,7 @@ True True True + True True True True diff --git a/src/SayMore/Model/Files/XmlFileSerializer.cs b/src/SayMore/Model/Files/XmlFileSerializer.cs index 468174051..9d96b98a0 100644 --- a/src/SayMore/Model/Files/XmlFileSerializer.cs +++ b/src/SayMore/Model/Files/XmlFileSerializer.cs @@ -10,6 +10,7 @@ using SayMore.Model.Fields; using SIL.Xml; using SayMore.Utilities; +using SIL.Extensions; namespace SayMore.Model.Files { @@ -33,7 +34,7 @@ public XmlFileSerializer(IDictionary xmlFieldSerial } /// ------------------------------------------------------------------------------------ - public void Save(IEnumerable fields, string path, string rootElementName) + public void Save(IEnumerable fields, string path, string rootElementName, bool verify = true) { var giveUpTime = DateTime.Now.AddSeconds(15); @@ -116,11 +117,29 @@ public void Save(IEnumerable fields, string path, string rootElem { GC.Collect(); root.Save(path); + // This is an attempt to put an end to problems like SP-2337, etc (search Jira + // for "0x00"), where an empty XML file is getting created. We have no actual + // evidence that SayMore is actually creating the empty files, but it seems + // improbable that it would be happening to more than 1 or 2 users if it were + // being caused by some other software or malware. I'm wondering if it could be + // caused by a crash or power failure during the process of saving. If so, this + // might not help, but it could force the disk IO buffer to flush or at least + // maybe give us some additional insight as to when this is happening. If this + // does not stop the problem, then maybe we need to start saving a local backup + // before each file is saved and use the backup as a fallback when loading + // (along with a warning to alert the user that something might have gotten + // lost). + if (verify) + { + var xmlRootForVerification = GetXmlDocumentRoot(path).GetXElement(); + if (!root.Nodes().Select(e => e.ToString()).SetEquals(xmlRootForVerification.Nodes().Select(v => v.ToString()))) + throw new IOException("File contents not saved correctly: " + path); + } break; } - catch (IOException e) + catch (Exception e) { - if (e.GetType() != typeof(IOException)) + if (e.GetType() != typeof(IOException) && !(e is XmlException)) throw; // Guarantee that it retries at least once @@ -204,6 +223,31 @@ public void Load(/*TODO: ClearShare.Work work,*/ List fields, str { fields.Clear(); + var root = GetXmlDocumentRoot(path); + + fields.AddRange(root.ChildNodes.Cast() + .Select(node => GetFieldFromNode(node, fileType.GetIsCustomFieldId)) + .Where(fieldInstance => fieldInstance != null)); + + var customFieldList = root.SelectSingleNode(kCustomFieldsElement); + if (customFieldList != null) + { + fields.AddRange(customFieldList.ChildNodes.Cast() + .Select(node => new FieldInstance(kCustomFieldIdPrefix + node.Name, FieldInstance.kStringType, + CleanupLineBreaks(node.InnerText)))); + } + + var additionalFieldList = root.SelectSingleNode(kAdditionalFieldsElement); + if (additionalFieldList != null) + { + fields.AddRange(additionalFieldList.ChildNodes.Cast() + .Select(node => new FieldInstance(kAdditionalFieldIdPrefix + node.Name, FieldInstance.kStringType, + CleanupLineBreaks(node.InnerText)))); + } + } + + private static XmlNode GetXmlDocumentRoot(string path) + { var doc = new XmlDocument(); var giveUpTime = DateTime.Now.AddSeconds(4); @@ -235,26 +279,7 @@ public void Load(/*TODO: ClearShare.Work work,*/ List fields, str } } - var root = doc.ChildNodes[1]; - fields.AddRange(root.ChildNodes.Cast() - .Select(node => GetFieldFromNode(node, fileType.GetIsCustomFieldId)) - .Where(fieldInstance => fieldInstance != null)); - - var customFieldList = root.SelectSingleNode(kCustomFieldsElement); - if (customFieldList != null) - { - fields.AddRange(customFieldList.ChildNodes.Cast() - .Select(node => new FieldInstance(kCustomFieldIdPrefix + node.Name, FieldInstance.kStringType, - CleanupLineBreaks(node.InnerText)))); - } - - var additionalFieldList = root.SelectSingleNode(kAdditionalFieldsElement); - if (additionalFieldList != null) - { - fields.AddRange(additionalFieldList.ChildNodes.Cast() - .Select(node => new FieldInstance(kAdditionalFieldIdPrefix + node.Name, FieldInstance.kStringType, - CleanupLineBreaks(node.InnerText)))); - } + return doc.ChildNodes[1]; } /// ------------------------------------------------------------------------------------ diff --git a/src/SayMore/ProjectContext.cs b/src/SayMore/ProjectContext.cs index 52ba727cf..24558fe97 100644 --- a/src/SayMore/ProjectContext.cs +++ b/src/SayMore/ProjectContext.cs @@ -115,7 +115,7 @@ public static void SetContributorsListToSession(string sessionsFolder) (Func)(fileName => true) : fileName => !Path.GetFileName(fileName).StartsWith(ProjectElement.kMacOsxResourceFilePrefix); var metaFilesList = filesInDir.Where(f => doesNotHaveIllegalPrefix(f) && - f.EndsWith(Settings.Default.MetadataFileExtension)).ToList(); + f.EndsWith(Settings.Default.MetadataFileExtension) && !f.Contains(Settings.Default.OralAnnotationGeneratedFileSuffix)).ToList(); var sessionDoc = LoadXmlDocument(sessionFile); LoadContributors(sessionDoc, namesList, nameRolesList, contributorLists); var root = sessionDoc.DocumentElement; @@ -214,10 +214,10 @@ private static void LoadContributors(XmlNode xmlDoc, SortedSet namesList // we need to migrate it. It might also have been edited outside SayMore and have // participants that are not known contributors, even though it has a contributions element. // So add in any participants we don't already have in some form. - var particpantsNode = xmlDoc.SelectSingleNode("//participants"); - if (particpantsNode == null) + var participantsNode = xmlDoc.SelectSingleNode("//participants"); + if (participantsNode == null) return; - foreach (var participant in particpantsNode.InnerText.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) + foreach (var participant in participantsNode.InnerText.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) { var name = participant.Trim(); diff --git a/src/SayMore/Transcription/Model/TierCollection.cs b/src/SayMore/Transcription/Model/TierCollection.cs index 07c262306..8479505b2 100644 --- a/src/SayMore/Transcription/Model/TierCollection.cs +++ b/src/SayMore/Transcription/Model/TierCollection.cs @@ -286,6 +286,8 @@ public string Save() public string Save(string annotatedMediaFile) { return AnnotationFileHelper.Save(annotatedMediaFile, this); + // TODO (SP-2326, SP-2336, etc.): Verify that saved EAF file has contents and is + // valid/readable. } #endregion diff --git a/src/SayMore/Utilities/ExtensionMethods.cs b/src/SayMore/Utilities/ExtensionMethods.cs index 6e3cc5d35..6532e00f5 100644 --- a/src/SayMore/Utilities/ExtensionMethods.cs +++ b/src/SayMore/Utilities/ExtensionMethods.cs @@ -1,6 +1,8 @@ using System.Reflection; using System.Windows.Forms; +using System.Xml; +using System.Xml.Linq; using ComboBox = System.Windows.Forms.ComboBox; using TextBox = System.Windows.Forms.TextBox; using DatePicker = SayMore.UI.LowLevelControls.DatePicker; @@ -9,12 +11,6 @@ namespace SayMore.Utilities { internal static class ExtensionMethods { - public static MethodInfo HasMethod(this object objectToCheck, string methodName) - { - var type = objectToCheck.GetType(); - return type.GetMethod(methodName, BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); - } - public static bool IsValidBirthYear(this string birthYear) { var val = birthYear.Trim(); @@ -70,5 +66,13 @@ public static bool IsMotionPicture(this SIL.Media.MediaInfo silMediaInfo) silMediaInfo.AnalysisData.PrimaryVideoStream.AvgFrameRate > 0 || silMediaInfo.AnalysisData.PrimaryVideoStream.FrameRate > 0); } + + public static XElement GetXElement(this XmlNode node) + { + XDocument xDoc = new XDocument(); + using (XmlWriter xmlWriter = xDoc.CreateWriter()) + node.WriteTo(xmlWriter); + return xDoc.Root; + } } } diff --git a/src/SayMoreTests/model/Files/FileSerializerTests.cs b/src/SayMoreTests/model/Files/FileSerializerTests.cs index 983e91162..1b6c77e48 100644 --- a/src/SayMoreTests/model/Files/FileSerializerTests.cs +++ b/src/SayMoreTests/model/Files/FileSerializerTests.cs @@ -224,7 +224,7 @@ public void Load_FileDoesNotExist_Throws() public void Save_DirectoryNotFound_Throws() { Assert.Throws(() => - _serializer.Save(_fields, _parentFolder.Combine("notthere", "test.txt"), "x")); + _serializer.Save(_fields, _parentFolder.Combine("notthere", "test.txt"), "x", false)); } /// ------------------------------------------------------------------------------------ @@ -353,7 +353,7 @@ private void DoRoundTrip() /// ------------------------------------------------------------------------------------ private void SaveToStandardPlace() { - _serializer.Save(_fields, _parentFolder.Combine("test.txt"), "x"); + _serializer.Save(_fields, _parentFolder.Combine("test.txt"), "x", false); } /// ------------------------------------------------------------------------------------