From 12e23f016c600e598c513cdbf9279e6626d213e1 Mon Sep 17 00:00:00 2001 From: Kevin Schneider Date: Mon, 18 Mar 2024 15:13:04 +0100 Subject: [PATCH] AVPRIndex: Fix line endings mattering for package content hash - AVPRIndex: normalize line endings to \n before creating checksum for indexed packages AVPRIndex: unify some creat functions - add tests --- src/AVPRIndex/AVPRIndex.fsproj | 2 +- src/AVPRIndex/Domain.fs | 54 +++++----- src/AVPRIndex/Frontmatter.fs | 8 +- src/AVPRIndex/RELEASE_NOTES.md | 5 + tests/IndexTests/DomainTests.fs | 41 ++++++++ tests/IndexTests/IndexTests.fsproj | 1 + tests/IndexTests/ReferenceObjects.fs | 54 ++++++++++ tests/IndexTests/Utils.fs | 9 ++ .../IndexTests/ValidationPackageIndexTests.fs | 99 +++++++++++++++++-- 9 files changed, 234 insertions(+), 39 deletions(-) create mode 100644 tests/IndexTests/DomainTests.fs diff --git a/src/AVPRIndex/AVPRIndex.fsproj b/src/AVPRIndex/AVPRIndex.fsproj index 277a51a..fe833b7 100644 --- a/src/AVPRIndex/AVPRIndex.fsproj +++ b/src/AVPRIndex/AVPRIndex.fsproj @@ -16,7 +16,7 @@ git $([System.IO.File]::ReadAllText("$(MSBuildProjectDirectory)/RELEASE_NOTES.md")) README.md - 0.0.7 + 0.0.8 diff --git a/src/AVPRIndex/Domain.fs b/src/AVPRIndex/Domain.fs index cfb2416..d0c8c74 100644 --- a/src/AVPRIndex/Domain.fs +++ b/src/AVPRIndex/Domain.fs @@ -2,6 +2,7 @@ open System open System.IO +open System.Text open System.Text.Json open System.Security.Cryptography @@ -45,15 +46,12 @@ module Domain = ?affiliation: string, ?affiliationLink: string ) = - let tmp = new Author() - - tmp.FullName <- fullName - if email.IsSome then - tmp.Email <- email.Value - if affiliation.IsSome then - tmp.Affiliation <- affiliation.Value - if affiliationLink.IsSome then - tmp.AffiliationLink <- affiliationLink.Value + let tmp = Author( + FullName = fullName + ) + email |> Option.iter (fun x -> tmp.Email <- x) + affiliation |> Option.iter (fun x -> tmp.Affiliation <- x) + affiliationLink |> Option.iter (fun x -> tmp.AffiliationLink <- x) tmp @@ -89,14 +87,9 @@ module Domain = ?termSourceRef: string, ?termAccessionNumber: string ) = - let tmp = new OntologyAnnotation() - - tmp.Name <- name - if termSourceRef.IsSome then - tmp.TermSourceREF <- termSourceRef.Value - if termAccessionNumber.IsSome then - tmp.TermAccessionNumber <- termAccessionNumber.Value - + let tmp = new OntologyAnnotation(Name = name) + termSourceRef |> Option.iter (fun x -> tmp.TermSourceREF <- x) + termAccessionNumber |> Option.iter (fun x -> tmp.TermAccessionNumber <- x) tmp type ValidationPackageMetadata() = @@ -167,13 +160,15 @@ module Domain = ?Tags: OntologyAnnotation [], ?ReleaseNotes ) = - let tmp = ValidationPackageMetadata() - tmp.Name <- name - tmp.Summary <- summary - tmp.Description <- description - tmp.MajorVersion <- majorVersion - tmp.MinorVersion <- minorVersion - tmp.PatchVersion <- patchVersion + let tmp = ValidationPackageMetadata( + Name = name, + Summary = summary, + Description = description, + MajorVersion = majorVersion, + MinorVersion = minorVersion, + PatchVersion = patchVersion + ) + Publish |> Option.iter (fun x -> tmp.Publish <- x) Authors |> Option.iter (fun x -> tmp.Authors <- x) Tags |> Option.iter (fun x -> tmp.Tags <- x) @@ -215,9 +210,16 @@ module Domain = ValidationPackageIndex.create( repoPath = repoPath, - fileName = Path.GetFileNameWithoutExtension(repoPath), + fileName = Path.GetFileName(repoPath), lastUpdated = lastUpdated, - contentHash = (md5.ComputeHash(File.ReadAllBytes(repoPath)) |> Convert.ToHexString), + contentHash = ( + repoPath + |> File.ReadAllText + |> fun s -> s.ReplaceLineEndings("\n") + |> Encoding.UTF8.GetBytes + |> md5.ComputeHash + |> Convert.ToHexString + ), metadata = metadata ) diff --git a/src/AVPRIndex/Frontmatter.fs b/src/AVPRIndex/Frontmatter.fs index 1ebae4c..9bb570f 100644 --- a/src/AVPRIndex/Frontmatter.fs +++ b/src/AVPRIndex/Frontmatter.fs @@ -2,12 +2,13 @@ open Domain open System +open System.Text open System.IO open System.Security.Cryptography open YamlDotNet.Serialization module Frontmatter = - + let [] frontMatterStart = "(*\n---" let [] frontMatterEnd = "---\n*)" @@ -71,13 +72,8 @@ module Frontmatter = repoPath: string, lastUpdated: System.DateTimeOffset ) = - - let md5 = MD5.Create() - ValidationPackageIndex.create( repoPath = repoPath, - fileName = Path.GetFileName(repoPath), lastUpdated = lastUpdated, - contentHash = (md5.ComputeHash(File.ReadAllBytes(repoPath)) |> Convert.ToHexString), metadata = ValidationPackageMetadata.extractFromScript(repoPath) ) \ No newline at end of file diff --git a/src/AVPRIndex/RELEASE_NOTES.md b/src/AVPRIndex/RELEASE_NOTES.md index 31f2ecc..8b808e3 100644 --- a/src/AVPRIndex/RELEASE_NOTES.md +++ b/src/AVPRIndex/RELEASE_NOTES.md @@ -1,3 +1,8 @@ +## v0.0.8 +- Fix content hash being dependent on line endings (now, all content is normalized to \n before hashing) +- Fix code duplication in create functions for `ValidationPackageIndex` +- Unify `create` functions for Domain types + ## v0.0.7 fix preview index download url diff --git a/tests/IndexTests/DomainTests.fs b/tests/IndexTests/DomainTests.fs new file mode 100644 index 0000000..dce5db8 --- /dev/null +++ b/tests/IndexTests/DomainTests.fs @@ -0,0 +1,41 @@ +namespace DomainTests + +open System +open System.IO +open Xunit +open AVPRIndex +open AVPRIndex.Domain +open ReferenceObjects + +module Author = + + [] + let ``create function for mandatory fields``() = + let actual = Author.create(fullName = "test") + Assert.Equivalent(Author.mandatoryFields, actual) + + [] + let ``create function for all fields``() = + let actual = Author.create( + fullName = "test", + Email = "test@test.test", + Affiliation = "testaffiliation", + AffiliationLink = "test.com" + ) + Assert.Equivalent(Author.allFields, actual) + +module OntologyAnnotation = + + [] + let ``create function for mandatory fields``() = + let actual = OntologyAnnotation.create(name = "test") + Assert.Equivalent(OntologyAnnotation.mandatoryFields, actual) + + [] + let ``create function for all fields``() = + let actual = OntologyAnnotation.create( + name = "test", + TermSourceREF = "REF", + TermAccessionNumber = "TAN" + ) + Assert.Equivalent(OntologyAnnotation.allFields, actual) diff --git a/tests/IndexTests/IndexTests.fsproj b/tests/IndexTests/IndexTests.fsproj index 65d939d..88077ae 100644 --- a/tests/IndexTests/IndexTests.fsproj +++ b/tests/IndexTests/IndexTests.fsproj @@ -13,6 +13,7 @@ + diff --git a/tests/IndexTests/ReferenceObjects.fs b/tests/IndexTests/ReferenceObjects.fs index c048139..ebc9903 100644 --- a/tests/IndexTests/ReferenceObjects.fs +++ b/tests/IndexTests/ReferenceObjects.fs @@ -1,8 +1,33 @@ module ReferenceObjects +open Utils open AVPRIndex open AVPRIndex.Domain +let testDate = System.DateTimeOffset.Parse("01/01/2024") + +module Author = + + let mandatoryFields = Author(FullName = "test") + + let allFields = + Author( + FullName = "test", + Email = "test@test.test", + Affiliation = "testaffiliation", + AffiliationLink = "test.com" + ) + +module OntologyAnnotation = + + let mandatoryFields = OntologyAnnotation(Name = "test") + + let allFields = OntologyAnnotation( + Name = "test", + TermSourceREF = "REF", + TermAccessionNumber = "TAN" + ) + module Frontmatter = let validMandatoryFrontmatter = """(* @@ -210,4 +235,33 @@ It does it very fast, it does it very swell. It does it very good, it does it very well. It does it very fast, it does it very swell. """.ReplaceLineEndings("\n") + ) + +module ValidationPackageIndex = + + let validMandatoryFrontmatter = + ValidationPackageIndex.create( + repoPath = "fixtures/valid@1.0.0.fsx", + fileName = "valid@1.0.0.fsx", + lastUpdated = testDate, + contentHash = (Frontmatter.validMandatoryFrontmatter |> md5hash), + metadata = Metadata.validMandatoryFrontmatter + ) + + let validFullFrontmatter = + ValidationPackageIndex.create( + repoPath = "fixtures/valid@2.0.0.fsx", + fileName = "valid@2.0.0.fsx", + lastUpdated = testDate, + contentHash = (Frontmatter.validFullFrontmatter |> md5hash), + metadata = Metadata.validFullFrontmatter + ) + + let invalidMissingMandatoryFrontmatter = + ValidationPackageIndex.create( + repoPath = "fixtures/invalid@0.0.fsx", + fileName = "invalid@0.0.fsx", + lastUpdated = testDate, + contentHash = (Frontmatter.invalidMissingMandatoryFrontmatter |> md5hash), + metadata = Metadata.invalidMissingMandatoryFrontmatter ) \ No newline at end of file diff --git a/tests/IndexTests/Utils.fs b/tests/IndexTests/Utils.fs index 90572bc..51ec3be 100644 --- a/tests/IndexTests/Utils.fs +++ b/tests/IndexTests/Utils.fs @@ -4,7 +4,16 @@ open AVPRIndex open AVPRIndex.Domain open Xunit open System +open System.Text open System.IO +open System.Security.Cryptography + +let md5hash (content: string) = + let md5 = MD5.Create() + content + |> Encoding.UTF8.GetBytes + |> md5.ComputeHash + |> Convert.ToHexString type Assert with static member MetadataValid(m: ValidationPackageMetadata) = diff --git a/tests/IndexTests/ValidationPackageIndexTests.fs b/tests/IndexTests/ValidationPackageIndexTests.fs index bd84034..6616836 100644 --- a/tests/IndexTests/ValidationPackageIndexTests.fs +++ b/tests/IndexTests/ValidationPackageIndexTests.fs @@ -1,16 +1,103 @@ namespace ValidationPackageIndexTests open System +open System.IO open Xunit +open AVPRIndex +open AVPRIndex.Domain +open AVPRIndex.Frontmatter +open Utils +open ReferenceObjects -module InMemory = +module IO = + + open System.IO [] - let ``My test`` () = - Assert.True(true) + let ``valid indexed package is extracted from valid mandatory field test file`` () = -module IO = + let actual = + ValidationPackageIndex.create( + repoPath = "fixtures/valid@1.0.0.fsx", + lastUpdated = testDate + ) + Assert.Equivalent(ValidationPackageIndex.validMandatoryFrontmatter, actual) + + + [] + let ``valid indexed package is extracted from all fields test file`` () = + + let actual = + ValidationPackageIndex.create( + repoPath = "fixtures/valid@2.0.0.fsx", + lastUpdated = testDate + ) + Assert.Equivalent(ValidationPackageIndex.validFullFrontmatter, actual) + + [] + let ``invalid indexed package is extracted from testfile with missing fields`` () = + + let actual = + ValidationPackageIndex.create( + repoPath = "fixtures/invalid@0.0.fsx", + lastUpdated = testDate + ) + Assert.Equivalent(ValidationPackageIndex.invalidMissingMandatoryFrontmatter, actual) + + [] + let ``CRLF: correct content hash (with line endings replaced) is extracted from valid mandatory field test file`` () = + let tmp_path = Path.GetTempFileName() + File.WriteAllText( + tmp_path, + Frontmatter.validMandatoryFrontmatter.ReplaceLineEndings("\r\n") + ) + let actual = + ValidationPackageIndex.create( + repoPath = tmp_path, + lastUpdated = testDate + ) + let expected = { + ValidationPackageIndex.validMandatoryFrontmatter with + RepoPath = tmp_path + FileName = Path.GetFileName(tmp_path) + } + Assert.Equivalent(expected, actual) + + + [] + let ``CRLF: correct content hash (with line endings replaced) is extracted from all fields test file`` () = + let tmp_path = Path.GetTempFileName() + File.WriteAllText( + tmp_path, + Frontmatter.validFullFrontmatter.ReplaceLineEndings("\r\n") + ) + let actual = + ValidationPackageIndex.create( + repoPath = tmp_path, + lastUpdated = testDate + ) + let expected = { + ValidationPackageIndex.validFullFrontmatter with + RepoPath = tmp_path + FileName = Path.GetFileName(tmp_path) + } + Assert.Equivalent(expected, actual) [] - let ``My test`` () = - Assert.True(true) \ No newline at end of file + let ```CRLF: correct content hash (with line endings replaced) is extracted from testfile with missing fields`` () = + let tmp_path = Path.GetTempFileName() + File.WriteAllText( + tmp_path, + Frontmatter.invalidMissingMandatoryFrontmatter.ReplaceLineEndings("\r\n") + ) + let actual = + ValidationPackageIndex.create( + repoPath = tmp_path, + lastUpdated = testDate + ) + let expected = { + ValidationPackageIndex.invalidMissingMandatoryFrontmatter with + RepoPath = tmp_path + FileName = Path.GetFileName(tmp_path) + } + Assert.Equivalent(expected, actual) \ No newline at end of file