AVPRIndex: Fix line endings mattering for package content hash

- AVPRIndex: normalize line endings  to \n before creating checksum for indexed packages
AVPRIndex: unify some creat functions
- add tests

src/AVPRIndex/AVPRIndex.fsproj

@@ -16,7 +16,7 @@
     <RepositoryType>git</RepositoryType>
     <PackageReleaseNotes>$([System.IO.File]::ReadAllText("$(MSBuildProjectDirectory)/RELEASE_NOTES.md"))</PackageReleaseNotes>
     <PackageReadmeFile>README.md</PackageReadmeFile>
-    <PackageVersion>0.0.7</PackageVersion>
+    <PackageVersion>0.0.8</PackageVersion>
   </PropertyGroup>
 
   <ItemGroup>

src/AVPRIndex/Domain.fs

@@ -2,6 +2,7 @@
 
 open System
 open System.IO
+open System.Text
 open System.Text.Json
 open System.Security.Cryptography
 
@@ -45,15 +46,12 @@ module Domain =
             ?affiliation: string,
             ?affiliationLink: string
         ) =
-            let tmp = new Author()
-
-            tmp.FullName <- fullName
-            if email.IsSome then
-                tmp.Email <- email.Value
-            if affiliation.IsSome then
-                tmp.Affiliation <- affiliation.Value
-            if affiliationLink.IsSome then
-                tmp.AffiliationLink <- affiliationLink.Value
+            let tmp = Author(
+                FullName = fullName
+            )
+            email |> Option.iter (fun x -> tmp.Email <- x)
+            affiliation |> Option.iter (fun x -> tmp.Affiliation <- x)
+            affiliationLink |> Option.iter (fun x -> tmp.AffiliationLink <- x)
 
             tmp
 
@@ -89,14 +87,9 @@ module Domain =
             ?termSourceRef: string,
             ?termAccessionNumber: string
         ) =
-            let tmp = new OntologyAnnotation()
-
-            tmp.Name <- name
-            if termSourceRef.IsSome then
-                tmp.TermSourceREF <- termSourceRef.Value
-            if termAccessionNumber.IsSome then
-                tmp.TermAccessionNumber <- termAccessionNumber.Value
-
+            let tmp = new OntologyAnnotation(Name = name)
+            termSourceRef |> Option.iter (fun x -> tmp.TermSourceREF <- x)
+            termAccessionNumber |> Option.iter (fun x -> tmp.TermAccessionNumber <- x)
             tmp
 
     type ValidationPackageMetadata() =
@@ -167,13 +160,15 @@ module Domain =
             ?Tags: OntologyAnnotation [],
             ?ReleaseNotes
         ) = 
-            let tmp = ValidationPackageMetadata()
-            tmp.Name <- name
-            tmp.Summary <- summary
-            tmp.Description <- description
-            tmp.MajorVersion <- majorVersion
-            tmp.MinorVersion <- minorVersion
-            tmp.PatchVersion <- patchVersion
+            let tmp = ValidationPackageMetadata(
+                Name = name,
+                Summary = summary,
+                Description = description,
+                MajorVersion = majorVersion,
+                MinorVersion = minorVersion,
+                PatchVersion = patchVersion
+            )
+
             Publish |> Option.iter (fun x -> tmp.Publish <- x)
             Authors |> Option.iter (fun x -> tmp.Authors <- x)
             Tags |> Option.iter (fun x -> tmp.Tags <- x)
@@ -215,9 +210,16 @@ module Domain =
 
                 ValidationPackageIndex.create(
                     repoPath = repoPath,
-                    fileName = Path.GetFileNameWithoutExtension(repoPath),
+                    fileName = Path.GetFileName(repoPath),
                     lastUpdated = lastUpdated,
-                    contentHash = (md5.ComputeHash(File.ReadAllBytes(repoPath)) |> Convert.ToHexString),
+                    contentHash = (
+                        repoPath
+                        |> File.ReadAllText
+                        |> fun s -> s.ReplaceLineEndings("\n")
+                        |> Encoding.UTF8.GetBytes
+                        |> md5.ComputeHash
+                        |> Convert.ToHexString
+                    ),
                     metadata = metadata
                 )
 

src/AVPRIndex/Frontmatter.fs

@@ -2,12 +2,13 @@
 
 open Domain
 open System
+open System.Text
 open System.IO
 open System.Security.Cryptography
 open YamlDotNet.Serialization
 
 module Frontmatter = 
-   
+
     let [<Literal>] frontMatterStart = "(*\n---"
     let [<Literal>] frontMatterEnd = "---\n*)"
 
@@ -71,13 +72,8 @@ module Frontmatter =
             repoPath: string, 
             lastUpdated: System.DateTimeOffset
         ) = 
-
-            let md5 = MD5.Create()
-
             ValidationPackageIndex.create(
                 repoPath = repoPath,
-                fileName = Path.GetFileName(repoPath),
                 lastUpdated = lastUpdated,
-                contentHash = (md5.ComputeHash(File.ReadAllBytes(repoPath)) |> Convert.ToHexString),
                 metadata = ValidationPackageMetadata.extractFromScript(repoPath)
             )

src/AVPRIndex/RELEASE_NOTES.md

@@ -1,3 +1,8 @@
+## v0.0.8
+- Fix content hash being dependent on line endings (now, all content is normalized to \n before hashing)
+- Fix code duplication in create functions for `ValidationPackageIndex`
+- Unify `create` functions for Domain types
+
 ## v0.0.7
 
 fix preview index download url

tests/IndexTests/DomainTests.fs

@@ -0,0 +1,41 @@
+namespace DomainTests
+
+open System
+open System.IO
+open Xunit
+open AVPRIndex
+open AVPRIndex.Domain
+open ReferenceObjects
+
+module Author =
+
+    [<Fact>]
+    let ``create function for mandatory fields``() =
+        let actual = Author.create(fullName = "test")
+        Assert.Equivalent(Author.mandatoryFields, actual)
+
+    [<Fact>]
+    let ``create function for all fields``() =
+        let actual = Author.create(
+            fullName = "test", 
+            Email = "[email protected]",
+            Affiliation = "testaffiliation",
+            AffiliationLink = "test.com"
+        )
+        Assert.Equivalent(Author.allFields, actual)
+
+module OntologyAnnotation =
+
+    [<Fact>]
+    let ``create function for mandatory fields``() =
+        let actual = OntologyAnnotation.create(name = "test")
+        Assert.Equivalent(OntologyAnnotation.mandatoryFields, actual)
+
+    [<Fact>]
+    let ``create function for all fields``() =
+        let actual = OntologyAnnotation.create(
+            name = "test",
+            TermSourceREF = "REF",
+            TermAccessionNumber = "TAN"
+        )
+        Assert.Equivalent(OntologyAnnotation.allFields, actual)

tests/IndexTests/IndexTests.fsproj

@@ -13,6 +13,7 @@
     <Content Include="..\..\StagingArea\**" CopyToOutputDirectory="Always" LinkBase="StagingArea" />
     <Compile Include="Utils.fs" />
     <Compile Include="ReferenceObjects.fs" />
+    <Compile Include="DomainTests.fs" />
     <Compile Include="FrontmatterTests.fs" />
     <Compile Include="MetadataTests.fs" />
     <Compile Include="ValidationPackageIndexTests.fs" />

tests/IndexTests/ReferenceObjects.fs

@@ -1,8 +1,33 @@
 module ReferenceObjects
 
+open Utils
 open AVPRIndex
 open AVPRIndex.Domain
 
+let testDate = System.DateTimeOffset.Parse("01/01/2024")
+
+module Author = 
+
+    let mandatoryFields = Author(FullName = "test")
+
+    let allFields =
+        Author(
+            FullName = "test",
+            Email = "[email protected]",
+            Affiliation = "testaffiliation",
+            AffiliationLink = "test.com"
+        )
+
+module OntologyAnnotation = 
+
+    let mandatoryFields = OntologyAnnotation(Name = "test")
+
+    let allFields = OntologyAnnotation(
+        Name = "test",
+        TermSourceREF = "REF",
+        TermAccessionNumber = "TAN"
+    )
+
 module Frontmatter = 
 
     let validMandatoryFrontmatter = """(*
@@ -210,4 +235,33 @@ It does it very fast, it does it very swell.
 It does it very good, it does it very well.
 It does it very fast, it does it very swell.
 """.ReplaceLineEndings("\n")
+        )
+
+module ValidationPackageIndex =
+
+    let validMandatoryFrontmatter = 
+        ValidationPackageIndex.create(
+            repoPath = "fixtures/[email protected]",
+            fileName = "[email protected]",
+            lastUpdated = testDate,
+            contentHash = (Frontmatter.validMandatoryFrontmatter |> md5hash),
+            metadata = Metadata.validMandatoryFrontmatter
+        )
+
+    let validFullFrontmatter = 
+        ValidationPackageIndex.create(
+            repoPath = "fixtures/[email protected]",
+            fileName = "[email protected]",
+            lastUpdated = testDate,
+            contentHash = (Frontmatter.validFullFrontmatter |> md5hash),
+            metadata = Metadata.validFullFrontmatter
+        )
+
+    let invalidMissingMandatoryFrontmatter = 
+        ValidationPackageIndex.create(
+            repoPath = "fixtures/[email protected]",
+            fileName = "[email protected]",
+            lastUpdated = testDate,
+            contentHash = (Frontmatter.invalidMissingMandatoryFrontmatter |> md5hash),
+            metadata = Metadata.invalidMissingMandatoryFrontmatter
         )

tests/IndexTests/Utils.fs

@@ -4,7 +4,16 @@ open AVPRIndex
 open AVPRIndex.Domain
 open Xunit
 open System 
+open System.Text
 open System.IO
+open System.Security.Cryptography
+
+let md5hash (content: string) =
+    let md5 = MD5.Create()
+    content
+    |> Encoding.UTF8.GetBytes
+    |> md5.ComputeHash
+    |> Convert.ToHexString
 
 type Assert with
     static member MetadataValid(m: ValidationPackageMetadata) =

tests/IndexTests/ValidationPackageIndexTests.fs

@@ -1,16 +1,103 @@
 namespace ValidationPackageIndexTests
 
 open System
+open System.IO
 open Xunit
+open AVPRIndex
+open AVPRIndex.Domain
+open AVPRIndex.Frontmatter
+open Utils
+open ReferenceObjects
 
-module InMemory =
+module IO =
+
+    open System.IO
 
     [<Fact>]
-    let ``My test`` () =
-        Assert.True(true)
+    let ``valid indexed package is extracted from valid mandatory field test file`` () =
 
-module IO =
+        let actual = 
+            ValidationPackageIndex.create(
+                repoPath = "fixtures/[email protected]",
+                lastUpdated = testDate
+            )
+        Assert.Equivalent(ValidationPackageIndex.validMandatoryFrontmatter, actual)
+
+
+    [<Fact>]
+    let ``valid indexed package is extracted from all fields test file`` () =
+
+        let actual = 
+            ValidationPackageIndex.create(
+                repoPath = "fixtures/[email protected]",
+                lastUpdated = testDate
+            )
+        Assert.Equivalent(ValidationPackageIndex.validFullFrontmatter, actual)
+
+    [<Fact>]
+    let ``invalid indexed package is extracted from testfile with missing fields`` () =
+
+        let actual = 
+            ValidationPackageIndex.create(
+                repoPath = "fixtures/[email protected]",
+                lastUpdated = testDate
+            )
+        Assert.Equivalent(ValidationPackageIndex.invalidMissingMandatoryFrontmatter, actual)
+
+    [<Fact>]
+    let ``CRLF: correct content hash (with line endings replaced) is extracted from valid mandatory field test file`` () =
+        let tmp_path = Path.GetTempFileName()
+        File.WriteAllText(
+            tmp_path,
+            Frontmatter.validMandatoryFrontmatter.ReplaceLineEndings("\r\n")
+        )
+        let actual = 
+            ValidationPackageIndex.create(
+                repoPath = tmp_path,
+                lastUpdated = testDate
+            )
+        let expected = {
+            ValidationPackageIndex.validMandatoryFrontmatter with 
+                RepoPath = tmp_path
+                FileName = Path.GetFileName(tmp_path)
+        }
+        Assert.Equivalent(expected, actual)
+
+
+    [<Fact>]
+    let ``CRLF: correct content hash (with line endings replaced) is extracted from all fields test file`` () =
+        let tmp_path = Path.GetTempFileName()
+        File.WriteAllText(
+            tmp_path,
+            Frontmatter.validFullFrontmatter.ReplaceLineEndings("\r\n")
+        )
+        let actual = 
+            ValidationPackageIndex.create(
+                repoPath = tmp_path,
+                lastUpdated = testDate
+            )
+        let expected = {
+            ValidationPackageIndex.validFullFrontmatter with 
+                RepoPath = tmp_path
+                FileName = Path.GetFileName(tmp_path)
+        }
+        Assert.Equivalent(expected, actual)
 
     [<Fact>]
-    let ``My test`` () =
-        Assert.True(true)
+    let ```CRLF: correct content hash (with line endings replaced) is extracted from testfile with missing fields`` () =
+        let tmp_path = Path.GetTempFileName()
+        File.WriteAllText(
+            tmp_path,
+            Frontmatter.invalidMissingMandatoryFrontmatter.ReplaceLineEndings("\r\n")
+        )
+        let actual = 
+            ValidationPackageIndex.create(
+                repoPath = tmp_path,
+                lastUpdated = testDate
+            )
+        let expected = {
+            ValidationPackageIndex.invalidMissingMandatoryFrontmatter with 
+                RepoPath = tmp_path
+                FileName = Path.GetFileName(tmp_path)
+        }
+        Assert.Equivalent(expected, actual)