Add more tests, fix path to timestamp algorithm bugs

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v2.0.0-beta.7 - 2023-08-21
+
+- Add more tests, fix path to timestamp algorithm bugs.
+
 ## v2.0.0-beta.6 - 2023-07-21
 
 - Fixed a local time zone bug.

src/Nexus.Sources.StructuredFile/Nexus.Sources.StructuredFile.csproj

@@ -24,6 +24,12 @@
     <None Include="../../README.md" Pack="true" PackagePath="/" />
   </ItemGroup>
 
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleToAttribute">
+      <_Parameter1>Nexus.Sources.StructuredFile.Tests</_Parameter1>
+    </AssemblyAttribute>
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Nexus.Extensibility" Version="2.0.0-beta.8" />
   </ItemGroup>

src/Nexus.Sources.StructuredFile/StructuredFileDataSource.cs

@@ -370,7 +370,7 @@ protected virtual async Task ReadAsync(
                             var filePaths = await FindFilePathsAsync(currentBegin, fileSource);
 
                             // determine file begin
-                            if (!TryGetFileBeginByPath(filePaths.First(), fileSource, out var fileBegin, default))
+                            if (!TryGetFileBeginByPath(filePaths.First(), fileSource, out var fileBegin, folderBegin: default))
                                 throw new Exception($"Unable to determine date/time of file {filePaths.First()}.");
 
                             /* CB = Current Begin, FP = File Period
@@ -721,7 +721,7 @@ async Task IDataSource.ReadAsync(
                 var candidateFiles = filePaths
                     .Select(filePath =>
                     {
-                        var success = TryGetFileBeginByPath(filePath, fileSource, out var fileBegin, currentFolder.DateTime);
+                        var success = TryGetFileBeginByPath(filePath, fileSource, out var fileBegin, folderBegin: currentFolder.DateTime);
 
                         return (success, filePath, fileBegin);
                     })
@@ -860,7 +860,7 @@ out var parsedDateTime
             }
         }
 
-        private static bool TryGetFileBeginByPath(
+        internal static bool TryGetFileBeginByPath(
             string filePath, 
             FileSource fileSource, 
             out DateTime fileBegin, 
@@ -893,39 +893,31 @@ private static bool TryGetFileBeginByPath(
                     // long way
                     else
                     {
-                        var pathSegments = filePath
-                            .Split('/', '\\');
-
-                        pathSegments = pathSegments
-                            .Skip(pathSegments.Length - fileSource.PathSegments.Length)
-                            .ToArray();
-
-                        for (int i = 0; i < pathSegments.Length; i++)
-                        {
-                            var folderName = pathSegments[i];
-                            var folderTemplate = fileSource.PathSegments[i];
-
-                            var _ = DateTime.TryParseExact(
-                                folderName,
-                                folderTemplate,
-                                default,
-                                DateTimeStyles.NoCurrentDateDefault | DateTimeStyles.AdjustToUniversal,
-                                out var currentFolderBegin
-                            );
-
-                            if (currentFolderBegin > folderBegin)
-                                folderBegin = currentFolderBegin;
-                        }
+                        folderBegin = GetFolderBegin(filePath, fileSource);
 
-                        fileBegin = folderBegin;
-                        isSuccess = fileBegin != default;
+                        fileBegin = folderBegin + fileBegin.TimeOfDay;
+                        isSuccess = folderBegin != default;
                     }
                 }
+
                 // default: use folder date/time
                 else
                 {
-                    fileBegin = folderBegin;
-                    isSuccess = fileBegin != default;
+                    // short cut
+                    if (folderBegin != default)
+                    {
+                        fileBegin = folderBegin;
+                        isSuccess = true;
+                    }
+
+                    // long way
+                    else
+                    {
+                        folderBegin = GetFolderBegin(filePath, fileSource);
+
+                        fileBegin = folderBegin;
+                        isSuccess = folderBegin != default;
+                    }
                 }
             }
 
@@ -941,6 +933,38 @@ out var currentFolderBegin
             return isSuccess;
         }
 
+        private static DateTime GetFolderBegin(string filePath, FileSource fileSource)
+        {
+            var folderBegin = default(DateTime);
+
+            var pathSegments = filePath
+                            .Split('/', '\\');
+
+            pathSegments = pathSegments
+                .Skip(pathSegments.Length - fileSource.PathSegments.Length - 1)
+                .Take(fileSource.PathSegments.Length)
+                .ToArray();
+
+            for (int i = 0; i < pathSegments.Length; i++)
+            {
+                var folderName = pathSegments[i];
+                var folderTemplate = fileSource.PathSegments[i];
+
+                var _ = DateTime.TryParseExact(
+                    folderName,
+                    folderTemplate,
+                    default,
+                    DateTimeStyles.NoCurrentDateDefault | DateTimeStyles.AdjustToUniversal,
+                    out var currentFolderBegin
+                );
+
+                if (currentFolderBegin > folderBegin)
+                    folderBegin = currentFolderBegin;
+            }
+
+            return folderBegin;
+        }
+
         private static bool TryGetFileBeginByName(
             string fileName, 
             FileSource fileSource, 

tests/Nexus.Sources.StructuredFile.Tests/DATABASES/G/config.json

@@ -3,7 +3,7 @@
     "default": [{
       "PathSegments": [
         "'DATA'",
-        "'prefix_'dd-MM-yyyy_HH"
+        "'prefix_'dd-MM-yyyy"
       ],
       "FileTemplate": "HH-mm-ss'.dat'",
       "FilePeriod": "00:00:01",

tests/Nexus.Sources.StructuredFile.Tests/StructuredFileDataSourceTests.cs

@@ -4,6 +4,7 @@
 using System.Globalization;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Text.Json;
 using Xunit;
 
 namespace Nexus.Sources.Tests
@@ -105,6 +106,35 @@ public async Task CanProvideAvailability(string root, string beginString, string
             Assert.Equal(expected, actual, precision);
         }
 
+        [Theory]
+        [InlineData("A", "calibrated", "DATA/calibrated/2019-12/2019-12-31/2019-12-31_12-00-00.dat", "2019-12-31T12-00-00Z")]
+        [InlineData("B", "calibrated", "DATA/2019-12/calibrated/2019-12-31/2019-12-31_12-00-00.dat", "2019-12-31T12-00-00Z")]
+        [InlineData("C", "default", "DATA/2019-12-31/__0_2019-12-31_12-00-00_000000.dat", "2019-12-31T12-00-00Z")]
+        [InlineData("D", "position_A", "DATA/position_A/__0_2019-12-31_12-00-00_000000.dat", "2019-12-31T10-00-00Z")]
+        [InlineData("E", "real_time", "DATA/2019-12/prefix_real_time_data_2019-12-31_12-00-00.dat", "2019-12-31T12-00-00Z")]
+        [InlineData("F", "default", "DATA/2019-12/20191231_12_x_0000.dat", "2019-12-31T12-00-00Z")]
+        [InlineData("G", "default", "DATA/prefix_01-01-2020/00-40-22.dat", "2020-01-01T00-40-22Z")]
+        [InlineData("H", "default", "2019-12-31_12-00-00.dat", "2019-12-31T12-00-00Z")]
+        [InlineData("I", "default", "DATA/2019-12-31_23-55-00/data.dat", "2019-12-31T23-55-00Z")]
+        [InlineData("J", "default", "DATA1/2020_01_01.dat", "2020-01-01T00-00-00Z")]
+        public void CanGetFileBeginByPath(string database, string key, string filePath, string expectedFileBeginString)
+        {
+            // Arrange
+            var expectedFileBegin = DateTime.ParseExact(expectedFileBeginString, "yyyy-MM-ddTHH-mm-ssZ", default, DateTimeStyles.AdjustToUniversal);
+            var configFilePath = $"DATABASES/{database}/config.json";
+            var configJson = File.ReadAllText(configFilePath);
+            var config = JsonSerializer.Deserialize<Dictionary<string, Dictionary<string, IReadOnlyList<FileSource>>>>(configJson)!;
+            var fileSource = config["/A/B/C"][key][0];
+            var fullFilePath = Path.Combine($"DATABASES/{database}", filePath);
+
+            // Act
+            var success = StructuredFileDataSource.TryGetFileBeginByPath(fullFilePath, fileSource, out var fileBegin, folderBegin: default);
+
+            // Assert
+            Assert.True(success);
+            Assert.Equal(expectedFileBegin, fileBegin);
+        }
+
         [Theory]
         [InlineData("2020-01-01T00-00-00Z", "2020-01-01T00-00-00Z")]
         [InlineData("2020-01-02T00-00-00Z", "2020-01-01T00-00-00Z")]

version.json

@@ -1,4 +1,4 @@
 {
     "version": "2.0.0",
-    "suffix": "beta.6"
+    "suffix": "beta.7"
 }