diff --git a/README.md b/README.md index 4e97a3c8..32af1427 100644 --- a/README.md +++ b/README.md @@ -199,17 +199,20 @@ This rule includes all files within a given folder. "type": "folder", "folder": "Ownership/Alternate", "recursive": true, + "pattern": "SomeRegex", "exclude": [ "EUR Islands.txt" ] } ``` -There are two optional flags available for the folder rule: +There are three optional flags available for the folder rule: - `recursive` (default: `false`) will cause the compiler to include all files in any subfolders contained within the main folder. -- `exclude` will cause the compiler to ignore any files with a particular name. +- `exclude` will cause the compiler to ignore any files with a particular name. Conversely, specifying `include` +will only include files with a certain name. +- `pattern` allows you to provide a regular expression, with only files matching the pattern being included. ## Comment Annotations diff --git a/src/Compiler/Config/ConfigIncludeLoader.cs b/src/Compiler/Config/ConfigIncludeLoader.cs index b01f9e97..7ca1a48a 100644 --- a/src/Compiler/Config/ConfigIncludeLoader.cs +++ b/src/Compiler/Config/ConfigIncludeLoader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text.RegularExpressions; using Compiler.Exception; using Compiler.Input; using Compiler.Output; @@ -19,14 +20,14 @@ string fileName JToken airportData = jsonConfig.SelectToken("includes.airports"); if (airportData != null) { - this.IterateAirportConfig(airportData, inclusionRules, fileName); + IterateAirportConfig(airportData, inclusionRules, fileName); } // Load enroute data JToken enrouteData = jsonConfig.SelectToken("includes.enroute"); if (enrouteData != null) { - this.IterateConfigFileSections( + IterateConfigFileSections( enrouteData, EnrouteConfigFileSections.ConfigFileSections, OutputGroupFactory.CreateEnroute, @@ -40,7 +41,7 @@ string fileName JToken miscData = jsonConfig.SelectToken("includes.misc"); if (miscData != null) { - this.IterateConfigFileSections( + IterateConfigFileSections( miscData, MiscConfigFileSections.ConfigFileSections, OutputGroupFactory.CreateMisc, @@ -65,6 +66,11 @@ private string GetInvalidFolderMessage(string section) { return $"Folder invalid in section {section} - must be string under key \"folder\""; } + + private string GetInvalidPatternMessage(string section) + { + return $"Pattern invalid in section {section} - must be a regular expression string"; + } private string GetRecursiveMessage(string section) { @@ -134,13 +140,13 @@ string configFilePath } // Get the airport folders - string configFileFolder = this.GetFolderForConfigFile(configFilePath); + string configFileFolder = GetFolderForConfigFile(configFilePath); string[] directories = Directory.GetDirectories(configFileFolder + Path.DirectorySeparatorChar + configItem.Key); // For each airport, iterate the config file sections foreach (string directory in directories) { - this.IterateConfigFileSections( + IterateConfigFileSections( configItem.Value, AirfieldConfigFileSections.ConfigFileSections, x => OutputGroupFactory.CreateAirport(x, Path.GetFileName(directory)), @@ -168,7 +174,7 @@ string sectionRootString continue; } - this.LoadConfigSection( + LoadConfigSection( configObjectSection, configSection, createOutputGroup(configSection), @@ -194,7 +200,7 @@ string sectionRootString { foreach (JToken token in (JArray) jsonConfig) { - this.ProcessConfigSectionObject( + ProcessConfigSectionObject( token, configFileSection, outputGroup, @@ -204,7 +210,7 @@ string sectionRootString ); } } else { - this.ProcessConfigSectionObject( + ProcessConfigSectionObject( jsonConfig, configFileSection, outputGroup, @@ -243,7 +249,7 @@ string sectionRootString if ((string)typeToken == "files") { - this.ProcessFilesList( + ProcessFilesList( configObject, configFileSection, outputGroup, @@ -253,7 +259,7 @@ string sectionRootString ); } else { - this.ProcessFolder( + ProcessFolder( configObject, configFileSection, outputGroup, @@ -276,9 +282,8 @@ private void ProcessFolder( string sectionRootString ) { // Get the folder - JToken folder; if ( - !folderObject.TryGetValue("folder", out folder) || + !folderObject.TryGetValue("folder", out JToken folder) || folder.Type != JTokenType.String ) { throw new ConfigFileInvalidException( @@ -300,6 +305,28 @@ string sectionRootString recursive = (bool)recursiveToken; } + + // Handle inclusion patterns + Regex patternRegex = null; + if (folderObject.TryGetValue("pattern", out JToken pattern)) { + if (pattern.Type != JTokenType.String) + { + throw new ConfigFileInvalidException( + GetInvalidPatternMessage($"{sectionRootString}.{configFileSection.JsonPath}") + ); + } + + try + { + patternRegex = new Regex(pattern.ToString()); + } + catch (ArgumentException) + { + throw new ConfigFileInvalidException( + GetInvalidPatternMessage($"{sectionRootString}.{configFileSection.JsonPath}") + ); + } + } // Get the include and exclude lists and check both aren't there @@ -317,10 +344,11 @@ string sectionRootString if (!isInclude && !isExclude) { addInclusionRule( new FolderInclusionRule( - this.NormaliseFilePath(rootPath, (string)folder), + NormaliseFilePath(rootPath, (string)folder), recursive, configFileSection.DataType, - outputGroup + outputGroup, + includePattern: patternRegex ) ); return; @@ -350,12 +378,13 @@ string sectionRootString addInclusionRule( new FolderInclusionRule( - this.NormaliseFilePath(rootPath, (string)folder), + NormaliseFilePath(rootPath, (string)folder), recursive, configFileSection.DataType, outputGroup, isExclude, - files + files, + patternRegex ) ); } @@ -408,7 +437,7 @@ string sectionRootString ); } - exceptWhereExists = this.NormaliseFilePath(rootPath, (string) exceptWhereExistsToken); + exceptWhereExists = NormaliseFilePath(rootPath, (string) exceptWhereExistsToken); } // Get the file paths and normalise against the config files folder @@ -422,7 +451,7 @@ string sectionRootString ); } - filePaths.Add(this.NormaliseFilePath(rootPath, (string)file)); + filePaths.Add(NormaliseFilePath(rootPath, (string)file)); } // Add the rule diff --git a/src/Compiler/Input/FolderInclusionRule.cs b/src/Compiler/Input/FolderInclusionRule.cs index 5faede45..96f3ee55 100644 --- a/src/Compiler/Input/FolderInclusionRule.cs +++ b/src/Compiler/Input/FolderInclusionRule.cs @@ -2,6 +2,7 @@ using System.IO; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using Compiler.Exception; using Compiler.Output; @@ -15,6 +16,7 @@ public class FolderInclusionRule : IInclusionRule public bool ExcludeList { get; } public List IncludeExcludeFiles { get; } private readonly OutputGroup outputGroup; + public Regex IncludePattern { get; } public FolderInclusionRule( string folder, @@ -22,38 +24,47 @@ public FolderInclusionRule( InputDataType inputDataType, OutputGroup outputGroup, bool excludeList = true, - List includeExcludeFiles = null + List includeExcludeFiles = null, + Regex includePattern = null ) { - this.Folder = folder; - this.Recursive = recursive; - this.InputDataType = inputDataType; + Folder = folder; + Recursive = recursive; + InputDataType = inputDataType; this.outputGroup = outputGroup; - this.ExcludeList = excludeList; - this.IncludeExcludeFiles = includeExcludeFiles != null + IncludePattern = includePattern; + ExcludeList = excludeList; + IncludeExcludeFiles = includeExcludeFiles != null ? includeExcludeFiles.ToList() : new List(); } public IEnumerable GetFilesToInclude(SectorDataFileFactory dataFileFactory) { - if (!Directory.Exists(this.Folder)) + if (!Directory.Exists(Folder)) { throw new InputDirectoryNotFoundException(Folder); } string[] allFiles = Directory.GetFiles( - this.Folder, + Folder, "*.*", - this.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly + Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly ); + + // Match on include patterns + if (IncludePattern != null) + { + allFiles = allFiles.Where(file => IncludePattern.IsMatch(file)).ToArray(); + } + Array.Sort(allFiles, StringComparer.InvariantCulture); List files = new List(); foreach (string path in allFiles) { - if (this.ShouldInclude(path)) + if (ShouldInclude(path)) { - files.Add(dataFileFactory.Create(path, this.InputDataType)); + files.Add(dataFileFactory.Create(path, InputDataType)); } } @@ -62,14 +73,14 @@ public IEnumerable GetFilesToInclude(SectorDataFileFacto private bool ShouldInclude(string path) { - return this.ExcludeList - ? !this.IncludeExcludeFiles.Contains(Path.GetFileName(path)) - : this.IncludeExcludeFiles.Contains(Path.GetFileName(path)); + return ExcludeList + ? !IncludeExcludeFiles.Contains(Path.GetFileName(path)) + : IncludeExcludeFiles.Contains(Path.GetFileName(path)); } public OutputGroup GetOutputGroup() { - return this.outputGroup; + return outputGroup; } } } diff --git a/tests/CompilerTest/Config/ConfigIncludeLoaderTest.cs b/tests/CompilerTest/Config/ConfigIncludeLoaderTest.cs index 48b6f0e3..df4c8bc5 100644 --- a/tests/CompilerTest/Config/ConfigIncludeLoaderTest.cs +++ b/tests/CompilerTest/Config/ConfigIncludeLoaderTest.cs @@ -17,8 +17,8 @@ public class ConfigIncludeLoaderTest public ConfigIncludeLoaderTest() { - this.fileLoader = new ConfigIncludeLoader(); - this.includes = new ConfigInclusionRules(); + fileLoader = new ConfigIncludeLoader(); + includes = new ConfigInclusionRules(); } [Theory] @@ -41,10 +41,12 @@ public ConfigIncludeLoaderTest() [InlineData("_TestData/ConfigIncludeLoader/FilePathInvalid/config.json", "Invalid file path in section misc.regions - must be a string")] [InlineData("_TestData/ConfigIncludeLoader/ParentSectionNotArrayOrObject/config.json", "Invalid config section for enroute - must be an object or array of objects") ] [InlineData("_TestData/ConfigIncludeLoader/ParentSectionNotArrayOfObjects/config.json", "Invalid config section for enroute - must be an object or array of objects")] + [InlineData("_TestData/ConfigIncludeLoader/PatternNotAString/config.json", "Pattern invalid in section enroute.ownership - must be a regular expression string")] + [InlineData("_TestData/ConfigIncludeLoader/PatternInvalid/config.json", "Pattern invalid in section enroute.ownership - must be a regular expression string")] public void TestItThrowsExceptionOnBadData(string fileToLoad, string expectedMessage) { ConfigFileInvalidException exception = Assert.Throws( - () => fileLoader.LoadConfig(this.includes, JObject.Parse(File.ReadAllText(fileToLoad)), fileToLoad) + () => fileLoader.LoadConfig(includes, JObject.Parse(File.ReadAllText(fileToLoad)), fileToLoad) ); Assert.Equal(expectedMessage, exception.Message); } @@ -58,18 +60,18 @@ private string GetFullFilePath(string relative) public void TestItHandlesNoIncludes() { fileLoader.LoadConfig( - this.includes, + includes, JObject.Parse(File.ReadAllText("_TestData/ConfigIncludeLoader/NoIncludes/config.json")), "_TestData/ConfigIncludeLoader/NoIncludes/config.json" ); - Assert.Empty(this.includes); + Assert.Empty(includes); } [Fact] public void TestItLoadsAConfigFile() { fileLoader.LoadConfig( - this.includes, + includes, JObject.Parse(File.ReadAllText("_TestData/ConfigIncludeLoader/ValidConfig/config.json")), "_TestData/ConfigIncludeLoader/ValidConfig/config.json" ); @@ -133,6 +135,7 @@ public void TestItLoadsAConfigFile() Assert.True(ownershipRule3.ExcludeList); Assert.Single(ownershipRule3.IncludeExcludeFiles); Assert.Equal("EUR Islands.txt", ownershipRule3.IncludeExcludeFiles[0]); + Assert.Equal(".*?", ownershipRule3.IncludePattern.ToString()); Assert.Equal(new OutputGroup("enroute.ESE_OWNERSHIP", "Start enroute Ownership"), ownershipRule3.GetOutputGroup()); // Misc regions diff --git a/tests/CompilerTest/Input/FolderInclusionRuleTest.cs b/tests/CompilerTest/Input/FolderInclusionRuleTest.cs index 6f1462dd..31874f3c 100644 --- a/tests/CompilerTest/Input/FolderInclusionRuleTest.cs +++ b/tests/CompilerTest/Input/FolderInclusionRuleTest.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using Compiler.Input; using Compiler.Output; using Compiler.Exception; @@ -44,9 +45,27 @@ public void TestItLoadsFiles() IEnumerable includeFiles = rule.GetFilesToInclude(fileFactory); List files = includeFiles.ToList(); Assert.Equal(3, files.Count); - Assert.Equal(this.GetFilePath("File1.txt"), files[0].FullPath); - Assert.Equal(this.GetFilePath("File2.txt"), files[1].FullPath); - Assert.Equal(this.GetFilePath("File3.txt"), files[2].FullPath); + Assert.Equal(GetFilePath("File1.txt"), files[0].FullPath); + Assert.Equal(GetFilePath("File2.txt"), files[1].FullPath); + Assert.Equal(GetFilePath("File3.txt"), files[2].FullPath); + } + + [Fact] + public void TestItLoadsFilesBasedOnARegularExpression() + { + FolderInclusionRule rule = new ( + "_TestData/FolderInclusionRule", + false, + InputDataType.ESE_AGREEMENTS, + new OutputGroup("test"), + includePattern: new Regex("File[1|3].txt") + ); + + IEnumerable includeFiles = rule.GetFilesToInclude(fileFactory); + List files = includeFiles.ToList(); + Assert.Equal(2, files.Count); + Assert.Equal(GetFilePath("File1.txt"), files[0].FullPath); + Assert.Equal(GetFilePath("File3.txt"), files[1].FullPath); } [Fact] @@ -62,10 +81,10 @@ public void TestItLoadsFilesRecursively() IEnumerable includeFiles = rule.GetFilesToInclude(fileFactory); List files = includeFiles.ToList(); Assert.Equal(4, files.Count); - Assert.Equal(this.GetFilePath("File1.txt"), files[0].FullPath); - Assert.Equal(this.GetFilePath("File2.txt"), files[1].FullPath); - Assert.Equal(this.GetFilePath("File3.txt"), files[2].FullPath); - Assert.Equal(this.GetFilePath($"Level2{Path.DirectorySeparatorChar}File4.txt"), files[3].FullPath); + Assert.Equal(GetFilePath("File1.txt"), files[0].FullPath); + Assert.Equal(GetFilePath("File2.txt"), files[1].FullPath); + Assert.Equal(GetFilePath("File3.txt"), files[2].FullPath); + Assert.Equal(GetFilePath($"Level2{Path.DirectorySeparatorChar}File4.txt"), files[3].FullPath); } [Fact] @@ -83,8 +102,8 @@ public void TestItHasAnExcludeList() IEnumerable includeFiles = rule.GetFilesToInclude(fileFactory); List files = includeFiles.ToList(); Assert.Equal(2, files.Count); - Assert.Equal(this.GetFilePath("File1.txt"), files[0].FullPath); - Assert.Equal(this.GetFilePath("File3.txt"), files[1].FullPath); + Assert.Equal(GetFilePath("File1.txt"), files[0].FullPath); + Assert.Equal(GetFilePath("File3.txt"), files[1].FullPath); } [Fact] @@ -102,7 +121,7 @@ public void TestItHasAnIncludeList() IEnumerable includeFiles = rule.GetFilesToInclude(fileFactory); List files = includeFiles.ToList(); Assert.Single(files); - Assert.Equal(this.GetFilePath("File2.txt"), files[0].FullPath); + Assert.Equal(GetFilePath("File2.txt"), files[0].FullPath); } [Fact] diff --git a/tests/CompilerTest/_TestData/ConfigIncludeLoader/PatternInvalid/config.json b/tests/CompilerTest/_TestData/ConfigIncludeLoader/PatternInvalid/config.json new file mode 100644 index 00000000..55c8bf0d --- /dev/null +++ b/tests/CompilerTest/_TestData/ConfigIncludeLoader/PatternInvalid/config.json @@ -0,0 +1,13 @@ +{ + "includes": { + "enroute": { + "ownership": [ + { + "type": "folder", + "folder": "test", + "pattern": "*" + } + ] + } + } +} \ No newline at end of file diff --git a/tests/CompilerTest/_TestData/ConfigIncludeLoader/PatternNotAString/config.json b/tests/CompilerTest/_TestData/ConfigIncludeLoader/PatternNotAString/config.json new file mode 100644 index 00000000..993bbf2d --- /dev/null +++ b/tests/CompilerTest/_TestData/ConfigIncludeLoader/PatternNotAString/config.json @@ -0,0 +1,13 @@ +{ + "includes": { + "enroute": { + "ownership": [ + { + "type": "folder", + "folder": "test", + "pattern": 123 + } + ] + } + } +} \ No newline at end of file diff --git a/tests/CompilerTest/_TestData/ConfigIncludeLoader/ValidConfig/config.json b/tests/CompilerTest/_TestData/ConfigIncludeLoader/ValidConfig/config.json index ba1057dd..b369112e 100644 --- a/tests/CompilerTest/_TestData/ConfigIncludeLoader/ValidConfig/config.json +++ b/tests/CompilerTest/_TestData/ConfigIncludeLoader/ValidConfig/config.json @@ -38,6 +38,7 @@ { "type": "folder", "folder": "Ownership/Non-UK", + "pattern": ".*?", "exclude": [ "EUR Islands.txt" ]