Skip to content

Commit

Permalink
Merge pull request #121 from AndyTWF/include-patterns
Browse files Browse the repository at this point in the history 
feat(config): allow folder includes matching a pattern
  • Loading branch information
@AndyTWF
AndyTWF authored Jan 30, 2021
2 parents e3e8cb5 + a8423f0 commit 522e00d
Show file tree
Hide file tree
Showing 8 changed files with 143 additions and 51 deletions.
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,17 +199,20 @@ This rule includes all files within a given folder.
"type": "folder",
"folder": "Ownership/Alternate",
"recursive": true,
"pattern": "SomeRegex",
"exclude": [
"EUR Islands.txt"
]
}
```

There are two optional flags available for the folder rule:
There are three optional flags available for the folder rule:

- `recursive` (default: `false`) will cause the compiler to include all files in any subfolders
contained within the main folder.
- `exclude` will cause the compiler to ignore any files with a particular name.
- `exclude` will cause the compiler to ignore any files with a particular name. Conversely, specifying `include`
will only include files with a certain name.
- `pattern` allows you to provide a regular expression, with only files matching the pattern being included.

## Comment Annotations

Expand Down
65 changes: 47 additions & 18 deletions src/Compiler/Config/ConfigIncludeLoader.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using Compiler.Exception;
using Compiler.Input;
using Compiler.Output;
Expand All @@ -19,14 +20,14 @@ string fileName
JToken airportData = jsonConfig.SelectToken("includes.airports");
if (airportData != null)
{
this.IterateAirportConfig(airportData, inclusionRules, fileName);
IterateAirportConfig(airportData, inclusionRules, fileName);
}

// Load enroute data
JToken enrouteData = jsonConfig.SelectToken("includes.enroute");
if (enrouteData != null)
{
this.IterateConfigFileSections(
IterateConfigFileSections(
enrouteData,
EnrouteConfigFileSections.ConfigFileSections,
OutputGroupFactory.CreateEnroute,
Expand All @@ -40,7 +41,7 @@ string fileName
JToken miscData = jsonConfig.SelectToken("includes.misc");
if (miscData != null)
{
this.IterateConfigFileSections(
IterateConfigFileSections(
miscData,
MiscConfigFileSections.ConfigFileSections,
OutputGroupFactory.CreateMisc,
Expand All @@ -65,6 +66,11 @@ private string GetInvalidFolderMessage(string section)
{
return $"Folder invalid in section {section} - must be string under key \"folder\"";
}

private string GetInvalidPatternMessage(string section)
{
return $"Pattern invalid in section {section} - must be a regular expression string";
}

private string GetRecursiveMessage(string section)
{
Expand Down Expand Up @@ -134,13 +140,13 @@ string configFilePath
}

// Get the airport folders
string configFileFolder = this.GetFolderForConfigFile(configFilePath);
string configFileFolder = GetFolderForConfigFile(configFilePath);
string[] directories = Directory.GetDirectories(configFileFolder + Path.DirectorySeparatorChar + configItem.Key);

// For each airport, iterate the config file sections
foreach (string directory in directories)
{
this.IterateConfigFileSections(
IterateConfigFileSections(
configItem.Value,
AirfieldConfigFileSections.ConfigFileSections,
x => OutputGroupFactory.CreateAirport(x, Path.GetFileName(directory)),
Expand Down Expand Up @@ -168,7 +174,7 @@ string sectionRootString
continue;
}

this.LoadConfigSection(
LoadConfigSection(
configObjectSection,
configSection,
createOutputGroup(configSection),
Expand All @@ -194,7 +200,7 @@ string sectionRootString
{
foreach (JToken token in (JArray) jsonConfig)
{
this.ProcessConfigSectionObject(
ProcessConfigSectionObject(
token,
configFileSection,
outputGroup,
Expand All @@ -204,7 +210,7 @@ string sectionRootString
);
}
} else {
this.ProcessConfigSectionObject(
ProcessConfigSectionObject(
jsonConfig,
configFileSection,
outputGroup,
Expand Down Expand Up @@ -243,7 +249,7 @@ string sectionRootString

if ((string)typeToken == "files")
{
this.ProcessFilesList(
ProcessFilesList(
configObject,
configFileSection,
outputGroup,
Expand All @@ -253,7 +259,7 @@ string sectionRootString
);
} else
{
this.ProcessFolder(
ProcessFolder(
configObject,
configFileSection,
outputGroup,
Expand All @@ -276,9 +282,8 @@ private void ProcessFolder(
string sectionRootString
) {
// Get the folder
JToken folder;
if (
!folderObject.TryGetValue("folder", out folder) ||
!folderObject.TryGetValue("folder", out JToken folder) ||
folder.Type != JTokenType.String
) {
throw new ConfigFileInvalidException(
Expand All @@ -300,6 +305,28 @@ string sectionRootString

recursive = (bool)recursiveToken;
}

// Handle inclusion patterns
Regex patternRegex = null;
if (folderObject.TryGetValue("pattern", out JToken pattern)) {
if (pattern.Type != JTokenType.String)
{
throw new ConfigFileInvalidException(
GetInvalidPatternMessage($"{sectionRootString}.{configFileSection.JsonPath}")
);
}

try
{
patternRegex = new Regex(pattern.ToString());
}
catch (ArgumentException)
{
throw new ConfigFileInvalidException(
GetInvalidPatternMessage($"{sectionRootString}.{configFileSection.JsonPath}")
);
}
}


// Get the include and exclude lists and check both aren't there
Expand All @@ -317,10 +344,11 @@ string sectionRootString
if (!isInclude && !isExclude) {
addInclusionRule(
new FolderInclusionRule(
this.NormaliseFilePath(rootPath, (string)folder),
NormaliseFilePath(rootPath, (string)folder),
recursive,
configFileSection.DataType,
outputGroup
outputGroup,
includePattern: patternRegex
)
);
return;
Expand Down Expand Up @@ -350,12 +378,13 @@ string sectionRootString

addInclusionRule(
new FolderInclusionRule(
this.NormaliseFilePath(rootPath, (string)folder),
NormaliseFilePath(rootPath, (string)folder),
recursive,
configFileSection.DataType,
outputGroup,
isExclude,
files
files,
patternRegex
)
);
}
Expand Down Expand Up @@ -408,7 +437,7 @@ string sectionRootString
);
}

exceptWhereExists = this.NormaliseFilePath(rootPath, (string) exceptWhereExistsToken);
exceptWhereExists = NormaliseFilePath(rootPath, (string) exceptWhereExistsToken);
}

// Get the file paths and normalise against the config files folder
Expand All @@ -422,7 +451,7 @@ string sectionRootString
);
}

filePaths.Add(this.NormaliseFilePath(rootPath, (string)file));
filePaths.Add(NormaliseFilePath(rootPath, (string)file));
}

// Add the rule
Expand Down
41 changes: 26 additions & 15 deletions src/Compiler/Input/FolderInclusionRule.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Compiler.Exception;
using Compiler.Output;

Expand All @@ -15,45 +16,55 @@ public class FolderInclusionRule : IInclusionRule
public bool ExcludeList { get; }
public List<string> IncludeExcludeFiles { get; }
private readonly OutputGroup outputGroup;
public Regex IncludePattern { get; }

public FolderInclusionRule(
string folder,
bool recursive,
InputDataType inputDataType,
OutputGroup outputGroup,
bool excludeList = true,
List<string> includeExcludeFiles = null
List<string> includeExcludeFiles = null,
Regex includePattern = null
) {
this.Folder = folder;
this.Recursive = recursive;
this.InputDataType = inputDataType;
Folder = folder;
Recursive = recursive;
InputDataType = inputDataType;
this.outputGroup = outputGroup;
this.ExcludeList = excludeList;
this.IncludeExcludeFiles = includeExcludeFiles != null
IncludePattern = includePattern;
ExcludeList = excludeList;
IncludeExcludeFiles = includeExcludeFiles != null
? includeExcludeFiles.ToList()
: new List<string>();
}

public IEnumerable<AbstractSectorDataFile> GetFilesToInclude(SectorDataFileFactory dataFileFactory)
{
if (!Directory.Exists(this.Folder))
if (!Directory.Exists(Folder))
{
throw new InputDirectoryNotFoundException(Folder);
}

string[] allFiles = Directory.GetFiles(
this.Folder,
Folder,
"*.*",
this.Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly
Recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly
);

// Match on include patterns
if (IncludePattern != null)
{
allFiles = allFiles.Where(file => IncludePattern.IsMatch(file)).ToArray();
}

Array.Sort(allFiles, StringComparer.InvariantCulture);

List<AbstractSectorDataFile> files = new List<AbstractSectorDataFile>();
foreach (string path in allFiles)
{
if (this.ShouldInclude(path))
if (ShouldInclude(path))
{
files.Add(dataFileFactory.Create(path, this.InputDataType));
files.Add(dataFileFactory.Create(path, InputDataType));
}
}

Expand All @@ -62,14 +73,14 @@ public IEnumerable<AbstractSectorDataFile> GetFilesToInclude(SectorDataFileFacto

private bool ShouldInclude(string path)
{
return this.ExcludeList
? !this.IncludeExcludeFiles.Contains(Path.GetFileName(path))
: this.IncludeExcludeFiles.Contains(Path.GetFileName(path));
return ExcludeList
? !IncludeExcludeFiles.Contains(Path.GetFileName(path))
: IncludeExcludeFiles.Contains(Path.GetFileName(path));
}

public OutputGroup GetOutputGroup()
{
return this.outputGroup;
return outputGroup;
}
}
}
15 changes: 9 additions & 6 deletions tests/CompilerTest/Config/ConfigIncludeLoaderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ public class ConfigIncludeLoaderTest

public ConfigIncludeLoaderTest()
{
this.fileLoader = new ConfigIncludeLoader();
this.includes = new ConfigInclusionRules();
fileLoader = new ConfigIncludeLoader();
includes = new ConfigInclusionRules();
}

[Theory]
Expand All @@ -41,10 +41,12 @@ public ConfigIncludeLoaderTest()
[InlineData("_TestData/ConfigIncludeLoader/FilePathInvalid/config.json", "Invalid file path in section misc.regions - must be a string")]
[InlineData("_TestData/ConfigIncludeLoader/ParentSectionNotArrayOrObject/config.json", "Invalid config section for enroute - must be an object or array of objects") ]
[InlineData("_TestData/ConfigIncludeLoader/ParentSectionNotArrayOfObjects/config.json", "Invalid config section for enroute - must be an object or array of objects")]
[InlineData("_TestData/ConfigIncludeLoader/PatternNotAString/config.json", "Pattern invalid in section enroute.ownership - must be a regular expression string")]
[InlineData("_TestData/ConfigIncludeLoader/PatternInvalid/config.json", "Pattern invalid in section enroute.ownership - must be a regular expression string")]
public void TestItThrowsExceptionOnBadData(string fileToLoad, string expectedMessage)
{
ConfigFileInvalidException exception = Assert.Throws<ConfigFileInvalidException>(
() => fileLoader.LoadConfig(this.includes, JObject.Parse(File.ReadAllText(fileToLoad)), fileToLoad)
() => fileLoader.LoadConfig(includes, JObject.Parse(File.ReadAllText(fileToLoad)), fileToLoad)
);
Assert.Equal(expectedMessage, exception.Message);
}
Expand All @@ -58,18 +60,18 @@ private string GetFullFilePath(string relative)
public void TestItHandlesNoIncludes()
{
fileLoader.LoadConfig(
this.includes,
includes,
JObject.Parse(File.ReadAllText("_TestData/ConfigIncludeLoader/NoIncludes/config.json")),
"_TestData/ConfigIncludeLoader/NoIncludes/config.json"
);
Assert.Empty(this.includes);
Assert.Empty(includes);
}

[Fact]
public void TestItLoadsAConfigFile()
{
fileLoader.LoadConfig(
this.includes,
includes,
JObject.Parse(File.ReadAllText("_TestData/ConfigIncludeLoader/ValidConfig/config.json")),
"_TestData/ConfigIncludeLoader/ValidConfig/config.json"
);
Expand Down Expand Up @@ -133,6 +135,7 @@ public void TestItLoadsAConfigFile()
Assert.True(ownershipRule3.ExcludeList);
Assert.Single(ownershipRule3.IncludeExcludeFiles);
Assert.Equal("EUR Islands.txt", ownershipRule3.IncludeExcludeFiles[0]);
Assert.Equal(".*?", ownershipRule3.IncludePattern.ToString());
Assert.Equal(new OutputGroup("enroute.ESE_OWNERSHIP", "Start enroute Ownership"), ownershipRule3.GetOutputGroup());

// Misc regions
Expand Down
Loading

0 comments on commit 522e00d

Please sign in to comment.