fix(config): Allow directories to be excluded on files rules (#158)

* fix(config): Allow directories to be excluded on files rules

* fix(include): Prevent double file inclusion

Fix #157

* Fix Warnings

README.md

@@ -196,7 +196,8 @@ This rule includes files as listed in the configuration file.
         "Basic.txt"
     ],
     "ignore_missing": true,
-    "except_where_exists": "Basic2.txt"
+    "except_where_exists": "Basic2.txt",
+    "exclude_directory": "EGAC"
 }
 ```
 There are two optional flags available for the file list rule:
@@ -205,6 +206,8 @@ There are two optional flags available for the file list rule:
 if it cannot be found (would usually cause an error). This is useful for airports where not all airports have all files.
 - `except_where_exists`instructs the compiler to skip the files, if another file is present.
 This is particularly useful for SMRs and Ground Maps.
+- `exclude_directory` useful for processing airport ownership data. Excludes a particular directory from
+the rule.
 
 #### Folders
 

src/Compiler/Config/FileInclusionRuleLoader.cs

@@ -29,16 +29,19 @@ InputFilePathNormaliser pathNormaliser
 
         public IInclusionRule CreateRule(JObject config)
         {
-            return ApplyIgnoreMissingFilter(
-                config,
-                ApplyExceptWhereExistsFilter(
+            return ApplyExcludeDirectory(
                     config,
-                    ApplyGenerator(
+                    ApplyIgnoreMissingFilter(
                         config,
-                        FileInclusionRuleBuilder.Begin()
+                        ApplyExceptWhereExistsFilter(
+                            config,
+                            ApplyGenerator(
+                                config,
+                                FileInclusionRuleBuilder.Begin()
+                            )
+                        )
                     )
                 )
-            )
                 .SetDescriptor(new RuleDescriptor($"File inclusion rule in {path}"))
                 .AddValidator(new FileExists())
                 .SetOutputGroup(outputGroup)
@@ -53,10 +56,11 @@ private IFileInclusionRuleBuilder ApplyGenerator(JObject config, IFileInclusionR
             if (
                 !config.TryGetValue("files", out filesList) ||
                 filesList.Type != JTokenType.Array
-            ) {
+            )
+            {
                 ThrowInvalidFilesListException();
             }
-            
+
             // Get the file paths and normalise them
             if (filesList!.Any(file => file.Type != JTokenType.String))
             {
@@ -67,7 +71,27 @@ private IFileInclusionRuleBuilder ApplyGenerator(JObject config, IFileInclusionR
                 new FileListGenerator(filesList.Select(file => pathNormaliser.NormaliseFilePath((string) file)))
             );
         }
-
+
+        private IFileInclusionRuleBuilder ApplyExcludeDirectory(JObject config, IFileInclusionRuleBuilder builder)
+        {
+            if (!config.TryGetValue("exclude_directory", out JToken excludeDirectory))
+            {
+                return builder;
+            }
+
+            if (
+                excludeDirectory.Type != JTokenType.Array ||
+                excludeDirectory.Any(directory => directory.Type != JTokenType.String)
+            )
+            {
+                ThrowInvalidExcludeDirectoryException();
+            }
+
+            return builder.AddFilter(
+                new ExcludeByParentFolder(excludeDirectory.Select(directory => (string) directory))
+            );
+        }
+
         private IFileInclusionRuleBuilder ApplyIgnoreMissingFilter(
             JObject config,
             IFileInclusionRuleBuilder builder
@@ -77,16 +101,17 @@ IFileInclusionRuleBuilder builder
             {
                 return builder;
             }
-            
+
             if (ignoreMissingToken.Type != JTokenType.Boolean)
             {
                 ThrowInvalidIgnoreMissingException();
             }
 
             return (bool) ignoreMissingToken ? builder.AddFilter(new Input.Filter.FileExists()) : builder;
         }
-
-        private IFileInclusionRuleBuilder ApplyExceptWhereExistsFilter(JObject config, IFileInclusionRuleBuilder builder)
+
+        private IFileInclusionRuleBuilder ApplyExceptWhereExistsFilter(JObject config,
+            IFileInclusionRuleBuilder builder)
         {
             if (!config.TryGetValue("except_where_exists", out var exceptWhereExistsToken))
             {
@@ -102,11 +127,18 @@ private IFileInclusionRuleBuilder ApplyExceptWhereExistsFilter(JObject config, I
                 new IgnoreWhenFileExists(pathNormaliser.NormaliseFilePath((string) exceptWhereExistsToken))
             );
         }
-        
+
         private void ThrowInvalidFilesListException()
         {
             ThrowConfigFileException(
-                $"Files list invalid in section {path} - must be array under key \"files\""    
+                $"Files list invalid in section {path} - must be array under key \"files\""
+            );
+        }
+
+        private void ThrowInvalidExcludeDirectoryException()
+        {
+            ThrowConfigFileException(
+                $"Invalid exclude_directory invalid in section {path} - must be an array of strings"
             );
         }
 

src/Compiler/Input/AbstractSectorDataFile.cs

@@ -13,7 +13,7 @@ abstract public class AbstractSectorDataFile: IEnumerable<SectorData>
         public string CurrentLine { get; protected set; } = "";
 
         // The full path to the file
-        public string FullPath { get; protected set; }
+        public string FullPath { get; }
 
         // The type of data this file contains
         public InputDataType DataType { get; }
@@ -24,19 +24,29 @@ public AbstractSectorDataFile(string fullPath, InputDataType dataType)
             DataType = dataType;
         }
 
+        public override bool Equals(object obj)
+        {
+            return obj is AbstractSectorDataFile file && Equals(file);
+        }
+
+        public override int GetHashCode()
+        {
+            return (FullPath != null ? FullPath.GetHashCode() : 0);
+        }
+
         public bool Equals(AbstractSectorDataFile compare)
         {
-            return this.FullPath == compare.FullPath;
+            return FullPath == compare.FullPath;
         }
 
         public string GetParentDirectoryName()
         {
-            return Path.GetFileName(Path.GetDirectoryName(this.FullPath));
+            return Path.GetFileName(Path.GetDirectoryName(FullPath));
         }
 
         public string GetFileName()
         {
-            return Path.GetFileNameWithoutExtension(this.FullPath);
+            return Path.GetFileNameWithoutExtension(FullPath);
         }
 
         public abstract IEnumerator<SectorData> GetEnumerator();

src/Compiler/Input/Filter/ExcludeByParentFolder.cs

@@ -6,16 +6,16 @@ namespace Compiler.Input.Filter
 {
     public class ExcludeByParentFolder: IFileFilter
     {
-        private readonly IEnumerable<string> parentFolders;
+        public IEnumerable<string> ParentFolders { get; }
 
         public ExcludeByParentFolder(IEnumerable<string> parentFolders)
         {
-            this.parentFolders = parentFolders;
+            ParentFolders = parentFolders;
         }
 
         public bool Filter(string path)
         {
-            return !parentFolders.Contains(
+            return !ParentFolders.Contains(
                 new FileInfo(Path.GetDirectoryName(path)).Name
             );
         }

tests/CompilerTest/Config/ConfigIncludeLoaderTest.cs

@@ -48,7 +48,8 @@ public ConfigIncludeLoaderTest()
         [InlineData("_TestData/ConfigIncludeLoader/ParentSectionNotArrayOrObject/config.json", "Invalid config section for enroute - must be an object or array of objects") ]
         [InlineData("_TestData/ConfigIncludeLoader/ParentSectionNotArrayOfObjects/config.json", "Invalid config section for enroute - must be an object or array of objects")]
         [InlineData("_TestData/ConfigIncludeLoader/PatternNotAString/config.json", "Pattern invalid in section enroute.ownership - must be a regular expression string")]
-        [InlineData("_TestData/ConfigIncludeLoader/PatternInvalid/config.json", "Pattern invalid in section enroute.ownership - must be a regular expression string")]
+        [InlineData("_TestData/ConfigIncludeLoader/ExcludeDirectoryNotAnArray/config.json", "Invalid exclude_directory invalid in section misc.regions - must be an array of strings")]
+        [InlineData("_TestData/ConfigIncludeLoader/ExcludeDirectoryContainsNonString/config.json", "Invalid exclude_directory invalid in section misc.regions - must be an array of strings")]
         public void TestItThrowsExceptionOnBadData(string fileToLoad, string expectedMessage)
         {
             ConfigFileInvalidException exception = Assert.Throws<ConfigFileInvalidException>(
@@ -98,7 +99,13 @@ public void TestItLoadsAConfigFile()
                 airportBasicRule.ListGenerator.GetPaths().ToList()
             );
             Assert.Contains(airportBasicRule.Validators, validator => validator.GetType() == typeof(FileExists));
-            Assert.DoesNotContain(airportBasicRule.Filters, validator => validator.GetType() == typeof(IgnoreWhenFileExists));
+            Assert.DoesNotContain(airportBasicRule.Filters, fileFilter => fileFilter.GetType() == typeof(IgnoreWhenFileExists));
+            Assert.Contains(airportBasicRule.Filters, fileFilter => fileFilter.GetType() == typeof(ExcludeByParentFolder));
+            Assert.Contains(
+                "EGLL",
+                (airportBasicRule.Filters.First(fileFilter => fileFilter.GetType() == typeof(ExcludeByParentFolder)) as ExcludeByParentFolder)?.ParentFolders
+                    ?? throw new InvalidOperationException()
+            );
             Assert.Equal(InputDataType.SCT_AIRPORT_BASIC, airportBasicRule.DataType);
             Assert.Equal(new OutputGroup("airport.SCT_AIRPORT_BASIC.EGLL", "Start EGLL Basic"), airportBasicRule.GetOutputGroup());
 

tests/CompilerTest/Input/SectorDataFileTest.cs

@@ -11,7 +11,7 @@ public class SectorDataFileTest
 
         public SectorDataFileTest()
         {
-            this.file = new SectorDataFile(
+            file = new SectorDataFile(
                 "_TestData/SectorDataFile/StreamTest.txt",
                 new InputFileStreamFactory(),
                 InputDataType.ESE_AGREEMENTS,
@@ -24,7 +24,7 @@ public void ItSetsFullPath()
         {
             Assert.Equal(
                 "_TestData/SectorDataFile/StreamTest.txt",
-                this.file.FullPath
+                file.FullPath
             );
         }
 
@@ -33,7 +33,7 @@ public void ItGetsParentDirectory()
         {
             Assert.Equal(
                 "SectorDataFile",
-                this.file.GetParentDirectoryName()
+                file.GetParentDirectoryName()
             );
         }
 
@@ -42,33 +42,33 @@ public void ItGetsFileName()
         {
             Assert.Equal(
                 "StreamTest",
-                this.file.GetFileName()
+                file.GetFileName()
             );
         }
 
         [Fact]
         public void CurrentLineNumberStartsAtZero()
         {
-            Assert.Equal(0, this.file.CurrentLineNumber);
+            Assert.Equal(0, file.CurrentLineNumber);
         }
 
         [Fact]
         public void CurrentLineStartsAtEmpty()
         {
-            Assert.Equal("", this.file.CurrentLine);
+            Assert.Equal("", file.CurrentLine);
         }
 
         [Fact]
         public void ItHasADataType()
         {
-            Assert.Equal(InputDataType.ESE_AGREEMENTS, this.file.DataType);
+            Assert.Equal(InputDataType.ESE_AGREEMENTS, file.DataType);
         }
 
         [Fact]
         public void TestItIteratesTheInputFile()
         {
             int expectedLine = 3;
-            foreach (SectorData dataLine in this.file)
+            foreach (SectorData dataLine in file)
             {
                 Assert.Equal(new List<string> { "Line", expectedLine.ToString() }, dataLine.dataSegments);
 
@@ -77,13 +77,43 @@ public void TestItIteratesTheInputFile()
                 expectedDocblock.AddLine(new Comment("Docblock " + (expectedLine - 1)));
                 Assert.Equal(expectedDocblock, dataLine.docblock);
                 Assert.Equal(new Comment("Inline " + expectedLine), dataLine.inlineComment);
-                Assert.Equal(expectedLine, this.file.CurrentLineNumber);
+                Assert.Equal(expectedLine, file.CurrentLineNumber);
                 Assert.Equal(new Definition("_TestData/SectorDataFile/StreamTest.txt", expectedLine), dataLine.definition);
 
                 expectedLine += 4;
             }
 
-            Assert.Equal(33, this.file.CurrentLineNumber);
+            Assert.Equal(33, file.CurrentLineNumber);
+        }
+
+        [Fact]
+        public void ItsEqualIfPathTheSame()
+        {
+            var file2 = new SectorDataFile(
+                "_TestData/SectorDataFile/StreamTest.txt",
+                new InputFileStreamFactory(),
+                InputDataType.ESE_VRPS,
+                new EseSectorDataReader()
+            );
+            Assert.True(file.Equals(file2));
+        }
+
+        [Fact]
+        public void ItsNotEqualIfPathDifferent()
+        {
+            var file2 = new SectorDataFile(
+                "_TestData/SectorDataFile/StreamTest2.txt",
+                new InputFileStreamFactory(),
+                InputDataType.ESE_AGREEMENTS,
+                new EseSectorDataReader()
+            );
+            Assert.False(file.Equals(file2));
+        }
+
+        [Fact]
+        public void ItsNotEqualIfNotSameType()
+        {
+            Assert.False(file.Equals(new object()));
         }
     }
 }

tests/CompilerTest/_TestData/ConfigIncludeLoader/ExcludeDirectoryContainsNonString/config.json

@@ -0,0 +1,13 @@
+{
+  "includes": {
+    "misc": {
+      "regions": {
+        "type": "files",
+        "files": [
+          "Misc/Regions_LTMA Airfield CAS.txt"
+        ],
+        "exclude_directory": ["foo", 123, "bar"]
+      }
+    }
+  }
+}

tests/CompilerTest/_TestData/ConfigIncludeLoader/ExcludeDirectoryNotAnArray/config.json

@@ -0,0 +1,13 @@
+{
+  "includes": {
+    "misc": {
+      "regions": {
+        "type": "files",
+        "files": [
+          "Misc/Regions_LTMA Airfield CAS.txt"
+        ],
+        "exclude_directory": "foo"
+      }
+    }
+  }
+}

tests/CompilerTest/_TestData/ConfigIncludeLoader/ValidConfig/config.json

@@ -7,7 +7,8 @@
           "files": [
             "Basic.txt",
             "Basic2.txt"
-          ]
+          ],
+          "exclude_directory": ["EGLL"]
         },
         "smr": {
           "geo": {
@@ -56,4 +57,4 @@
       }
     }
   }
-}
+}