Merge pull request #251 from AsherGlick/reorganizing_tests

Reorganizing Tests

.github/workflows/main.yml

@@ -54,9 +54,9 @@ jobs:
         run: |
           pip3 install cpplint
 
-      - name: Install xml_converter/generators Dependencies
+      - name: Install xml_converter python Dependencies
         run: |
-          cd xml_converter/generators
+          cd xml_converter
           python3 -m venv venv
           source venv/bin/activate
           pip install -r requirements.txt

xml_converter/intigration_tests/make_test.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Base testcase directory
+mkdir -p test_cases/$1
+
+# Testcase configuration
+touch test_cases/$1/testcase.yaml
+echo 'input_paths: ' >> test_cases/$1/testcase.yaml
+echo '    "pack": "xml"' >> test_cases/$1/testcase.yaml
+echo 'expected_stdout: |' >> test_cases/$1/testcase.yaml
+echo 'expected_stderr: |' >> test_cases/$1/testcase.yaml
+echo 'expected_returncode: 0' >> test_cases/$1/testcase.yaml
+
+# Input folder and test marker pack
+mkdir -p test_cases/$1/input/pack
+
+# Output folders and blank output files
+mkdir -p test_cases/$1/output_proto
+touch test_cases/$1/output_proto/markers.bin
+mkdir -p test_cases/$1/output_xml
+touch test_cases/$1/output_xml/xml_file.xml

xml_converter/intigration_tests/run_tests.py

@@ -7,9 +7,9 @@
 import re
 import os
 from typing import List, Optional, Final, Tuple
-from testcases import testcases
+from src.testcase_loader import load_testcases
 import shutil
-from proto_utils import compare_protos
+from src.proto_utils import compare_protos
 
 # Path to compiled C++ executable
 xml_converter_binary_path: str = "../build/xml_converter"
@@ -157,7 +157,7 @@ def main() -> None:
 
     rebuild_xml_converter_binary()
 
-    for testcase in testcases:
+    for testcase in load_testcases():
         xml_output_dir_path = os.path.join(output_parent_dirpath, "xml", testcase.name)
         proto_output_dir_path = os.path.join(output_parent_dirpath, "proto", testcase.name)
 

xml_converter/intigration_tests/src/testcase_loader.py

@@ -0,0 +1,129 @@
+from typing import List, Optional
+from dataclasses import dataclass
+import yaml
+import os
+
+
+################################################################################
+# Testcase
+#
+# A dataclass for storing all the structured data for a single testcase.
+################################################################################
+@dataclass
+class Testcase:
+    name: str
+    xml_input_paths: List[str]
+    proto_input_paths: List[str]
+
+    expected_output_xml_path: str
+    expected_output_proto_path: str
+
+    expected_stdout: List[str]
+    expected_stderr: List[str]
+    expected_returncode: int
+
+
+################################################################################
+# load_testcases
+#
+# Load all of the testcases found in the `test_cases` directory.
+################################################################################
+def load_testcases() -> List[Testcase]:
+    testcase_dirs: List[Testcase] = []
+
+    for testcase_dir in os.listdir("test_cases"):
+        testcase = load_testcase(os.path.join("test_cases", testcase_dir))
+
+        if testcase is None:
+            continue
+
+        testcase_dirs.append(testcase)
+
+    return testcase_dirs
+
+
+################################################################################
+# load_testcase
+#
+# A simple loader that loads the testcase from a specific directory, doing
+# typechecking on the testcase data to be sure it is properly structured.
+################################################################################
+def load_testcase(path) -> Optional[Testcase]:
+    test_info_path = os.path.join(path, "testcase.yaml")
+    with open(test_info_path) as f:
+        data = yaml.safe_load(f)
+
+    inputs_path = os.path.join(path, "input")
+
+    # Load all of the input paths into either xml input or proto inputs
+    xml_input_paths: List[str] = []
+    proto_input_paths: List[str] = []
+    for pack_name, pack_type in data["input_paths"].items():
+        if not isinstance(pack_name, str):
+            print(f"Invalid pack name, expecting a string but got {pack_name}")
+            return
+
+        pack_path = os.path.join(inputs_path, pack_name)
+
+        if not os.path.exists(pack_path):
+            print(f"Input pack path {pack_path} not found")
+            return
+
+        if pack_type == "xml":
+            xml_input_paths.append(pack_path)
+        elif pack_type == "proto":
+            proto_input_paths.append(pack_path)
+        else:
+            print(f"Invalid pack type {pack_type} found in {test_info_path}")
+            return
+
+    # Sanity check that all the input directories were accounted for
+    for possible_input_path in os.listdir(inputs_path):
+        if possible_input_path not in data["input_paths"]:
+            print(f"Found the input directory {possible_input_path} in {path} but no config for it")
+
+    # Typecheck the expected stdout, stderr, and returncode values
+    if "expected_stdout" not in data:
+        print(f"Expected 'expected_stdout' field in {test_info_path}")
+        return
+    elif not isinstance(data["expected_stdout"], str):
+        print(f"Invalid Test, expecting string value for 'expected_stdout' in {path}")
+        return
+
+    if "expected_stderr" not in data:
+        print(f"Expected 'expected_stderr' field in {test_info_path}")
+        return
+    elif not isinstance(data["expected_stderr"], str):
+        print(f"Invalid Test, expecting string value for 'expected_stderr' in {path}")
+        return
+
+    if "expected_returncode" not in data:
+        print(f"Expected 'expected_returncode' field in {test_info_path}")
+        return
+    elif not isinstance(data["expected_returncode"], int):
+        print(f"Invalid Test, expecting string value for 'expected_returncode' in {path}")
+        return
+
+    return Testcase(
+        name=os.path.basename(path),
+        xml_input_paths=xml_input_paths,
+        proto_input_paths=proto_input_paths,
+        expected_output_xml_path=os.path.join(path, "output_xml"),
+        expected_output_proto_path=os.path.join(path, "output_proto"),
+        expected_stdout=to_lines(data["expected_stdout"]),
+        expected_stderr=to_lines(data["expected_stderr"]),
+        expected_returncode=data["expected_returncode"]
+    )
+
+
+################################################################################
+# to_lines
+#
+# A helper function to remove final empty lines from split arrays.
+################################################################################
+def to_lines(value: str) -> List[str]:
+    output = value.split("\n")
+    if output[-1] == "":
+        return output[:-1]
+    else:
+        return output

xml_converter/intigration_tests/test_cases/achievement_bitmask_valid/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/achievement_id/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/animation_speed/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/canfade_invalid/testcase.yaml

@@ -0,0 +1,13 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+    Error: Found a boolean value that was not a '1', '0', 'true', or 'false'
+    test_cases/canfade_invalid/input/pack/xml_file.xml
+    6 |        <POI CanFade="yes" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
+      |                      ^^^
+    Error: Found a boolean value that was not a '1', '0', 'true', or 'false'
+    test_cases/canfade_invalid/input/pack/xml_file.xml
+    7 |        <POI CanFade="はい" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
+      |                      ^^^^^^
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/canfade_valid/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/cull_chirality/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/fade_distance/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/festival_filter/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/is_wall/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/map_id/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/map_type_filter/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/mount_filter_invalid/testcase.yaml

@@ -0,0 +1,21 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+    Error: Invalid Filter for MountFilter. Found 
+    test_cases/mount_filter_invalid/input/pack/xml_file.xml
+    6 |        <POI Mount="" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
+      |                    
+    Error: Invalid Filter for MountFilter. Found NotAMount
+    test_cases/mount_filter_invalid/input/pack/xml_file.xml
+    7 |        <POI Mount="NotAMount" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
+      |                    ^^^^^^^^^
+    Error: Invalid Filter for MountFilter. Found 
+    test_cases/mount_filter_invalid/input/pack/xml_file.xml
+    8 |        <POI Mount="Raptor,Springer," Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
+      |                    ^^^^^^^^^^^^^^^^
+    Error: Invalid Filter for MountFilter. Found NotAMount
+    test_cases/mount_filter_invalid/input/pack/xml_file.xml
+    9 |        <POI Mount="Raptor,NotAMount,Springer" Type="mycategory" XPos="169.81" YPos="210.65" ZPos="215.83" MapID="50" />
+      |                    ^^^^^^^^^^^^^^^^^^^^^^^^^
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/mountfilter_valid/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/profession_filter/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/specialization_filter/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/species_filter/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0

xml_converter/intigration_tests/test_cases/texture/testcase.yaml

@@ -0,0 +1,5 @@
+input_paths: 
+    "pack": "xml"
+expected_stdout: |
+expected_stderr: |
+expected_returncode: 0