Restructured collation data generation

executors/cpp/coll.cpp

@@ -54,6 +54,7 @@ string TestCollator(json_object *json_in) {
   json_object *str1 = json_object_object_get(json_in, "s1");
   json_object *str2 = json_object_object_get(json_in, "s2");
 
+  // Unescape the input strings?
   string string1 = json_object_get_string(str1);
   string string2 = json_object_get_string(str2);
 
@@ -78,6 +79,11 @@ string TestCollator(json_object *json_in) {
   string compare_type_string = "";
   if (compare_type_obj != nullptr) {
     compare_type_string = json_object_get_string(compare_type_obj);
+    if (compare_type_string.substr(0,4) == "<") {
+      compare_type_string = "<" + compare_type_string.substr(4,1);
+    }
+    // !!!
+    cout << "COMPARE_TYPE_STRING: " << compare_type_string << endl;
   }
 
   // Strength of comparison
@@ -108,13 +114,6 @@ string TestCollator(json_object *json_in) {
   }
   UnicodeString uni_rules = UnicodeString::fromUTF8(rules_string);
 
-  // Allow for different levels or types of comparison.
-  json_object *compare_type = json_object_object_get(json_in, "compare_type");
-  if (compare_type != nullptr) {
-    // TODO: Apply this in tests.
-    const char *comparison_type = json_object_get_string(compare_type);
-  }
-
   // Handle some options
   json_object *ignore_obj =
       json_object_object_get(json_in, "ignorePunctuation");
@@ -153,9 +152,20 @@ string TestCollator(json_object *json_in) {
   } else {
     // Not a rule-based collator.
     if (strlen(locale_string) <= 0) {
+      // Uses the default Locale.
       uni_coll = Collator::createInstance(status);
     } else {
-      uni_coll = Collator::createInstance(Locale(locale_string), status);
+      Locale this_locale;
+      if (locale_string == "root") {
+        // !!!
+        cout << "&& ROOT LOCAL: " << locale_string << endl;
+        this_locale = Locale::getRoot();
+      } else {
+        // !!!
+        cout << "&& OTHER LOCAL: " << locale_string << endl;
+        this_locale = Locale(locale_string);
+      }
+      uni_coll = Collator::createInstance(this_locale, status);
     }
 
     if (check_icu_error(
@@ -209,7 +219,16 @@ string TestCollator(json_object *json_in) {
     }
   }
 
-  coll_result = (uni_result != UCOL_GREATER);
+  // Use the compare_type to see if "<" or "=" should be applied.
+  if (compare_type_string == "" || compare_type_string.substr(0, 1) == "<") {
+    // Default checking for <= 0.
+    coll_result = (uni_result != UCOL_GREATER);
+  } else {
+    // !!!
+    cout << "CHECK for EQUAL: " << compare_type_string << endl;
+    coll_result = (uni_result == UCOL_EQUAL);
+  }
+
   if (!coll_result) {
     // Test did not succeed!
     // Include data compared in the failing test

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/Icu4jExecutor.java

@@ -156,8 +156,8 @@ public static String getTestCaseResponse(String inputLine) throws Exception {
                 return ExecutorUtils.formatAsJson(
                     testType.convertOutputToMap(defaultOutput)
                         .put("label", parsedInputPersistentMap.get("label", null))
-                        .put("error", "Error in input" + e.getMessage())
-                        .put("error_message", "Error in handling test case: " + e.getMessage())
+                        .put("error", "Error in input" + t.getMessage())
+                        .put("error_message", "Error in handling test case: " + t.getMessage())
                 );
             }
         }

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorTester.java

@@ -43,7 +43,15 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
     result.ignorePunctuation = (boolean) inputMapData.get("ignorePunctuation", false);
     result.line = (int) ((double) inputMapData.get("line", 0.0));
 
+    // Resolve "&lt;"
     result.compare_type = (String) inputMapData.get("compare_type", null);
+    if (result.compare_type != null && ! result.compare_type.equals("") && result.compare_type.length() > 4) {
+      String first_part = result.compare_type.substring(0,4);
+      if (first_part.equals("&lt;")) {
+        String next_part = result.compare_type.substring(4,5);
+        result.compare_type = "<" + next_part;
+      }
+    }
     result.test_description = (String) inputMapData.get("test_description", null);
 
     // TODO: implement this correctly recursively (either using APIs or else DIY)
@@ -91,6 +99,7 @@ public ITestTypeOutputJson execute(ITestTypeInputJson inputJson) {
 
     try {
       int collResult = coll.compare(input.s1, input.s2);
+      // TODO! Use compare_type to check for <= or ==.
       if (collResult > 0) {
         // failure
         output.result = false;
@@ -138,7 +147,7 @@ public String formatOutputJson(ITestTypeOutputJson outputJson) {
   public Collator getCollatorForInput(CollatorInputJson input) {
     RuleBasedCollator result = null;
 
-    if (input.locale == null) {
+    if (input.locale == null || input.locale == "root") {
       if (input.rules == null) {
         result = (RuleBasedCollator) Collator.getInstance(ULocale.ROOT);
       } else {

executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/collator/icu74/CollatorTest.java

@@ -41,4 +41,13 @@ public void testAttributesAsArrayList() {
     assertTrue(output.result);
   }
 
+/*  @Test
+  public void testCompareLT2() {
+    String testInput =
+        "{\"test_type\": \"collation_short\", \"label\":\"00115\",\"s1\":\"cote\",\"s2\":\"coté\",\"line\":329,\"source_file\":\"collationtest.txt\",\"compare_type\":\"<2\",\"test_description\":\" discontiguous contractions\",\"hexhash\":\"b56b2f345f58f7044c14e392ea94304c075cbaf5\"}";
+    CollatorOutputJson output =
+        (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }*/
 }

schema/collation_short/test_schema.json

@@ -48,6 +48,10 @@
             "description": "If collation should ignore punctuatio",
             "type": "boolean"
           },
+          "source_file" : {
+            "description": "source file of test data",
+            "type": "string"
+          },
           "line" : {
             "description": "line of the source of test data",
             "type": "integer"
@@ -60,6 +64,14 @@
             "description": "Strength of comparison: primary, secondary, ... identical",
             "type": "string"
           },
+          "alternate": {
+            "description": "shifted, non-ignorable",
+            "type": "string"
+          },
+          "numeric": {
+            "description": "sort as a number value",
+            "type": "string"
+          },
           "test_description": {
             "description": "Option field telling about this particular test",
             "type": "string"
@@ -68,16 +80,20 @@
             "description": "instructions to reorder",
             "type": "string"
           },
-          "case_first": {
+          "caseFirst": {
             "description": "indicates how case should be considered",
             "type": "string"
           },
           "case_level": {
             "description": "indicates the level  case should be considered",
             "type": "string"
           },
-          "caseFirst": {
-            "description": "indicates how case should be considered",
+          "backwards": {
+            "description": "look at it backwards",
+            "type": "string"
+          },
+          "maxVariable": {
+            "description": "which variable is the maximum, e.g., currency",
             "type": "string"
           },
           "attributes": {

testgen/generators/base.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# -*- coding: utf_8 -*-
 from abc import ABC, abstractmethod
 import copy
 import hashlib
@@ -69,7 +69,7 @@ def generateTestHashValues(self, testdata):
 
             # Create the 32 byte hasn, consisten with Javascript
             hasher = hashlib.sha1()
-            hasher.update(test_no_label_string.encode("utf-8"))
+            hasher.update(test_no_label_string.encode("utf_8"))
             hex_digest = hasher.hexdigest()
             test['hexhash'] = hex_digest
 
@@ -83,7 +83,7 @@ def saveJsonFile(self, filename, data, indent=None):
                               filename)
 
         output_path = os.path.join(self.icu_version, filename)
-        output_file = open(output_path, "w", encoding="UTF-8")
+        output_file = open(output_path, "w", encoding="utf_8")
         json.dump(data, output_file, indent=indent)
         output_file.close()
 
@@ -133,7 +133,7 @@ def readFile(self, filename, version="", filetype="txt"):
         if version:
             path = os.path.join(version, filename)
         try:
-            with open(path, "r", encoding="utf-8") as testdata:
+            with open(path, "r", encoding="utf_8") as testdata:
                 return json.load(testdata) if filetype == "json" else testdata.read()
         except BaseException as err:
             logging.warning("** readFile: %s", err)