From 34f5f0cee5cacc1d438a05833459feabd5cdf767 Mon Sep 17 00:00:00 2001
From: Craig Cornelius <ccornelius@google.com>
Date: Fri, 17 Nov 2023 17:17:22 -0800
Subject: [PATCH] Advanced collation - fixes many NodeJS collation failures
 (#124)

* Resolve diff in node version

* Adding ICU74 test data and collationtest.txt for older versions

* Fix error in collation parameters. fix rust formatting.

* Fix collation call in Node - removes 85% of test failures
---
 executors/node/collator.js | 2 +-
 testgen/testdata_gen.py    | 3 +++
 verifier/testreport.py     | 9 +++++++++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/executors/node/collator.js b/executors/node/collator.js
index 02732307..e49ac336 100644
--- a/executors/node/collator.js
+++ b/executors/node/collator.js
@@ -14,7 +14,7 @@ module.exports = {
     let testCollOptions = {};
     if ('ignorePunctuation' in json) {
       testCollOptions = {
-        ignorePunctuation:json['ignorePunction']}
+        ignorePunctuation:json['ignorePunctuation']}
     }
 
     // Get other fields if provided
diff --git a/testgen/testdata_gen.py b/testgen/testdata_gen.py
index 56d48b06..1b02c0b2 100644
--- a/testgen/testdata_gen.py
+++ b/testgen/testdata_gen.py
@@ -734,6 +734,9 @@ def generateCollTestData2(filename,
                     compare_type = is_comparison.group(1)
                     compare_string = is_comparison.group(2)
                     # Note that this doesn't seem to handle \x encoding, howeveer.
+                    string2 = compare_string.encode().decode('unicode_escape')
+                    compare_comment = is_comparison.group(3)
+                    # Generate the test case
                     try:
                         s = compare_string.encode()
                         string2 = s.decode('unicode_escape')
diff --git a/verifier/testreport.py b/verifier/testreport.py
index f923f24d..a7f1156b 100644
--- a/verifier/testreport.py
+++ b/verifier/testreport.py
@@ -577,6 +577,15 @@ def characterize_failures_by_options(self, failing_tests):
                     except:
                         continue
 
+                for key in ['language_label', 'ignorePunctuation', 'compare_result', 'compare_type', 'test_description']:
+                    if test.get(key):  # For collation results
+                        value = test[key]
+                        if key not in results:
+                            results[key] = {}
+                        if value in results[key]:
+                            results[key][value].append(label)
+                        else:
+                            results[key][value] = [label]
                 for key in ['ignorePunctuation']:
                     try:
                         if (test.get('input_data') and test['input_data'].get(key)):  # For collation results