Implement UAX31's quick check algorithm

dlang · Apr 15, 2024 · a44fc2a · a44fc2a
1 parent e397d5e
commit a44fc2a
Show file tree

Hide file tree

Showing 23 changed files with 2,320 additions and 4,380 deletions.
diff --git a/changelog/dmd.identifier-quick-check.dd b/changelog/dmd.identifier-quick-check.dd
@@ -0,0 +1,11 @@
+Identifiers are verifiable against normalization form C
+
+It is now possible to verify identifiers against normalization form C, as per UAX31 and C23.
+
+The quick check algorithm that is in use is an incomplete form that does not handle maybes and can result in false positives in errors.
+The purpose of this algorithm in its incomplete state is to minimize normalization in a future update.
+
+It is configurable for both D and ImportC using ``-normalization=<strategy>`` and ``-normalization-importc=<strategy>``.
+Using the options ``ignore``, ``deprecate``, ``warn``, and ``error``.
+
+Unless you use non-ASCII characters, this change will not affect you.
diff --git a/compiler/src/dmd/cli.d b/compiler/src/dmd/cli.d
@@ -639,6 +639,26 @@ dmd -cov -unittest myprog.d
             `Turns off all array bounds checking, even for safe functions. $(RED Deprecated
             (use $(TT $(SWLINK -boundscheck)=off) instead).)`,
         ),
+        Option("normalization=<strategy>",
+            "Specify what to do when an identifier is not normalized",
+            `The strategy that will be used when an identifier may not be normalized (quick check algorithm).
+                $(UL
+                    $(LI $(I ignore): Silently accept any failing identifiers (default))
+                    $(LI $(I deprecate): Emit a deprecation but accept)
+                    $(LI $(I warn): Emit a warning but accept)
+                    $(LI $(I error): Emit an error and fail)
+                )`
+        ),
+        Option("normalization-importc=<strategy>",
+            "Specify what to do when an identifier is not normalized",
+            `The strategy that will be used when an identifier may not be normalized (quick check algorithm).
+                $(UL
+                    $(LI $(I ignore): Silently accept any failing identifiers (default))
+                    $(LI $(I deprecate): Emit a deprecation but accept)
+                    $(LI $(I warn): Emit a warning but accept)
+                    $(LI $(I error): Emit an error and fail)
+                )`
+        ),
         Option("nothrow",
             "assume no Exceptions will be thrown",
             `Turns off generation of exception stack unwinding code, enables