Ucdxml and TR42

.gitignore

@@ -43,6 +43,7 @@ perf-*.xml
 test-*.xml
 
 # Directories
+.idea/
 .settings/
 .vs/
 .vscode/

docs/ucdxml.md

@@ -0,0 +1,22 @@
+# Generating TR42
+
+## Step 1 - Generate property value fragments
+
+- mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.GeneratePropertyValues"' '-Dexec.args="--ucdversion 16.0.0 -f $(cd ./unicodetools/src/main/resources/org/unicode/uax42/fragments; pwd)"' -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+
+## Step 2 - Generate TR42 index.html and index.rnc 
+
+- mvn xml:transform -f $(cd ./unicodetools/src/main/resources/org/unicode/uax42/fragments; pwd) -Doutputdir=../Generated/uax42/
+
+## Step 3 - Validate generated UAX XML files
+
+You'll need a [RELAX NG](https://relaxng.org/) schema validator. We'll use [jing-trang](https://github.
+com/relaxng/jing-trang) in this example.
+
+1. Clone and build [jing-trang](https://github.com/relaxng/jing-trang)
+2. Run the following:
+    ```
+   java -jar C:\_git\jing-trang\build\jing.jar -c UNICODETOOLS_REPO_DIR\uax\uax42\output\index.rnc <path to UAX xml file>
+   ```
+   Note that the UAX xml file has to be saved as NFD as the Unihan syntax regular expressions are expecting NFD.
+

unicodetools/src/main/java/org/unicode/props/PropertyParsingInfo.java

@@ -760,10 +760,20 @@ private static void parsePropertyValueFile(
                 assert propInfo.property.getType() == PropertyType.Binary;
                 value = "Yes";
             } else {
-                value =
-                        propInfo.property.getType() == PropertyType.Binary
-                                ? "Yes"
-                                : line.getParts()[2];
+                if (propInfo.property.getType() == PropertyType.Binary) {
+                    // Handle @missing values for binary attributes (see 13.0.0 emoji-data.txt)
+                    if (line.getParts().length == 3) {
+                        if (line.getParts()[2].equals("No")) {
+                            value = "No";
+                        } else {
+                            value = "Yes";
+                        }
+                    } else {
+                        value = "Yes";
+                    }
+                } else {
+                    value = line.getParts()[2];
+                }
                 // The value should not be an empty string.
                 // Exception: NFKC_Casefold does remove some characters by mapping them to nothing.
                 assert !value.isEmpty()

unicodetools/src/main/java/org/unicode/props/UcdProperty.java

@@ -85,12 +85,16 @@ public enum UcdProperty {
     Emoji_SB(PropertyType.Miscellaneous, "ESB"),
     ISO_Comment(PropertyType.Miscellaneous, "isc"),
     Jamo_Short_Name(PropertyType.Miscellaneous, "JSN"),
+    NC_Corrected(PropertyType.Miscellaneous, "ncCorrected"),
+    NC_Original(PropertyType.Miscellaneous, "ncOriginal"),
+    NC_Version(PropertyType.Miscellaneous, "ncVersion"),
     Name(PropertyType.Miscellaneous, "na"),
     Name_Alias(PropertyType.Miscellaneous, null, ValueCardinality.Unordered, "Name_Alias"),
     Named_Sequences(PropertyType.Miscellaneous, "NS"),
     Named_Sequences_Prov(PropertyType.Miscellaneous, "NSP"),
     Standardized_Variant(PropertyType.Miscellaneous, null, ValueCardinality.Unordered, "SV"),
     Unicode_1_Name(PropertyType.Miscellaneous, "na1"),
+    emoji_variation_sequence(PropertyType.Miscellaneous, "EVS"),
     kAlternateTotalStrokes(PropertyType.Miscellaneous, "cjkAlternateTotalStrokes"),
     kBigFive(PropertyType.Miscellaneous, "cjkBigFive"),
     kCCCII(PropertyType.Miscellaneous, "cjkCCCII"),
@@ -225,6 +229,7 @@ public enum UcdProperty {
     kXHC1983(PropertyType.Miscellaneous, null, ValueCardinality.Unordered, "cjkXHC1983"),
     kXerox(PropertyType.Miscellaneous, "cjkXerox"),
     kZVariant(PropertyType.Miscellaneous, "cjkZVariant"),
+    kZhuang(PropertyType.Miscellaneous, "cjkZhuang"),
     kZhuangNumeric(PropertyType.Miscellaneous, "cjkZhuangNumeric"),
 
     // Catalog

unicodetools/src/main/java/org/unicode/props/UcdPropertyValues.java

@@ -757,6 +757,7 @@ public static East_Asian_Width_Values forName(String name) {
     // Emoji_DCM
     // Emoji_KDDI
     // Emoji_SB
+    // emoji_variation_sequence
     // Equivalent_Unified_Ideograph
     // FC_NFKC_Closure
     public enum General_Category_Values implements Named {
@@ -1569,6 +1570,7 @@ public static kEH_Core_Values forName(String name) {
     // kVietnameseNumeric
     // kXerox
     // kXHC1983
+    // kZhuang
     // kZhuangNumeric
     // kZVariant
     public enum Line_Break_Values implements Named {
@@ -1651,6 +1653,9 @@ public static Line_Break_Values forName(String name) {
     // Name_Alias
     // Named_Sequences
     // Named_Sequences_Prov
+    // NC_Corrected
+    // NC_Original
+    // NC_Version
     public enum NFC_Quick_Check_Values implements Named {
         Maybe("M"),
         No("N"),