Ucdxml 17v1 (#1104)

* Initial checkin to support Unikemet

* Added note about grouping Unihan attributes, and more work for Unikemet

* Removed Deprecated properties, removed sections that only contained historical data, and added partial support for kEH_AltSeq

* Updated version information

* Renamed index to tr42 to make it easier to copy to unicode-reports

* Added removed/changed support

* kGB7 is a removed property

* Latest Unihan, Tangut, and Nushu changes

Author: jowilco

docs/ucdxml.md

@@ -64,6 +64,12 @@ We'll use [jing-trang](https://github.com/relaxng/jing-trang) in this example.
 1. Clone and build [jing-trang](https://github.com/relaxng/jing-trang)
 2. Run the following:
     ```
-   java -jar C:\_git\jing-trang\build\jing.jar -c UNICODETOOLS_REPO_DIR\uax\uax42\output\index.rnc <path to UAX xml file>
+   java -jar C:\_git\jing-trang\build\jing.jar -c UNICODETOOLS_REPO_DIR\uax\uax42\output\tr42.rnc <path to UAX xml file>
    ```
    Note that the UAX xml file has to be saved as NFD as the Unihan syntax regular expressions are expecting NFD.
+   
+   To convert to NFD, use ICU's uconv.exe:
+   ```
+   uconv.exe uconv -f utf8 -t utf8 -x nfd -o {outputfile} {originalfile}
+   ```
+

unicodetools/data/ucdxml/dev/ucd.nounihan.grouped.xml

@@ -1,19 +1,18 @@
-XML Representation of Unicode 16.0.0 UCD
+XML Representation of Unicode 17.0.0 UCD
 
 
-© 2024 Unicode®, Inc.
+© 2025 Unicode®, Inc.
 For terms of use, see https://www.unicode.org/terms_of_use.html
 
 
-This directory contains the representation in XML of Version 16.0.0 of
+This directory contains the representation in XML of Version 17.0.0 of
 the UCD, using the schema defined by UAX #42: Unicode Character
 Database in XML, at https://www.unicode.org/reports/tr42/
 
 While every effort has been made to ensure consistency of the 
 XML representation with the UCD files, there may be some errors;
 the UCD files are authoritative.
 
-
 There are six files, available in zip/jar format:
   - flat vs. grouped
   - no Unihan data vs. Unihan data only vs. complete UCD. 

unicodetools/data/ucdxml/dev/ucd.unihan.grouped.xml

@@ -217,11 +217,36 @@ public enum UcdProperty {
     kEACC(PropertyType.Miscellaneous, DerivedPropertyStatus.Provisional, "cjkEACC"),
     kEH_Cat(PropertyType.Miscellaneous, DerivedPropertyStatus.Approved, "kEH_Cat"),
     kEH_Desc(PropertyType.Miscellaneous, DerivedPropertyStatus.Approved, "kEH_Desc"),
-    kEH_FVal(PropertyType.Miscellaneous, DerivedPropertyStatus.Provisional, "kEH_FVal"),
-    kEH_Func(PropertyType.Miscellaneous, DerivedPropertyStatus.Provisional, "kEH_Func"),
-    kEH_HG(PropertyType.Miscellaneous, DerivedPropertyStatus.Approved, "kEH_HG"),
-    kEH_IFAO(PropertyType.Miscellaneous, DerivedPropertyStatus.Approved, "kEH_IFAO"),
-    kEH_JSesh(PropertyType.Miscellaneous, DerivedPropertyStatus.Approved, "kEH_JSesh"),
+    kEH_FVal(
+            PropertyType.Miscellaneous,
+            DerivedPropertyStatus.Provisional,
+            null,
+            ValueCardinality.Unordered,
+            "kEH_FVal"),
+    kEH_Func(
+            PropertyType.Miscellaneous,
+            DerivedPropertyStatus.Provisional,
+            null,
+            ValueCardinality.Unordered,
+            "kEH_Func"),
+    kEH_HG(
+            PropertyType.Miscellaneous,
+            DerivedPropertyStatus.Approved,
+            null,
+            ValueCardinality.Unordered,
+            "kEH_HG"),
+    kEH_IFAO(
+            PropertyType.Miscellaneous,
+            DerivedPropertyStatus.Approved,
+            null,
+            ValueCardinality.Unordered,
+            "kEH_IFAO"),
+    kEH_JSesh(
+            PropertyType.Miscellaneous,
+            DerivedPropertyStatus.Approved,
+            null,
+            ValueCardinality.Unordered,
+            "kEH_JSesh"),
     kEH_UniK(PropertyType.Miscellaneous, DerivedPropertyStatus.Provisional, "kEH_UniK"),
     kFanqie(
             PropertyType.Miscellaneous,

unicodetools/data/ucdxml/dev/ucdxml.readme.txt

@@ -22,6 +22,7 @@
  */
 public class AttributeResolver {
 
+    static final String SET_SEPARATOR = "|";
     private final IndexUnicodeProperties indexUnicodeProperties;
     private final UnicodeMap<UcdPropertyValues.Age_Values> map_age;
     private final UnicodeMap<UcdPropertyValues.Block_Values> map_block;
@@ -147,6 +148,9 @@ public String getAttributeValue(UcdProperty prop, int codepoint) {
                     case kOtherNumeric:
                     case kPrimaryNumeric:
                     case kAccountingNumeric:
+                        if (resolvedValue != null) {
+                            resolvedValue = resolvedValue.replaceAll("\\" + SET_SEPARATOR, " ");
+                        }
                         return (resolvedValue.equals("NaN")) ? null : resolvedValue;
                     default:
                         return Optional.ofNullable(resolvedValue).orElse("NaN");
@@ -211,7 +215,7 @@ public String getAttributeValue(UcdProperty prop, int codepoint) {
                         return resolvedValue;
                     default:
                         if (resolvedValue != null) {
-                            return resolvedValue.replaceAll("\\|", " ");
+                            return resolvedValue.replaceAll("\\" + SET_SEPARATOR, " ");
                         }
                         return "";
                 }
@@ -226,7 +230,8 @@ public String getAttributeValue(UcdProperty prop, int codepoint) {
                         return map_script.get(codepoint).getShortName();
                     case Script_Extensions:
                         StringBuilder extensionBuilder = new StringBuilder();
-                        String[] extensions = map_script_extensions.get(codepoint).split("\\|", 0);
+                        String[] extensions =
+                                map_script_extensions.get(codepoint).split("\\" + SET_SEPARATOR, 0);
                         for (String extension : extensions) {
                             extensionBuilder.append(
                                     UcdPropertyValues.Script_Values.valueOf(extension)
@@ -348,4 +353,8 @@ public boolean isUnifiedIdeograph(int codepoint) {
         return getAttributeValue(UcdProperty.Unified_Ideograph, codepoint).equals("Y")
                 && getAttributeValue(UcdProperty.Name, codepoint).equals("CJK UNIFIED IDEOGRAPH-#");
     }
+
+    public boolean isUnikemetAttributeRange(int codepoint) {
+        return !getAttributeValue(UcdProperty.kEH_Cat, codepoint).isEmpty();
+    }
 }