CodePointTrie data provider

components/properties/Cargo.toml

@@ -36,6 +36,7 @@ icu_provider = { version = "0.3", path = "../../provider/core", features = ["mac
 icu_uniset = { version = "0.3", path = "../../utils/uniset", features = ["serde"] }
 num_enum = { version = "0.5.4", default-features = false }
 serde = { version = "1.0", default-features = false, features = ["derive", "alloc"], optional = true }
+tinystr = { version = "0.4.10", features = ["alloc", "serde"], default-features = false }
 zerovec = { version = "0.3", path = "../../utils/zerovec", features = ["serde"] }
 
 [dev-dependencies]

components/properties/src/props.rs

@@ -5,19 +5,34 @@
 //! A collection of enums for enumerated properties.
 
 use num_enum::{TryFromPrimitive, UnsafeFromPrimitive};
+use tinystr::TinyStr16;
 
 /// Selection constants for Unicode properties.
 /// These constants are used to select one of the Unicode properties.
 /// See `UProperty` in ICU4C.
 #[derive(Clone, PartialEq, Debug)]
 #[non_exhaustive]
+#[repr(i32)]
 pub enum EnumeratedProperty {
     /// The General Category property.
     GeneralCategory = 0x1005,
     /// The Script property. See [`Script`].
     Script = 0x100A,
     /// The Script_Extensions property. See [`Script`].
-    ScriptExtensions = 0x7000,
+    ScriptExtensions = 0x7000, // TODO(#1160) - this is a Miscellaneous property, not Enumerated
+    /// Represents an invalid or unknown Unicode property.
+    InvalidCode = -1, // TODO(#1160) - taken from ICU4C UProperty::UCHAR_INVALID_CODE
+}
+
+impl From<&TinyStr16> for EnumeratedProperty {
+    fn from(prop_short_alias: &TinyStr16) -> Self {
+        match prop_short_alias.as_str() {
+            "gc" => EnumeratedProperty::GeneralCategory,
+            "sc" => EnumeratedProperty::Script,
+            "scx" => EnumeratedProperty::ScriptExtensions,
+            _ => EnumeratedProperty::InvalidCode,
+        }
+    }
 }
 
 /// Enumerated Unicode general category types.

docs/tutorials/writing_a_new_data_struct.md

@@ -60,7 +60,7 @@ Examples of source data providers include:
     - [`PluralsProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/struct.PluralsProvider.html)
     - [`DateSymbolsProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/struct.DateSymbolsProvider.html)
     - [… more examples](https://unicode-org.github.io/icu4x-docs/doc/icu_provider_cldr/transform/index.html)
-- `BinaryPropertiesDataProvider`
+- `BinaryPropertyUnicodeSetDataProvider`
 - [`HelloWorldProvider`](https://unicode-org.github.io/icu4x-docs/doc/icu_provider/hello_world/struct.HelloWorldProvider.html)
 
 Source data providers must implement the following traits:

provider/uprops/Cargo.toml

@@ -28,11 +28,13 @@ all-features = true
 
 [dependencies]
 displaydoc = { version = "0.2.3", default-features = false }
+icu_codepointtrie = { version = "0.3", path = "../../utils/codepointtrie", features = ["provider_serde"] }
 icu_properties = { version = "0.3", path = "../../components/properties", features = ["provider_serde"] }
 icu_provider = { version = "0.3", path = "../../provider/core", features = ["provider_serde"] }
 icu_uniset = { version = "0.3", path = "../../utils/uniset", features = ["provider_serde"] }
 serde = { version = "1.0", features = ["derive"] }
 toml = { version = "0.5" }
+zerovec = { version = "0.3", path = "../../utils/zerovec", features = ["serde", "yoke"] }
 
 [dev-dependencies]
 icu_testdata = { version = "0.3", path = "../../provider/testdata" }

provider/uprops/src/binary.rs → provider/uprops/src/bin_uniset.rs

@@ -12,14 +12,14 @@ use icu_uniset::UnicodeSetBuilder;
 use std::fs;
 use std::path::PathBuf;
 
-pub struct BinaryPropertiesDataProvider {
+pub struct BinaryPropertyUnicodeSetDataProvider {
     root_dir: PathBuf,
 }
 
 /// A data provider reading from .toml files produced by the ICU4C icuwriteuprops tool.
-impl BinaryPropertiesDataProvider {
+impl BinaryPropertyUnicodeSetDataProvider {
     pub fn new(root_dir: PathBuf) -> Self {
-        BinaryPropertiesDataProvider { root_dir }
+        BinaryPropertyUnicodeSetDataProvider { root_dir }
     }
     fn get_toml_data(&self, name: &str) -> Result<uprops_serde::binary::Main, Error> {
         let mut path: PathBuf = self.root_dir.clone().join(name);
@@ -29,7 +29,7 @@ impl BinaryPropertiesDataProvider {
     }
 }
 
-impl<'data> DataProvider<'data, UnicodePropertyV1Marker> for BinaryPropertiesDataProvider {
+impl<'data> DataProvider<'data, UnicodePropertyV1Marker> for BinaryPropertyUnicodeSetDataProvider {
     fn load_payload(
         &self,
         req: &DataRequest,
@@ -55,11 +55,11 @@ impl<'data> DataProvider<'data, UnicodePropertyV1Marker> for BinaryPropertiesDat
     }
 }
 
-icu_provider::impl_dyn_provider!(BinaryPropertiesDataProvider, {
+icu_provider::impl_dyn_provider!(BinaryPropertyUnicodeSetDataProvider, {
     _ => UnicodePropertyV1Marker,
 }, SERDE_SE, 'data);
 
-impl IterableDataProviderCore for BinaryPropertiesDataProvider {
+impl IterableDataProviderCore for BinaryPropertyUnicodeSetDataProvider {
     fn supported_options_for_key(
         &self,
         _resc_key: &ResourceKey,
@@ -76,7 +76,7 @@ fn test_basic() {
     use std::convert::TryInto;
 
     let root_dir = icu_testdata::paths::data_root().join("uprops");
-    let provider = BinaryPropertiesDataProvider::new(root_dir);
+    let provider = BinaryPropertyUnicodeSetDataProvider::new(root_dir);
 
     let payload: DataPayload<'_, UnicodePropertyV1Marker> = provider
         .load_payload(&DataRequest {

provider/uprops/src/enum_codepointtrie.rs

@@ -0,0 +1,113 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+use crate::error::Error;
+use crate::uprops_serde;
+use crate::uprops_serde::enumerated::EnumeratedPropertyCodePointTrie;
+
+use icu_codepointtrie::codepointtrie::{CodePointTrie, CodePointTrieHeader, TrieType, TrieValue};
+use icu_codepointtrie::provider::{UnicodePropertyMapV1, UnicodePropertyMapV1Marker};
+use icu_provider::prelude::*;
+use zerovec::ZeroVec;
+
+use core::convert::TryFrom;
+
+use std::fs;
+use std::path::PathBuf;
+
+pub struct EnumeratedPropertyCodePointTrieProvider {
+    root_dir: PathBuf,
+}
+
+impl EnumeratedPropertyCodePointTrieProvider {
+    pub fn _new(root_dir: PathBuf) -> Self {
+        EnumeratedPropertyCodePointTrieProvider { root_dir }
+    }
+
+    fn get_toml_data(&self, name: &str) -> Result<uprops_serde::enumerated::Main, Error> {
+        let mut path: PathBuf = self.root_dir.clone().join(name);
+        path.set_extension("toml");
+        let toml_str = fs::read_to_string(&path).map_err(|e| Error::Io(e, path.clone()))?;
+        toml::from_str(&toml_str).map_err(|e| Error::Toml(e, path))
+    }
+}
+
+impl<T: TrieValue> TryFrom<uprops_serde::enumerated::EnumeratedPropertyCodePointTrie>
+    for UnicodePropertyMapV1<'static, T>
+{
+    type Error = DataError;
+
+    fn try_from(
+        cpt_data: EnumeratedPropertyCodePointTrie,
+    ) -> Result<UnicodePropertyMapV1<'static, T>, DataError> {
+        let trie_type_enum: TrieType =
+            TrieType::try_from(cpt_data.trie_type_enum_val).map_err(DataError::new_resc_error)?;
+        let header = CodePointTrieHeader {
+            high_start: cpt_data.high_start,
+            shifted12_high_start: cpt_data.shifted12_high_start,
+            index3_null_offset: cpt_data.index3_null_offset,
+            data_null_offset: cpt_data.data_null_offset,
+            null_value: cpt_data.null_value,
+            trie_type: trie_type_enum,
+        };
+        let index: ZeroVec<u16> = ZeroVec::clone_from_slice(&cpt_data.index);
+        let data: Result<ZeroVec<'static, T>, String> = if let Some(data_8) = cpt_data.data_8 {
+            data_8
+                .iter()
+                .map(|i| T::parse_from_u32(*i as u32))
+                .collect()
+        } else if let Some(data_16) = cpt_data.data_16 {
+            data_16
+                .iter()
+                .map(|i| T::parse_from_u32(*i as u32))
+                .collect()
+        } else if let Some(data_32) = cpt_data.data_32 {
+            data_32
+                .iter()
+                .map(|i| T::parse_from_u32(*i as u32))
+                .collect()
+        } else {
+            return Err(DataError::new_resc_error(
+                icu_codepointtrie::error::Error::FromDeserialized {
+                    reason: "Cannot deserialize data array for CodePointTrie in TOML",
+                },
+            ));
+        };
+
+        let data = data.map_err(DataError::new_resc_error)?;
+        let trie =
+            CodePointTrie::<T>::try_new(header, index, data).map_err(DataError::new_resc_error);
+        trie.map(|t| UnicodePropertyMapV1 { codepoint_trie: t })
+    }
+}
+
+impl<'data, T: TrieValue> DataProvider<'data, UnicodePropertyMapV1Marker<T>>
+    for EnumeratedPropertyCodePointTrieProvider
+{
+    fn load_payload(
+        &self,
+        req: &DataRequest,
+    ) -> Result<DataResponse<'data, UnicodePropertyMapV1Marker<T>>, DataError> {
+        // For data resource keys that represent the CodePointTrie data for an enumerated
+        // property, the ResourceKey sub-category string will just be the short alias
+        // for the property.
+        let prop_name = &req.resource_path.key.sub_category;
+
+        let toml_data: uprops_serde::enumerated::Main = self
+            .get_toml_data(prop_name)
+            .map_err(DataError::new_resc_error)?;
+
+        let source_cpt_data: uprops_serde::enumerated::EnumeratedPropertyCodePointTrie =
+            toml_data.enum_property.data.code_point_trie;
+
+        let data_struct = UnicodePropertyMapV1::<T>::try_from(source_cpt_data)?;
+
+        Ok(DataResponse {
+            metadata: DataResponseMetadata {
+                data_langid: req.resource_path.options.langid.clone(),
+            },
+            payload: Some(DataPayload::from_owned(data_struct)),
+        })
+    }
+}

provider/uprops/src/enumerated.rs → provider/uprops/src/enum_uniset.rs

@@ -12,14 +12,14 @@ use icu_uniset::UnicodeSetBuilder;
 use std::fs;
 use std::path::PathBuf;
 
-pub struct EnumeratedPropertiesDataProvider {
+pub struct EnumeratedPropertyUnicodeSetDataProvider {
     root_dir: PathBuf,
 }
 
 /// A data provider reading from .toml files produced by the ICU4C icuwriteuprops tool.
-impl EnumeratedPropertiesDataProvider {
+impl EnumeratedPropertyUnicodeSetDataProvider {
     pub fn new(root_dir: PathBuf) -> Self {
-        EnumeratedPropertiesDataProvider { root_dir }
+        EnumeratedPropertyUnicodeSetDataProvider { root_dir }
     }
     fn get_toml_data(&self, name: &str) -> Result<uprops_serde::enumerated::Main, Error> {
         let mut path: PathBuf = self.root_dir.clone().join(name);
@@ -62,7 +62,9 @@ fn expand_groupings<'a>(prop_name: &str, prop_val: &'a str) -> Vec<&'a str> {
     }
 }
 
-impl<'data> DataProvider<'data, UnicodePropertyV1Marker> for EnumeratedPropertiesDataProvider {
+impl<'data> DataProvider<'data, UnicodePropertyV1Marker>
+    for EnumeratedPropertyUnicodeSetDataProvider
+{
     fn load_payload(
         &self,
         req: &DataRequest,
@@ -105,11 +107,11 @@ impl<'data> DataProvider<'data, UnicodePropertyV1Marker> for EnumeratedPropertie
     }
 }
 
-icu_provider::impl_dyn_provider!(EnumeratedPropertiesDataProvider, {
+icu_provider::impl_dyn_provider!(EnumeratedPropertyUnicodeSetDataProvider, {
     _ => UnicodePropertyV1Marker,
 }, SERDE_SE, 'data);
 
-impl IterableDataProviderCore for EnumeratedPropertiesDataProvider {
+impl IterableDataProviderCore for EnumeratedPropertyUnicodeSetDataProvider {
     fn supported_options_for_key(
         &self,
         _resc_key: &ResourceKey,
@@ -130,7 +132,7 @@ mod tests {
         use std::convert::TryInto;
 
         let root_dir = icu_testdata::paths::data_root().join("uprops");
-        let provider = EnumeratedPropertiesDataProvider::new(root_dir);
+        let provider = EnumeratedPropertyUnicodeSetDataProvider::new(root_dir);
 
         let payload: DataPayload<'_, UnicodePropertyV1Marker> = provider
             .load_payload(&DataRequest {
@@ -158,7 +160,7 @@ mod tests {
         use std::convert::TryInto;
 
         let root_dir = icu_testdata::paths::data_root().join("uprops");
-        let provider = EnumeratedPropertiesDataProvider::new(root_dir);
+        let provider = EnumeratedPropertyUnicodeSetDataProvider::new(root_dir);
 
         let payload: DataPayload<'_, UnicodePropertyV1Marker> = provider
             .load_payload(&DataRequest {
@@ -189,7 +191,7 @@ mod tests {
             key: ResourceKey,
         ) -> DataPayload<'data, UnicodePropertyV1Marker> {
             let root_dir = icu_testdata::paths::data_root().join("uprops");
-            let provider = EnumeratedPropertiesDataProvider::new(root_dir);
+            let provider = EnumeratedPropertyUnicodeSetDataProvider::new(root_dir);
             let payload: DataPayload<'_, UnicodePropertyV1Marker> = provider
                 .load_payload(&DataRequest {
                     resource_path: ResourcePath {
@@ -301,7 +303,7 @@ mod tests {
         use std::convert::TryInto;
 
         let root_dir = icu_testdata::paths::data_root().join("uprops");
-        let provider = EnumeratedPropertiesDataProvider::new(root_dir);
+        let provider = EnumeratedPropertyUnicodeSetDataProvider::new(root_dir);
 
         let payload: DataPayload<'_, UnicodePropertyV1Marker> = provider
             .load_payload(&DataRequest {

provider/uprops/src/lib.rs

@@ -17,10 +17,11 @@
 //! [`FsDataProvider`]: ../icu_provider_fs/struct.FsDataProvider.html
 //! [`StaticDataProvider`]: ../icu_provider_blob/struct.StaticDataProvider.html
 
-mod binary;
-mod enumerated;
+mod bin_uniset;
+mod enum_codepointtrie;
+mod enum_uniset;
 mod error;
-mod provider;
 mod uprops_serde;
 
-pub use provider::PropertiesDataProvider;
+pub use bin_uniset::BinaryPropertyUnicodeSetDataProvider;
+pub use enum_uniset::EnumeratedPropertyUnicodeSetDataProvider;