Replace OmnibusDataProvider with make_forking_provider (#1788)

provider/core/src/datagen/mod.rs

@@ -14,27 +14,3 @@ mod iter;
 pub use data_conversion::{DataConverter, ReturnedPayloadError};
 pub use heap_measure::{HeapStats, HeapStatsMarker};
 pub use iter::{IterableDynProvider, IterableResourceProvider};
-
-use crate::any::AnyMarker;
-use crate::buf::BufferMarker;
-use crate::serde::SerializeMarker;
-use crate::DataMarker;
-
-/// A blanket-implemented trait that allows for all the datagen-relevant traits
-/// to be used in trait objects. The type parameter is the marker type needed for
-/// [`IterableDynProvider`].
-pub trait OmnibusDatagenProvider<M: DataMarker>:
-    DataConverter<AnyMarker, SerializeMarker>
-    + DataConverter<BufferMarker, HeapStatsMarker>
-    + IterableDynProvider<M>
-{
-}
-
-impl<M, P> OmnibusDatagenProvider<M> for P
-where
-    P: DataConverter<AnyMarker, SerializeMarker>
-        + DataConverter<BufferMarker, HeapStatsMarker>
-        + IterableDynProvider<M>,
-    M: DataMarker,
-{
-}

provider/datagen/src/bin/datagen.rs

@@ -4,19 +4,18 @@
 
 use clap::{App, Arg, ArgGroup, ArgMatches};
 use eyre::WrapErr;
-use icu_datagen::cldr;
+
 use icu_datagen::cldr::CldrPathsAllInOne;
-use icu_datagen::get_all_keys;
-use icu_datagen::segmenter;
-use icu_datagen::uprops;
+use icu_datagen::{get_all_keys, DatagenOptions};
+
 use icu_locid::LanguageIdentifier;
 use icu_provider::datagen::IterableDynProvider;
 use icu_provider::export::DataExporter;
 use icu_provider::hello_world::{HelloWorldProvider, HelloWorldV1Marker};
 use icu_provider::prelude::*;
 use icu_provider::serde::SerializeMarker;
 use icu_provider_adapters::filter::Filterable;
-use icu_provider_adapters::fork::by_key::MultiForkByKeyProvider;
+
 use icu_provider_blob::export::BlobExporter;
 use icu_provider_fs::export::fs_exporter;
 use icu_provider_fs::export::serializers;
@@ -358,19 +357,14 @@ fn main() -> eyre::Result<()> {
 
         let segmenter_data_root = icu_datagen::segmenter::segmenter_data_root();
 
-        Box::new(MultiForkByKeyProvider {
-            providers: vec![
-                Box::new(cldr::create_exportable_provider(
-                    &cldr_paths,
-                    uprops_root.clone(),
-                )?),
-                Box::new(uprops::create_exportable_provider(&uprops_root)?),
-                Box::new(segmenter::create_exportable_provider(
-                    &segmenter_data_root,
-                    &uprops_root,
-                )?),
-            ],
-        })
+        let options = DatagenOptions {
+            cldr_paths: Some(Box::new(cldr_paths)),
+            uprops_root: Some(uprops_root),
+            segmenter_data_root: Some(segmenter_data_root),
+        };
+        let p = icu_datagen::create_datagen_provider!(options);
+
+        Box::new(p)
     };
 
     if let Some(locales) = selected_locales.as_ref() {

provider/datagen/src/bin/verify-zero-copy.rs

@@ -3,7 +3,7 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use clap::{App, Arg, ArgGroup};
-use icu_datagen::get_all_keys;
+use icu_datagen::{get_all_keys, DatagenOptions};
 use icu_provider::datagen::IterableDynProvider;
 use icu_provider::datagen::{DataConverter, HeapStatsMarker};
 use icu_provider_adapters::filter::Filterable;
@@ -176,7 +176,12 @@ fn main() -> eyre::Result<()> {
 
     let segmenter_data_root = icu_datagen::segmenter::segmenter_data_root();
 
-    let converter = icu_datagen::get_registry(&*cldr_paths, &uprops_root, &segmenter_data_root)?;
+    let options = DatagenOptions {
+        cldr_paths: Some(cldr_paths),
+        uprops_root: Some(uprops_root),
+        segmenter_data_root: Some(segmenter_data_root),
+    };
+    let converter = icu_datagen::create_datagen_provider!(options);
 
     let selected_locales = icu_testdata::metadata::load()?.package_metadata.locales;
 

provider/datagen/src/cldr/mod.rs

@@ -3,8 +3,10 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 //! `icu_datagen::cldr` contains implementations of the [`ICU4X`] [data provider] interface
-//! based on the JSON files shipped by CLDR. Create a [`CldrPaths`] and then pass it into
-//! [`create_exportable_provider`].
+//! based on the JSON files shipped by CLDR.
+//!
+//! This module exports feature-specific providers. Use [`crate::create_datagen_provider`]
+//! for an all-inclusive provider.
 //!
 //! This crate contains two implementations of [`CldrPaths`]:
 //!
@@ -28,51 +30,17 @@ pub use cldr_paths::CldrPathsAllInOne;
 pub use cldr_paths::CldrPathsLocal;
 pub use error::Error as CldrError;
 
-use icu_provider::datagen::OmnibusDatagenProvider;
 use icu_provider::prelude::*;
-use icu_provider_adapters::fork::by_key::MultiForkByKeyProvider;
-use std::convert::TryFrom;
-use std::path::PathBuf;
-use transform::calendar::japanese::JapaneseErasProvider;
-use transform::datetime::week_data::WeekDataProvider;
-use transform::datetime::CommonDateProvider;
-use transform::decimal::NumbersProvider;
-use transform::list::ListProvider;
-use transform::locale_canonicalizer::aliases::AliasesProvider;
-use transform::locale_canonicalizer::likely_subtags::LikelySubtagsProvider;
-use transform::plurals::PluralsProvider;
-use transform::time_zones::TimeZonesProvider;
 
-pub fn create_exportable_provider<T: DataMarker>(
-    cldr_paths: &dyn CldrPaths,
-    _uprops_root: PathBuf,
-) -> Result<MultiForkByKeyProvider<Box<dyn OmnibusDatagenProvider<T> + Sync>>, CldrError>
-where
-    AliasesProvider: OmnibusDatagenProvider<T>,
-    CommonDateProvider: OmnibusDatagenProvider<T>,
-    JapaneseErasProvider: OmnibusDatagenProvider<T>,
-    LikelySubtagsProvider: OmnibusDatagenProvider<T>,
-    NumbersProvider: OmnibusDatagenProvider<T>,
-    PluralsProvider: OmnibusDatagenProvider<T>,
-    TimeZonesProvider: OmnibusDatagenProvider<T>,
-    ListProvider: OmnibusDatagenProvider<T>,
-    WeekDataProvider: OmnibusDatagenProvider<T>,
-{
-    #[allow(unused_variables)] // uprops_root is only used if icu_list
-    Ok(MultiForkByKeyProvider {
-        providers: vec![
-            Box::new(AliasesProvider::try_from(cldr_paths)?),
-            Box::new(CommonDateProvider::try_from(cldr_paths)?),
-            Box::new(JapaneseErasProvider::try_from(cldr_paths)?),
-            Box::new(LikelySubtagsProvider::try_from(cldr_paths)?),
-            Box::new(NumbersProvider::try_from(cldr_paths)?),
-            Box::new(PluralsProvider::try_from(cldr_paths)?),
-            Box::new(TimeZonesProvider::try_from(cldr_paths)?),
-            Box::new(WeekDataProvider::try_from(cldr_paths)?),
-            Box::new(ListProvider::try_from(cldr_paths, _uprops_root)?),
-        ],
-    })
-}
+pub use transform::calendar::japanese::JapaneseErasProvider;
+pub use transform::datetime::week_data::WeekDataProvider;
+pub use transform::datetime::CommonDateProvider;
+pub use transform::decimal::NumbersProvider;
+pub use transform::list::ListProvider;
+pub use transform::locale_canonicalizer::aliases::AliasesProvider;
+pub use transform::locale_canonicalizer::likely_subtags::LikelySubtagsProvider;
+pub use transform::plurals::PluralsProvider;
+pub use transform::time_zones::TimeZonesProvider;
 
 pub const ALL_KEYS: [ResourceKey; 19] = [
     icu_calendar::provider::JapaneseErasV1Marker::KEY,

provider/datagen/src/cldr/transform/calendar/japanese.rs

@@ -28,9 +28,9 @@ pub struct JapaneseErasProvider {
     era_dates_path: PathBuf,
 }
 
-impl TryFrom<&dyn CldrPaths> for JapaneseErasProvider {
-    type Error = Error;
-    fn try_from(cldr_paths: &dyn CldrPaths) -> Result<Self, Self::Error> {
+impl JapaneseErasProvider {
+    /// Constructs an instance from paths to source data.
+    pub fn try_new(cldr_paths: &(impl CldrPaths + ?Sized)) -> eyre::Result<Self> {
         // The era codes depend on the Latin romanizations of the eras, found
         // in the `en` locale. We load this data to construct era codes but
         // actual user code only needs to load the data for the locales it cares about.
@@ -48,6 +48,18 @@ impl TryFrom<&dyn CldrPaths> for JapaneseErasProvider {
     }
 }
 
+impl TryFrom<&crate::DatagenOptions> for JapaneseErasProvider {
+    type Error = eyre::ErrReport;
+    fn try_from(options: &crate::DatagenOptions) -> eyre::Result<Self> {
+        JapaneseErasProvider::try_new(
+            &**options
+                .cldr_paths
+                .as_ref()
+                .ok_or_else(|| eyre::eyre!("JapaneseErasProvider requires cldr_paths"))?,
+        )
+    }
+}
+
 impl ResourceProvider<JapaneseErasV1Marker> for JapaneseErasProvider {
     fn load_resource(
         &self,

provider/datagen/src/cldr/transform/datetime/mod.rs

@@ -26,16 +26,28 @@ pub struct CommonDateProvider {
     data: FrozenBTreeMap<ResourceOptions, Box<cldr_serde::ca::Dates>>,
 }
 
-impl TryFrom<&dyn CldrPaths> for CommonDateProvider {
-    type Error = Error;
-    fn try_from(cldr_paths: &dyn CldrPaths) -> Result<Self, Self::Error> {
+impl CommonDateProvider {
+    /// Constructs an instance from paths to source data.
+    pub fn try_new(cldr_paths: &(impl CldrPaths + ?Sized)) -> eyre::Result<Self> {
         Ok(Self {
             paths: cldr_paths.cldr_dates_all(),
             data: FrozenBTreeMap::new(),
         })
     }
 }
 
+impl TryFrom<&crate::DatagenOptions> for CommonDateProvider {
+    type Error = eyre::ErrReport;
+    fn try_from(options: &crate::DatagenOptions) -> eyre::Result<Self> {
+        CommonDateProvider::try_new(
+            &**options
+                .cldr_paths
+                .as_ref()
+                .ok_or_else(|| eyre::eyre!("CommonDateProvider requires cldr_paths"))?,
+        )
+    }
+}
+
 macro_rules! impl_resource_provider {
     ($(($marker:ident, $expr:expr)),+) => {
         $(
@@ -149,8 +161,8 @@ mod test {
     #[test]
     fn test_basic_patterns() {
         let cldr_paths = crate::cldr::cldr_paths::for_test();
-        let provider = CommonDateProvider::try_from(&cldr_paths as &dyn CldrPaths)
-            .expect("Failed to retrieve provider");
+        let provider =
+            CommonDateProvider::try_new(&cldr_paths).expect("Failed to retrieve provider");
 
         let cs_dates: DataPayload<DatePatternsV1Marker> = provider
             .load_resource(&DataRequest {
@@ -170,8 +182,8 @@ mod test {
     #[test]
     fn test_with_numbering_system() {
         let cldr_paths = crate::cldr::cldr_paths::for_test();
-        let provider = CommonDateProvider::try_from(&cldr_paths as &dyn CldrPaths)
-            .expect("Failed to retrieve provider");
+        let provider =
+            CommonDateProvider::try_new(&cldr_paths).expect("Failed to retrieve provider");
 
         let cs_dates: DataPayload<DatePatternsV1Marker> = provider
             .load_resource(&DataRequest {
@@ -193,8 +205,8 @@ mod test {
     #[test]
     fn test_datetime_skeletons() {
         let cldr_paths = crate::cldr::cldr_paths::for_test();
-        let provider = CommonDateProvider::try_from(&cldr_paths as &dyn CldrPaths)
-            .expect("Failed to retrieve provider");
+        let provider =
+            CommonDateProvider::try_new(&cldr_paths).expect("Failed to retrieve provider");
 
         let skeletons: DataPayload<DateSkeletonPatternsV1Marker> = provider
             .load_resource(&DataRequest {
@@ -239,7 +251,7 @@ mod test {
     #[test]
     fn test_basic_symbols() {
         let cldr_paths = crate::cldr::cldr_paths::for_test();
-        let provider = CommonDateProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap();
+        let provider = CommonDateProvider::try_new(&cldr_paths).unwrap();
 
         let cs_dates: DataPayload<DateSymbolsV1Marker> = provider
             .load_resource(&DataRequest {
@@ -264,7 +276,7 @@ mod test {
     #[test]
     fn unalias_contexts() {
         let cldr_paths = crate::cldr::cldr_paths::for_test();
-        let provider = CommonDateProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap();
+        let provider = CommonDateProvider::try_new(&cldr_paths).unwrap();
 
         let cs_dates: DataPayload<DateSymbolsV1Marker> = provider
             .load_resource(&DataRequest {

provider/datagen/src/cldr/transform/datetime/week_data.rs

@@ -9,6 +9,7 @@ use crate::cldr::cldr_serde::{
 use crate::cldr::error::Error;
 use crate::cldr::reader::open_reader;
 use crate::cldr::CldrPaths;
+use eyre::Context;
 use icu_calendar::arithmetic::week_of::CalendarInfo;
 use icu_datetime::provider::week_data::*;
 use icu_provider::datagen::IterableResourceProvider;
@@ -25,13 +26,13 @@ pub struct WeekDataProvider {
     week_data: cldr_serde::week_data::WeekData,
 }
 
-impl TryFrom<&dyn CldrPaths> for WeekDataProvider {
-    type Error = Error;
-    fn try_from(cldr_paths: &dyn CldrPaths) -> Result<Self, Self::Error> {
+impl WeekDataProvider {
+    /// Constructs an instance from paths to source data.
+    pub fn try_new(cldr_paths: &(impl CldrPaths + ?Sized)) -> eyre::Result<Self> {
         let path = cldr_paths.cldr_core()?.join("supplemental/weekData.json");
 
         let resource: cldr_serde::week_data::Resource =
-            serde_json::from_reader(open_reader(&path)?).map_err(|e| (e, path))?;
+            serde_json::from_reader(open_reader(&path)?).with_context(|| format!("{:?}", path))?;
         let week_data = resource.supplemental.week_data;
 
         let default = CalendarInfo {
@@ -61,6 +62,18 @@ impl TryFrom<&dyn CldrPaths> for WeekDataProvider {
     }
 }
 
+impl TryFrom<&crate::DatagenOptions> for WeekDataProvider {
+    type Error = eyre::ErrReport;
+    fn try_from(options: &crate::DatagenOptions) -> eyre::Result<Self> {
+        WeekDataProvider::try_new(
+            &**options
+                .cldr_paths
+                .as_ref()
+                .ok_or_else(|| eyre::eyre!("WeekDataProvider requires cldr_paths"))?,
+        )
+    }
+}
+
 impl IterableResourceProvider<WeekDataV1Marker> for WeekDataProvider {
     #[allow(clippy::needless_collect)] // https://github.com/rust-lang/rust-clippy/issues/7526
     fn supported_options(&self) -> Result<Box<dyn Iterator<Item = ResourceOptions>>, DataError> {
@@ -137,7 +150,7 @@ fn basic_cldr_week_data() {
     use icu_locid::langid;
 
     let cldr_paths = crate::cldr::cldr_paths::for_test();
-    let provider = WeekDataProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap();
+    let provider = WeekDataProvider::try_new(&cldr_paths).unwrap();
 
     let default_week_data: DataPayload<WeekDataV1Marker> = provider
         .load_resource(&DataRequest {

provider/datagen/src/cldr/transform/decimal/mod.rs

@@ -27,9 +27,9 @@ pub struct NumbersProvider {
         RwLock<Option<LiteMap<TinyStr8, cldr_serde::numbering_systems::NumberingSystem>>>,
 }
 
-impl TryFrom<&dyn CldrPaths> for NumbersProvider {
-    type Error = Error;
-    fn try_from(cldr_paths: &dyn CldrPaths) -> Result<Self, Self::Error> {
+impl NumbersProvider {
+    /// Constructs an instance from paths to source data.
+    pub fn try_new(cldr_paths: &(impl CldrPaths + ?Sized)) -> eyre::Result<Self> {
         Ok(Self {
             numbers_path: cldr_paths.cldr_numbers()?.join("main"),
             numbering_systems_path: cldr_paths
@@ -41,6 +41,18 @@ impl TryFrom<&dyn CldrPaths> for NumbersProvider {
     }
 }
 
+impl TryFrom<&crate::DatagenOptions> for NumbersProvider {
+    type Error = eyre::ErrReport;
+    fn try_from(options: &crate::DatagenOptions) -> eyre::Result<Self> {
+        NumbersProvider::try_new(
+            &**options
+                .cldr_paths
+                .as_ref()
+                .ok_or_else(|| eyre::eyre!("NumbersProvider requires cldr_paths"))?,
+        )
+    }
+}
+
 impl NumbersProvider {
     /// Returns the digits for the given numbering system name.
     fn get_digits_for_numbering_system(&self, nsname: TinyStr8) -> Option<[char; 10]> {
@@ -194,7 +206,7 @@ fn test_basic() {
     use icu_locid::locale;
 
     let cldr_paths = crate::cldr::cldr_paths::for_test();
-    let provider = NumbersProvider::try_from(&cldr_paths as &dyn CldrPaths).unwrap();
+    let provider = NumbersProvider::try_new(&cldr_paths).unwrap();
 
     let ar_decimal: DataPayload<DecimalSymbolsV1Marker> = provider
         .load_resource(&DataRequest {