Cleanup datagen

boa-dev · Apr 23, 2023 · e08c498 · e08c498
1 parent af3c5cd
commit e08c498
Show file tree

Hide file tree

Showing 13 changed files with 15 additions and 216 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/boa_icu_provider/Cargo.toml b/boa_icu_provider/Cargo.toml
@@ -22,15 +22,18 @@ icu_properties = "1.2.0"
 zerovec = "0.9.4"
 
 icu_datagen = { version = "1.2.3", optional = true }
-once_cell = {version = "1.17.1", default-features = false, features = ["critical-section"], optional = true }
+once_cell = { version = "1.17.1", default-features = false, features = [
+    "critical-section",
+], optional = true }
 log = { version = "0.4.17", optional = true }
 simple_logger = { version = "4.1.0", optional = true }
+icu_casemapping = { version = "0.7.2", optional = true }
 
 [features]
 default = ["std"]
 full = ["dep:once_cell"]
 std = ["once_cell?/std"]
-bin = ["dep:icu_datagen", "dep:simple_logger", "dep:log"]
+bin = ["dep:icu_datagen", "dep:simple_logger", "dep:log", "dep:icu_casemapping"]
 
 [[bin]]
 name = "boa-datagen"

diff --git a/boa_icu_provider/data/icudata.postcard b/boa_icu_provider/data/icudata.postcard
diff --git a/boa_icu_provider/data/min/fallback/likelysubtags_v1/mod.rs b/boa_icu_provider/data/min/fallback/likelysubtags_v1/mod.rs
diff --git a/boa_icu_provider/data/min/fallback/likelysubtags_v1/und.rs.data b/boa_icu_provider/data/min/fallback/likelysubtags_v1/und.rs.data
diff --git a/boa_icu_provider/data/min/fallback/mod.rs b/boa_icu_provider/data/min/fallback/mod.rs
diff --git a/boa_icu_provider/data/min/fallback/parents_v1/mod.rs b/boa_icu_provider/data/min/fallback/parents_v1/mod.rs
diff --git a/boa_icu_provider/data/min/fallback/parents_v1/und.rs.data b/boa_icu_provider/data/min/fallback/parents_v1/und.rs.data
diff --git a/boa_icu_provider/data/min/fallback/supplement/co_v1/mod.rs b/boa_icu_provider/data/min/fallback/supplement/co_v1/mod.rs
diff --git a/boa_icu_provider/data/min/fallback/supplement/co_v1/und.rs.data b/boa_icu_provider/data/min/fallback/supplement/co_v1/und.rs.data
diff --git a/boa_icu_provider/data/min/fallback/supplement/mod.rs b/boa_icu_provider/data/min/fallback/supplement/mod.rs
diff --git a/boa_icu_provider/data/min/mod.rs b/boa_icu_provider/data/min/mod.rs
@@ -1,7 +1,5 @@
 // @generated
 #[clippy::msrv = "1.61"]
-mod fallback;
-#[clippy::msrv = "1.61"]
 mod normalizer;
 #[clippy::msrv = "1.61"]
 mod props;
@@ -123,66 +121,6 @@ macro_rules! impl_data_provider {
                     .ok_or_else(|| DataErrorKind::MissingLocale.with_req(::icu_properties::provider::IdStartV1Marker::KEY, req))
             }
         }
-        #[clippy::msrv = "1.61"]
-        impl DataProvider<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker> for $provider {
-            fn load(
-                &self,
-                req: DataRequest,
-            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker>, DataError> {
-                fallback::supplement::co_v1::lookup(&req.locale)
-                    .map(zerofrom::ZeroFrom::zero_from)
-                    .map(DataPayload::from_owned)
-                    .map(|payload| DataResponse {
-                        metadata: Default::default(),
-                        payload: Some(payload),
-                    })
-                    .ok_or_else(|| {
-                        DataErrorKind::MissingLocale.with_req(
-                            ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY,
-                            req,
-                        )
-                    })
-            }
-        }
-        #[clippy::msrv = "1.61"]
-        impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker> for $provider {
-            fn load(
-                &self,
-                req: DataRequest,
-            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker>, DataError> {
-                fallback::likelysubtags_v1::lookup(&req.locale)
-                    .map(zerofrom::ZeroFrom::zero_from)
-                    .map(DataPayload::from_owned)
-                    .map(|payload| DataResponse {
-                        metadata: Default::default(),
-                        payload: Some(payload),
-                    })
-                    .ok_or_else(|| {
-                        DataErrorKind::MissingLocale.with_req(
-                            ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY,
-                            req,
-                        )
-                    })
-            }
-        }
-        #[clippy::msrv = "1.61"]
-        impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker> for $provider {
-            fn load(
-                &self,
-                req: DataRequest,
-            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker>, DataError> {
-                fallback::parents_v1::lookup(&req.locale)
-                    .map(zerofrom::ZeroFrom::zero_from)
-                    .map(DataPayload::from_owned)
-                    .map(|payload| DataResponse {
-                        metadata: Default::default(),
-                        payload: Some(payload),
-                    })
-                    .ok_or_else(|| {
-                        DataErrorKind::MissingLocale.with_req(::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY, req)
-                    })
-            }
-        }
     };
 }
 /// Implement [`AnyProvider`] on the given struct using the data
@@ -215,12 +153,6 @@ macro_rules! impl_any_provider {
                     ::icu_normalizer::provider::CompatibilityDecompositionTablesV1Marker::KEY.hashed();
                 const IDCONTINUEV1MARKER: ::icu_provider::DataKeyHash = ::icu_properties::provider::IdContinueV1Marker::KEY.hashed();
                 const IDSTARTV1MARKER: ::icu_provider::DataKeyHash = ::icu_properties::provider::IdStartV1Marker::KEY.hashed();
-                const COLLATIONFALLBACKSUPPLEMENTV1MARKER: ::icu_provider::DataKeyHash =
-                    ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY.hashed();
-                const LOCALEFALLBACKLIKELYSUBTAGSV1MARKER: ::icu_provider::DataKeyHash =
-                    ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY.hashed();
-                const LOCALEFALLBACKPARENTSV1MARKER: ::icu_provider::DataKeyHash =
-                    ::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY.hashed();
                 match key.hashed() {
                     CANONICALCOMPOSITIONSV1MARKER => normalizer::comp_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     CANONICALDECOMPOSITIONDATAV1MARKER => normalizer::nfd_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
@@ -229,9 +161,6 @@ macro_rules! impl_any_provider {
                     COMPATIBILITYDECOMPOSITIONTABLESV1MARKER => normalizer::nfkdex_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     IDCONTINUEV1MARKER => props::idc_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     IDSTARTV1MARKER => props::ids_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
-                    COLLATIONFALLBACKSUPPLEMENTV1MARKER => fallback::supplement::co_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
-                    LOCALEFALLBACKLIKELYSUBTAGSV1MARKER => fallback::likelysubtags_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
-                    LOCALEFALLBACKPARENTSV1MARKER => fallback::parents_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
                     _ => return Err(DataErrorKind::MissingDataKey.with_req(key, req)),
                 }
                 .map(|payload| AnyResponse {

diff --git a/boa_icu_provider/src/bin/datagen.rs b/boa_icu_provider/src/bin/datagen.rs
@@ -6,20 +6,15 @@
 use std::{error::Error, fs::File};
 
 use boa_icu_provider::data_root;
-use icu_datagen::{
-    all_keys_with_experimental, datagen, BakedOptions, CldrLocaleSubset, Out, SourceData,
-};
+use icu_casemapping::provider::CaseMappingV1Marker;
+use icu_datagen::{all_keys, datagen, BakedOptions, CoverageLevel, Out, SourceData};
 use icu_normalizer::provider::{
     CanonicalCompositionsV1Marker, CanonicalDecompositionDataV1Marker,
     CanonicalDecompositionTablesV1Marker, CompatibilityDecompositionSupplementV1Marker,
     CompatibilityDecompositionTablesV1Marker,
 };
 use icu_properties::provider::{IdContinueV1Marker, IdStartV1Marker};
 use icu_provider::KeyedDataMarker;
-use icu_provider_adapters::fallback::provider::{
-    CollationFallbackSupplementV1Marker, LocaleFallbackLikelySubtagsV1Marker,
-    LocaleFallbackParentsV1Marker,
-};
 
 fn main() -> Result<(), Box<dyn Error>> {
     simple_logger::SimpleLogger::new()
@@ -28,7 +23,7 @@ fn main() -> Result<(), Box<dyn Error>> {
         .init()?;
 
     let source_data = SourceData::default()
-        .with_cldr_for_tag(SourceData::LATEST_TESTED_CLDR_TAG, CldrLocaleSubset::Modern)?
+        .with_cldr_for_tag(SourceData::LATEST_TESTED_CLDR_TAG, Default::default())?
         .with_icuexport_for_tag(SourceData::LATEST_TESTED_ICUEXPORT_TAG)?
         .with_collations(vec![String::from("search*")]);
 
@@ -47,30 +42,26 @@ fn main() -> Result<(), Box<dyn Error>> {
         },
     };
 
+    let locales = source_data.locales(&[CoverageLevel::Modern])?;
+
     datagen(
-        None,
+        Some(&locales),
         &[
             CanonicalDecompositionDataV1Marker::KEY,
             CanonicalDecompositionTablesV1Marker::KEY,
             CanonicalCompositionsV1Marker::KEY,
             CompatibilityDecompositionSupplementV1Marker::KEY,
             CompatibilityDecompositionTablesV1Marker::KEY,
-            LocaleFallbackLikelySubtagsV1Marker::KEY,
-            LocaleFallbackParentsV1Marker::KEY,
-            CollationFallbackSupplementV1Marker::KEY,
             IdContinueV1Marker::KEY,
             IdStartV1Marker::KEY,
         ],
         &source_data,
         [normalization_out].into(),
     )?;
 
-    datagen(
-        None,
-        &all_keys_with_experimental(),
-        &source_data,
-        [full_blob_out].into(),
-    )?;
+    let keys = &mut all_keys();
+    keys.push(CaseMappingV1Marker::KEY);
+    datagen(Some(&locales), keys, &source_data, [full_blob_out].into())?;
 
     Ok(())
 }