Skip to content

Commit

Permalink
(icu_datagen 1.4.1) Remove fraction dep from icu_datagen
Browse files Browse the repository at this point in the history 
Backports #4459

Unlike that PR, this makes these deps dev-deps since they're not yet
used by an experimental feature
  • Loading branch information
@Manishearth
Manishearth committed Dec 19, 2023
1 parent cd9cf17 commit 7583943
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 90 deletions.
37 changes: 2 additions & 35 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions provider/datagen/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ edition.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
version.workspace = true
version = "1.4.1"

[package.metadata.docs.rs]
all-features = true
Expand Down Expand Up @@ -87,8 +87,6 @@ zip = { version = ">=0.5, <0.7", default-features = false, features = ["deflate"
rayon = { version = "1.5", optional = true }
ureq = { version = "2", optional = true }

fraction = {version = "0.14.0", default-features = false }
num-bigint = { version = "0.4.4", default-features = false }


# Dependencies for "bin" feature
Expand All @@ -102,6 +100,8 @@ crlify = { path = "../../utils/crlify" }
icu = { path = "../../components/icu" }
postcard = "1"
simple_logger = { version = "4.1.0", default-features = false }
num-bigint = { version = "0.4.4", default-features = false }
num-rational = { version = "0.4", default-features = false }

# Pre-experimental components with limited data generation
icu_singlenumberformatter = { path = "../../experimental/singlenumberformatter", features = ["datagen"] }
Expand Down
105 changes: 59 additions & 46 deletions provider/datagen/src/transform/cldr/units/helpers.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ use core::ops::{Div, Mul};
use core::str::FromStr;
use std::collections::{BTreeMap, VecDeque};

use fraction::GenericFraction;
use icu_provider::DataError;
use icu_unitsconversion::provider::{ConversionInfo, Exactness, Sign};
use num_bigint::BigUint;
use num_bigint::BigInt;
use num_rational::Ratio;

use crate::transform::cldr::cldr_serde::units::units_constants::Constant;

Expand Down Expand Up @@ -273,9 +273,7 @@ pub fn process_constants<'a>(
/// - " 1.5 E -2 " is converted to 15/1000
/// - " 1.5 E - 2" is an invalid scientific notation number
/// - "1.5E-2.5" is an invalid scientific notation number
pub fn convert_scientific_notation_to_fraction(
number: &str,
) -> Result<GenericFraction<BigUint>, DataError> {
pub fn convert_scientific_notation_to_fraction(number: &str) -> Result<Ratio<BigInt>, DataError> {
let mut parts = number.split('E');
let base = parts.next().unwrap_or("1").trim();
let exponent = parts.next().unwrap_or("0").trim();
Expand All @@ -285,51 +283,70 @@ pub fn convert_scientific_notation_to_fraction(
));
}

let base = GenericFraction::<BigUint>::from_str(base)
.map_err(|_| DataError::custom("the base is not a valid decimal number"))?;
let mut split = base.split('.');
let base_whole = split.next().ok_or(DataError::custom("the base is empty"))?;
let base_whole = if base_whole.is_empty() {
BigInt::from(0)
} else {
BigInt::from_str(base_whole).map_err(|_| {
DataError::custom("the whole part of the base is not a valid whole number")
})?
};

let base_dec = if let Some(dec_str) = split.next() {
let dec = BigInt::from_str(dec_str).map_err(|_| {
DataError::custom("the decimal part of the base is not a valid whole number")
})?;
let exp = dec_str.chars().filter(char::is_ascii_digit).count();
Ratio::new(dec, BigInt::from(10u32).pow(exp as u32))
} else {
Ratio::new(BigInt::from(0), BigInt::from(1))
};

let base = base_dec + base_whole;

let exponent = i32::from_str(exponent)
.map_err(|_| DataError::custom("the exponent is not a valid integer"))?;

let result = if exponent >= 0 {
base.mul(GenericFraction::new(
BigUint::from(10u32).pow(exponent as u32),
BigUint::from(1u32),
base.mul(Ratio::new(
BigInt::from(10u32).pow(exponent as u32),
BigInt::from(1u32),
))
} else {
base.div(GenericFraction::new(
BigUint::from(10u32).pow((-exponent) as u32),
BigUint::from(1u32),
base.div(Ratio::new(
BigInt::from(10u32).pow((-exponent) as u32),
BigInt::from(1u32),
))
};

Ok(result)
}

// TODO: move this to the comment above.
#[test]
fn test_convert_scientific_notation_to_fraction() {
let input = "1E2";
let expected = GenericFraction::<BigUint>::new(BigUint::from(100u32), BigUint::from(1u32));
let expected = Ratio::<BigInt>::new(BigInt::from(100u32), BigInt::from(1u32));
let actual = convert_scientific_notation_to_fraction(input).unwrap();
assert_eq!(expected, actual);

let input = "1E-2";
let expected = GenericFraction::<BigUint>::new(BigUint::from(1u32), BigUint::from(100u32));
let expected = Ratio::<BigInt>::new(BigInt::from(1u32), BigInt::from(100u32));
let actual = convert_scientific_notation_to_fraction(input).unwrap();
assert_eq!(expected, actual);

let input = "1.5E2";
let expected = GenericFraction::<BigUint>::new(BigUint::from(150u32), BigUint::from(1u32));
let expected = Ratio::<BigInt>::new(BigInt::from(150u32), BigInt::from(1u32));
let actual = convert_scientific_notation_to_fraction(input).unwrap();
assert_eq!(expected, actual);

let input = "1.5E-2";
let expected = GenericFraction::<BigUint>::new(BigUint::from(15u32), BigUint::from(1000u32));
let expected = Ratio::<BigInt>::new(BigInt::from(15u32), BigInt::from(1000u32));
let actual = convert_scientific_notation_to_fraction(input).unwrap();
assert_eq!(expected, actual);

let input = " 1.5 E -2 ";
let expected = GenericFraction::<BigUint>::new(BigUint::from(15u32), BigUint::from(1000u32));
let expected = Ratio::<BigInt>::new(BigInt::from(15u32), BigInt::from(1000u32));
let actual = convert_scientific_notation_to_fraction(input).unwrap();
assert_eq!(expected, actual);

Expand All @@ -340,6 +357,11 @@ fn test_convert_scientific_notation_to_fraction() {
let input = "1.5E-2.5";
let actual = convert_scientific_notation_to_fraction(input);
assert!(actual.is_err());

let input = "0.308";
let expected = Ratio::<BigInt>::new(BigInt::from(308), BigInt::from(1000u32));
let actual = convert_scientific_notation_to_fraction(input).unwrap();
assert_eq!(expected, actual);
}

/// Determines if a string contains any alphabetic characters.
Expand Down Expand Up @@ -390,28 +412,19 @@ pub fn is_scientific_number(s: &str) -> bool {
}

/// Transforms a fractional number into byte numerators, byte denominators, and a sign.
pub fn flatten_fraction(
fraction: GenericFraction<BigUint>,
) -> Result<(Vec<u8>, Vec<u8>, Sign), DataError> {
let numerator = match fraction.numer() {
Some(numerator) => numerator.to_bytes_le(),
None => return Err(DataError::custom("the numerator is too large")),
};

let denominator = match fraction.denom() {
Some(denominator) => denominator.to_bytes_le(),
None => return Err(DataError::custom("the denominator is too large")),
};

let sign = match fraction.sign() {
Some(fraction::Sign::Plus) => Sign::Positive,
Some(fraction::Sign::Minus) => Sign::Negative,
None => {
return Err(DataError::custom("the sign is not defined"));
}
pub fn flatten_fraction(fraction: Ratio<BigInt>) -> Result<(Vec<u8>, Vec<u8>, Sign), DataError> {
let (n_sign, numer) = fraction.numer().to_bytes_le();
let (d_sign, denom) = fraction.denom().to_bytes_le();

// Ratio's constructor sets denom > 0 but it's worth
// checking in case we decide to switch to new_raw() to avoid reducing
let sign = if n_sign * d_sign == num_bigint::Sign::Minus {
Sign::Negative
} else {
Sign::Positive
};

Ok((numerator, denominator, sign))
Ok((numer, denom, sign))
}

/// Converts slices of numerator and denominator strings to a fraction.
Expand All @@ -425,8 +438,8 @@ pub fn flatten_fraction(
pub fn convert_slices_to_fraction(
numerator_strings: &[&str],
denominator_strings: &[&str],
) -> Result<GenericFraction<BigUint>, DataError> {
let mut fraction = GenericFraction::new(BigUint::from(1u32), BigUint::from(1u32));
) -> Result<Ratio<BigInt>, DataError> {
let mut fraction = Ratio::new(BigInt::from(1u32), BigInt::from(1u32));

for numerator in numerator_strings {
let num_fraction = convert_scientific_notation_to_fraction(numerator)?;
Expand All @@ -446,19 +459,19 @@ pub fn convert_slices_to_fraction(
fn test_convert_array_of_strings_to_fraction() {
let numerator: Vec<&str> = vec!["1"];
let denominator: Vec<&str> = vec!["2"];
let expected = GenericFraction::new(BigUint::from(1u32), BigUint::from(2u32));
let expected = Ratio::new(BigInt::from(1i32), BigInt::from(2i32));
let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap();
assert_eq!(expected, actual);

let numerator = vec!["1", "2"];
let denominator = vec!["3", "1E2"];
let expected = GenericFraction::new(BigUint::from(2u32), BigUint::from(300u32));
let expected = Ratio::new(BigInt::from(2i32), BigInt::from(300i32));
let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap();
assert_eq!(expected, actual);

let numerator = vec!["1", "2"];
let denominator = vec!["3", "1E-2"];
let expected = GenericFraction::new(BigUint::from(200u32), BigUint::from(3u32));
let expected = Ratio::new(BigInt::from(200i32), BigInt::from(3i32));
let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap();
assert_eq!(expected, actual);

Expand All @@ -469,13 +482,13 @@ fn test_convert_array_of_strings_to_fraction() {

let numerator = vec!["1E2"];
let denominator = vec!["2"];
let expected = GenericFraction::new(BigUint::from(50u32), BigUint::from(1u32));
let expected = Ratio::new(BigInt::from(50i32), BigInt::from(1i32));
let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap();
assert_eq!(expected, actual);

let numerator = vec!["1E2", "2"];
let denominator = vec!["3", "1E2"];
let expected = GenericFraction::new(BigUint::from(2u32), BigUint::from(3u32));
let expected = Ratio::new(BigInt::from(2i32), BigInt::from(3i32));
let actual = convert_slices_to_fraction(&numerator, &denominator).unwrap();
assert_eq!(expected, actual);
}
Expand Down
12 changes: 6 additions & 6 deletions provider/datagen/src/transform/cldr/units/info.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
use std::{borrow::Cow, collections::BTreeMap};

use crate::transform::cldr::cldr_serde::{self};
use fraction::{GenericFraction, Zero};
use icu_provider::{
datagen::IterableDataProvider, DataError, DataLocale, DataPayload, DataProvider, DataRequest,
DataResponse,
Expand Down Expand Up @@ -116,6 +115,7 @@ fn test_basic() {
use icu_provider::prelude::*;
use icu_unitsconversion::provider::*;
use num_bigint::BigUint;
use num_rational::Ratio;
use zerofrom::ZeroFrom;
use zerovec::maps::ZeroVecLike;

Expand Down Expand Up @@ -157,7 +157,7 @@ fn test_basic() {
factor_num: ZeroVec::from(big_one.to_bytes_le()),
factor_den: ZeroVec::from(big_one.to_bytes_le()),
offset_sign: Sign::Positive,
offset_num: ZeroVec::from(BigUint::zero().to_bytes_le()),
offset_num: ZeroVec::from(BigUint::from(0u32).to_bytes_le()),
offset_den: ZeroVec::from(big_one.to_bytes_le()),
exactness: Exactness::Exact,
}
Expand All @@ -167,17 +167,17 @@ fn test_basic() {
let foot_convert_index = foot.convert_info.get().unwrap().as_unsigned_int() as usize;
let foot_convert_ule = convert_units.zvl_get(foot_convert_index).unwrap();
let foot_convert: ConversionInfo = ZeroFrom::zero_from(foot_convert_ule);
let ft_to_m = GenericFraction::<BigUint>::new(BigUint::from(3048u32), BigUint::from(10000u32));
let ft_to_m = Ratio::new(BigUint::from(3048u32), BigUint::from(10000u32));

assert_eq!(
foot_convert,
ConversionInfo {
base_unit: Cow::Borrowed("meter"),
factor_sign: Sign::Positive,
factor_num: ZeroVec::from(ft_to_m.numer().unwrap().to_bytes_le()),
factor_den: ZeroVec::from(ft_to_m.denom().unwrap().to_bytes_le()),
factor_num: ZeroVec::from(ft_to_m.numer().to_bytes_le()),
factor_den: ZeroVec::from(ft_to_m.denom().to_bytes_le()),
offset_sign: Sign::Positive,
offset_num: ZeroVec::from(BigUint::zero().to_bytes_le()),
offset_num: ZeroVec::from(BigUint::from(0u32).to_bytes_le()),
offset_den: ZeroVec::from(big_one.to_bytes_le()),
exactness: Exactness::Exact,
}
Expand Down

0 comments on commit 7583943

Please sign in to comment.