Merge pull request #27 from dtolnay/unicode

Update to Unicode 16.0.0

README.md

@@ -41,10 +41,10 @@ different ratios of ASCII to non-ASCII codepoints in the input data.
 
 | | static storage | 0% nonascii | 1% | 10% | 100% nonascii |
 |---|---|---|---|---|---|
-| **`unicode-ident`** | 10.1 K | 0.96 ns | 0.95 ns | 1.09 ns | 1.55 ns |
-| **`unicode-xid`** | 11.5 K | 1.88 ns | 2.14 ns | 3.48 ns | 15.63 ns |
-| **`ucd-trie`** | 10.2 K | 1.29 ns | 1.28 ns | 1.36 ns | 2.15 ns |
-| **`fst`** | 139 K | 55.1 ns | 54.9 ns | 53.2 ns | 28.5 ns |
+| **`unicode-ident`** | 10.4 K | 0.96 ns | 0.95 ns | 1.09 ns | 1.55 ns |
+| **`unicode-xid`** | 11.8 K | 1.88 ns | 2.14 ns | 3.48 ns | 15.63 ns |
+| **`ucd-trie`** | 10.3 K | 1.29 ns | 1.28 ns | 1.36 ns | 2.15 ns |
+| **`fst`** | 144 K | 55.1 ns | 54.9 ns | 53.2 ns | 28.5 ns |
 | **`roaring`** | 66.1 K | 2.78 ns | 3.09 ns | 3.37 ns | 4.70 ns |
 
 Source code for the benchmark is provided in the *bench* directory of this repo

generate/src/main.rs

@@ -1,7 +1,7 @@
 // To regenerate tables, run the following in the repo root:
 //
 // $ cargo install ucd-generate
-// $ curl -LO https://www.unicode.org/Public/zipped/15.0.0/UCD.zip
+// $ curl -LO https://www.unicode.org/Public/zipped/16.0.0/UCD.zip
 // $ unzip UCD.zip -d UCD
 // $ ucd-generate property-bool UCD --include XID_Start,XID_Continue > tests/table/tables.rs
 // $ ucd-generate property-bool UCD --include XID_Start,XID_Continue --fst-dir tests/fst

generate/src/write.rs

@@ -5,7 +5,7 @@ use crate::CHUNK;
 const HEAD: &str = "\
 // \x40generated by ../generate. To regenerate, run the following in the repo root:
 //
-// $ curl -LO https://www.unicode.org/Public/zipped/15.0.0/UCD.zip
+// $ curl -LO https://www.unicode.org/Public/zipped/16.0.0/UCD.zip
 // $ unzip UCD.zip -d UCD
 // $ cargo run --manifest-path generate/Cargo.toml
 

src/lib.rs

@@ -43,10 +43,10 @@
 //!
 //! | | static storage | 0% nonascii | 1% | 10% | 100% nonascii |
 //! |---|---|---|---|---|---|
-//! | **`unicode-ident`** | 10.1 K | 0.96 ns | 0.95 ns | 1.09 ns | 1.55 ns |
-//! | **`unicode-xid`** | 11.5 K | 1.88 ns | 2.14 ns | 3.48 ns | 15.63 ns |
-//! | **`ucd-trie`** | 10.2 K | 1.29 ns | 1.28 ns | 1.36 ns | 2.15 ns |
-//! | **`fst`** | 139 K | 55.1 ns | 54.9 ns | 53.2 ns | 28.5 ns |
+//! | **`unicode-ident`** | 10.4 K | 0.96 ns | 0.95 ns | 1.09 ns | 1.55 ns |
+//! | **`unicode-xid`** | 11.8 K | 1.88 ns | 2.14 ns | 3.48 ns | 15.63 ns |
+//! | **`ucd-trie`** | 10.3 K | 1.29 ns | 1.28 ns | 1.36 ns | 2.15 ns |
+//! | **`fst`** | 144 K | 55.1 ns | 54.9 ns | 53.2 ns | 28.5 ns |
 //! | **`roaring`** | 66.1 K | 2.78 ns | 3.09 ns | 3.37 ns | 4.70 ns |
 //!
 //! Source code for the benchmark is provided in the *bench* directory of this

tests/compare.rs

@@ -18,7 +18,7 @@ fn compare_all_implementations() {
         let thought_to_be_continue = unicode_ident::is_xid_continue(ch);
 
         // unicode-xid
-        // FIXME: unicode-xid does not support Unicode 15.1.0 yet.
+        // FIXME: unicode-xid does not support Unicode 16.0.0 yet.
         /*
         assert_eq!(
             thought_to_be_start,

tests/static_size.rs

@@ -13,7 +13,7 @@ fn test_size() {
         + size_of_val(&tables::TRIE_START)
         + size_of_val(&tables::TRIE_CONTINUE)
         + size_of_val(&tables::LEAF);
-    assert_eq!(10080, size);
+    assert_eq!(10400, size);
 }
 
 #[test]
@@ -23,7 +23,7 @@ fn test_xid_size() {
     mod tables;
 
     let size = size_of_val(tables::XID_START) + size_of_val(tables::XID_CONTINUE);
-    assert_eq!(11544, size);
+    assert_eq!(11872, size);
 
     let _ = tables::BY_NAME;
 }
@@ -70,7 +70,7 @@ fn test_trieset_size() {
         + size_of_val(tree3_level2)
         + size_of_val(tree3_level3);
 
-    assert_eq!(10200, start_size + continue_size);
+    assert_eq!(10344, start_size + continue_size);
 
     let _ = trie::BY_NAME;
 }
@@ -80,7 +80,7 @@ fn test_fst_size() {
     let xid_start_fst = include_bytes!("fst/xid_start.fst");
     let xid_continue_fst = include_bytes!("fst/xid_continue.fst");
     let size = xid_start_fst.len() + xid_continue_fst.len();
-    assert_eq!(138736, size);
+    assert_eq!(144140, size);
 }
 
 #[test]