From 425e7e5596c0ab6555fa75292d38863280d4a3d7 Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sat, 7 Mar 2020 21:29:09 +0100
Subject: [PATCH 01/26] Don't insert panic when generator can not return

---
 src/librustc_mir/transform/generator.rs | 31 +++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index d060a0eab3db0..cc8d2807b472b 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -991,12 +991,30 @@ fn insert_panic_block<'tcx>(
     assert_block
 }
 
+fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
+    // Returning from a function with an uninhabited return type is undefined behavior.
+    if body.return_ty().conservative_is_privately_uninhabited(tcx) {
+        return false;
+    }
+
+    // If there's no return terminator the function also won't return.
+    for block in body.basic_blocks() {
+        if let TerminatorKind::Return = block.terminator().kind {
+            return true;
+        }
+    }
+
+    // Otherwise we assume that the function may return.
+    false
+}
+
 fn create_generator_resume_function<'tcx>(
     tcx: TyCtxt<'tcx>,
     transform: TransformVisitor<'tcx>,
     def_id: DefId,
     source: MirSource<'tcx>,
     body: &mut BodyAndCache<'tcx>,
+    can_return: bool,
 ) {
     // Poison the generator when it unwinds
     for block in body.basic_blocks_mut() {
@@ -1015,7 +1033,14 @@ fn create_generator_resume_function<'tcx>(
 
     // Panic when resumed on the returned or poisoned state
     let generator_kind = body.generator_kind.unwrap();
-    cases.insert(1, (RETURNED, insert_panic_block(tcx, body, ResumedAfterReturn(generator_kind))));
+
+    if can_return {
+        cases.insert(
+            1,
+            (RETURNED, insert_panic_block(tcx, body, ResumedAfterReturn(generator_kind))),
+        );
+    }
+
     cases.insert(2, (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))));
 
     insert_switch(body, cases, &transform, TerminatorKind::Unreachable);
@@ -1200,6 +1225,8 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         let (remap, layout, storage_liveness) =
             compute_layout(tcx, source, &upvars, interior, movable, body);
 
+        let can_return = can_return(tcx, body);
+
         // Run the transformation which converts Places from Local to generator struct
         // accesses for locals in `remap`.
         // It also rewrites `return x` and `yield y` as writing a new generator state and returning
@@ -1243,6 +1270,6 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         body.generator_drop = Some(box drop_shim);
 
         // Create the Generator::resume function
-        create_generator_resume_function(tcx, transform, def_id, source, body);
+        create_generator_resume_function(tcx, transform, def_id, source, body, can_return);
     }
 }

From 5ac41a1a8d28493c7aa927ae49c664b9b9dce476 Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sat, 7 Mar 2020 21:51:34 +0100
Subject: [PATCH 02/26] Don't insert panic when generator can not unwind

---
 src/librustc_mir/transform/generator.rs | 46 ++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index cc8d2807b472b..0502e57533d02 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -1008,6 +1008,45 @@ fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
     false
 }
 
+fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
+    // Nothing can unwind when landing pads are off.
+    if tcx.sess.no_landing_pads() {
+        return false;
+    }
+
+    // Unwinds can only start at certain terminators.
+    for block in body.basic_blocks() {
+        match block.terminator().kind {
+            // These never unwind.
+            TerminatorKind::Goto { .. }
+            | TerminatorKind::SwitchInt { .. }
+            | TerminatorKind::Abort
+            | TerminatorKind::Return
+            | TerminatorKind::Unreachable
+            | TerminatorKind::GeneratorDrop
+            | TerminatorKind::FalseEdges { .. }
+            | TerminatorKind::FalseUnwind { .. } => {}
+
+            // Resume will *continue* unwinding, but if there's no other unwinding terminator it
+            // will never be reached.
+            TerminatorKind::Resume => {}
+
+            TerminatorKind::Yield { .. } => {
+                unreachable!("`can_unwind` called before generator transform")
+            }
+
+            // These may unwind.
+            TerminatorKind::Drop { .. }
+            | TerminatorKind::DropAndReplace { .. }
+            | TerminatorKind::Call { .. }
+            | TerminatorKind::Assert { .. } => return true,
+        }
+    }
+
+    // If we didn't find an unwinding terminator, the function cannot unwind.
+    false
+}
+
 fn create_generator_resume_function<'tcx>(
     tcx: TyCtxt<'tcx>,
     transform: TransformVisitor<'tcx>,
@@ -1041,7 +1080,12 @@ fn create_generator_resume_function<'tcx>(
         );
     }
 
-    cases.insert(2, (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))));
+    if can_unwind(tcx, body) {
+        cases.insert(
+            2,
+            (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))),
+        );
+    }
 
     insert_switch(body, cases, &transform, TerminatorKind::Unreachable);
 

From 46aeef6e655a85087f82bb0d1f94a97e83009218 Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sun, 8 Mar 2020 00:55:29 +0100
Subject: [PATCH 03/26] Poison generators when any terminator unwinds

This didn't cause issues before since generator types were always
considered to "need drop", leading to unwind paths
(including a `Resume` block) always getting generated.
---
 src/librustc_mir/transform/generator.rs | 36 +++++++++++++++++++++----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index 0502e57533d02..ffbf4ae939334 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -1055,11 +1055,37 @@ fn create_generator_resume_function<'tcx>(
     body: &mut BodyAndCache<'tcx>,
     can_return: bool,
 ) {
+    let can_unwind = can_unwind(tcx, body);
+
     // Poison the generator when it unwinds
-    for block in body.basic_blocks_mut() {
-        let source_info = block.terminator().source_info;
-        if let &TerminatorKind::Resume = &block.terminator().kind {
-            block.statements.push(transform.set_discr(VariantIdx::new(POISONED), source_info));
+    if can_unwind {
+        let poison_block = BasicBlock::new(body.basic_blocks().len());
+        let source_info = source_info(body);
+        body.basic_blocks_mut().push(BasicBlockData {
+            statements: vec![transform.set_discr(VariantIdx::new(POISONED), source_info)],
+            terminator: Some(Terminator { source_info, kind: TerminatorKind::Resume }),
+            is_cleanup: true,
+        });
+
+        for (idx, block) in body.basic_blocks_mut().iter_enumerated_mut() {
+            let source_info = block.terminator().source_info;
+
+            if let TerminatorKind::Resume = block.terminator().kind {
+                // An existing `Resume` terminator is redirected to jump to our dedicated
+                // "poisoning block" above.
+                if idx != poison_block {
+                    *block.terminator_mut() = Terminator {
+                        source_info,
+                        kind: TerminatorKind::Goto { target: poison_block },
+                    };
+                }
+            } else if !block.is_cleanup {
+                // Any terminators that *can* unwind but don't have an unwind target set are also
+                // pointed at our poisoning block (unless they're part of the cleanup path).
+                if let Some(unwind @ None) = block.terminator_mut().unwind_mut() {
+                    *unwind = Some(poison_block);
+                }
+            }
         }
     }
 
@@ -1080,7 +1106,7 @@ fn create_generator_resume_function<'tcx>(
         );
     }
 
-    if can_unwind(tcx, body) {
+    if can_unwind {
         cases.insert(
             2,
             (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))),

From 8d9f633f4d78777487010ae119454dc9674e42ee Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sun, 8 Mar 2020 01:23:00 +0100
Subject: [PATCH 04/26] Change index for SwitchInt case

The indices do not matter here, and this fixes an index out of bounds
panic when compiling a generator that can unwind but not return.
---
 src/librustc_mir/transform/generator.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index ffbf4ae939334..841d18268b13a 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -1108,7 +1108,7 @@ fn create_generator_resume_function<'tcx>(
 
     if can_unwind {
         cases.insert(
-            2,
+            1,
             (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))),
         );
     }

From 2cbccced0856260c6acafabecb3030465dff28c2 Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sun, 8 Mar 2020 01:47:54 +0100
Subject: [PATCH 05/26] Add test for unnecessary panic branches

---
 src/test/mir-opt/generator-tiny.rs | 34 ++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 src/test/mir-opt/generator-tiny.rs

diff --git a/src/test/mir-opt/generator-tiny.rs b/src/test/mir-opt/generator-tiny.rs
new file mode 100644
index 0000000000000..09e943bd962e6
--- /dev/null
+++ b/src/test/mir-opt/generator-tiny.rs
@@ -0,0 +1,34 @@
+//! Tests that generators that cannot return or unwind don't have unnecessary
+//! panic branches.
+
+// compile-flags: -Zno-landing-pads
+
+#![feature(generators, generator_trait)]
+
+struct HasDrop;
+
+impl Drop for HasDrop {
+    fn drop(&mut self) {}
+}
+
+fn callee() {}
+
+fn main() {
+    let _gen = |_x: u8| {
+        let _d = HasDrop;
+        loop {
+            yield;
+            callee();
+        }
+    };
+}
+
+// END RUST SOURCE
+
+// START rustc.main-{{closure}}.generator_resume.0.mir
+// bb0: {
+//     ...
+//     switchInt(move _11) -> [0u32: bb1, 3u32: bb5, otherwise: bb6];
+// }
+// ...
+// END rustc.main-{{closure}}.generator_resume.0.mir

From cb58de81654d92074a4abdbf90400f35f6a0062f Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sun, 8 Mar 2020 13:51:26 +0100
Subject: [PATCH 06/26] Apply suggestions from code review

Co-Authored-By: bjorn3 <bjorn3@users.noreply.github.com>
---
 src/librustc_mir/transform/generator.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index 841d18268b13a..b386582947f1a 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -997,14 +997,14 @@ fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
         return false;
     }
 
-    // If there's no return terminator the function also won't return.
+    // If there's a return terminator the function may return.
     for block in body.basic_blocks() {
         if let TerminatorKind::Return = block.terminator().kind {
             return true;
         }
     }
 
-    // Otherwise we assume that the function may return.
+    // Otherwise the function can't return.
     false
 }
 

From 543832b06c7fccd570a85e2f30a447303fd58602 Mon Sep 17 00:00:00 2001
From: Josh Stone <jistone@redhat.com>
Date: Wed, 11 Mar 2020 09:19:59 -0700
Subject: [PATCH 07/26] Regenerate tables for Unicode 13.0.0

---
 src/libcore/unicode/printable.rs    | 173 ++++---
 src/libcore/unicode/unicode_data.rs | 734 ++++++++++++++--------------
 2 files changed, 462 insertions(+), 445 deletions(-)

diff --git a/src/libcore/unicode/printable.rs b/src/libcore/unicode/printable.rs
index eee9ea52ef0d2..9680aa14d3b54 100644
--- a/src/libcore/unicode/printable.rs
+++ b/src/libcore/unicode/printable.rs
@@ -44,7 +44,7 @@ pub(crate) fn is_printable(x: char) -> bool {
     } else if x < 0x20000 {
         check(lower, SINGLETONS1U, SINGLETONS1L, NORMAL1)
     } else {
-        if 0x2a6d7 <= x && x < 0x2a700 {
+        if 0x2a6de <= x && x < 0x2a700 {
             return false;
         }
         if 0x2b735 <= x && x < 0x2b740 {
@@ -59,7 +59,10 @@ pub(crate) fn is_printable(x: char) -> bool {
         if 0x2ebe1 <= x && x < 0x2f800 {
             return false;
         }
-        if 0x2fa1e <= x && x < 0xe0100 {
+        if 0x2fa1e <= x && x < 0x30000 {
+            return false;
+        }
+        if 0x3134b <= x && x < 0xe0100 {
             return false;
         }
         if 0xe01f0 <= x && x < 0x110000 {
@@ -81,7 +84,7 @@ const SINGLETONS0U: &[(u8, u8)] = &[
     (0x0a, 28),
     (0x0b, 25),
     (0x0c, 20),
-    (0x0d, 18),
+    (0x0d, 16),
     (0x0e, 13),
     (0x0f, 4),
     (0x10, 3),
@@ -96,7 +99,7 @@ const SINGLETONS0U: &[(u8, u8)] = &[
     (0x1d, 1),
     (0x1f, 22),
     (0x20, 3),
-    (0x2b, 4),
+    (0x2b, 3),
     (0x2c, 2),
     (0x2d, 11),
     (0x2e, 1),
@@ -129,29 +132,29 @@ const SINGLETONS0L: &[u8] = &[
     0x4a, 0x5e, 0x64, 0x65, 0x84, 0x91, 0x9b, 0x9d,
     0xc9, 0xce, 0xcf, 0x0d, 0x11, 0x29, 0x45, 0x49,
     0x57, 0x64, 0x65, 0x8d, 0x91, 0xa9, 0xb4, 0xba,
-    0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x04,
-    0x0d, 0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x81,
-    0x84, 0xb2, 0xbc, 0xbe, 0xbf, 0xd5, 0xd7, 0xf0,
-    0xf1, 0x83, 0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf,
-    0xc5, 0xc7, 0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98,
-    0xbd, 0xcd, 0xc6, 0xce, 0xcf, 0x49, 0x4e, 0x4f,
-    0x57, 0x59, 0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1,
-    0xb6, 0xb7, 0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11,
-    0x16, 0x17, 0x5b, 0x5c, 0xf6, 0xf7, 0xfe, 0xff,
-    0x80, 0x0d, 0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f,
-    0x1f, 0x6e, 0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e,
-    0xae, 0xaf, 0xbb, 0xbc, 0xfa, 0x16, 0x17, 0x1e,
-    0x1f, 0x46, 0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c,
-    0x5e, 0x7e, 0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc,
-    0xf0, 0xf1, 0xf5, 0x72, 0x73, 0x8f, 0x74, 0x75,
-    0x96, 0x97, 0x2f, 0x5f, 0x26, 0x2e, 0x2f, 0xa7,
-    0xaf, 0xb7, 0xbf, 0xc7, 0xcf, 0xd7, 0xdf, 0x9a,
-    0x40, 0x97, 0x98, 0x30, 0x8f, 0x1f, 0xc0, 0xc1,
-    0xce, 0xff, 0x4e, 0x4f, 0x5a, 0x5b, 0x07, 0x08,
-    0x0f, 0x10, 0x27, 0x2f, 0xee, 0xef, 0x6e, 0x6f,
-    0x37, 0x3d, 0x3f, 0x42, 0x45, 0x90, 0x91, 0xfe,
-    0xff, 0x53, 0x67, 0x75, 0xc8, 0xc9, 0xd0, 0xd1,
-    0xd8, 0xd9, 0xe7, 0xfe, 0xff,
+    0xbb, 0xc5, 0xc9, 0xdf, 0xe4, 0xe5, 0xf0, 0x0d,
+    0x11, 0x45, 0x49, 0x64, 0x65, 0x80, 0x84, 0xb2,
+    0xbc, 0xbe, 0xbf, 0xd5, 0xd7, 0xf0, 0xf1, 0x83,
+    0x85, 0x8b, 0xa4, 0xa6, 0xbe, 0xbf, 0xc5, 0xc7,
+    0xce, 0xcf, 0xda, 0xdb, 0x48, 0x98, 0xbd, 0xcd,
+    0xc6, 0xce, 0xcf, 0x49, 0x4e, 0x4f, 0x57, 0x59,
+    0x5e, 0x5f, 0x89, 0x8e, 0x8f, 0xb1, 0xb6, 0xb7,
+    0xbf, 0xc1, 0xc6, 0xc7, 0xd7, 0x11, 0x16, 0x17,
+    0x5b, 0x5c, 0xf6, 0xf7, 0xfe, 0xff, 0x80, 0x0d,
+    0x6d, 0x71, 0xde, 0xdf, 0x0e, 0x0f, 0x1f, 0x6e,
+    0x6f, 0x1c, 0x1d, 0x5f, 0x7d, 0x7e, 0xae, 0xaf,
+    0xbb, 0xbc, 0xfa, 0x16, 0x17, 0x1e, 0x1f, 0x46,
+    0x47, 0x4e, 0x4f, 0x58, 0x5a, 0x5c, 0x5e, 0x7e,
+    0x7f, 0xb5, 0xc5, 0xd4, 0xd5, 0xdc, 0xf0, 0xf1,
+    0xf5, 0x72, 0x73, 0x8f, 0x74, 0x75, 0x96, 0x2f,
+    0x5f, 0x26, 0x2e, 0x2f, 0xa7, 0xaf, 0xb7, 0xbf,
+    0xc7, 0xcf, 0xd7, 0xdf, 0x9a, 0x40, 0x97, 0x98,
+    0x30, 0x8f, 0x1f, 0xc0, 0xc1, 0xce, 0xff, 0x4e,
+    0x4f, 0x5a, 0x5b, 0x07, 0x08, 0x0f, 0x10, 0x27,
+    0x2f, 0xee, 0xef, 0x6e, 0x6f, 0x37, 0x3d, 0x3f,
+    0x42, 0x45, 0x90, 0x91, 0xfe, 0xff, 0x53, 0x67,
+    0x75, 0xc8, 0xc9, 0xd0, 0xd1, 0xd8, 0xd9, 0xe7,
+    0xfe, 0xff,
 ];
 #[rustfmt::skip]
 const SINGLETONS1U: &[(u8, u8)] = &[
@@ -163,14 +166,15 @@ const SINGLETONS1U: &[(u8, u8)] = &[
     (0x09, 2),
     (0x0a, 5),
     (0x0b, 2),
+    (0x0e, 4),
     (0x10, 1),
-    (0x11, 4),
+    (0x11, 2),
     (0x12, 5),
     (0x13, 17),
-    (0x14, 2),
+    (0x14, 1),
     (0x15, 2),
     (0x17, 2),
-    (0x19, 4),
+    (0x19, 13),
     (0x1c, 5),
     (0x1d, 8),
     (0x24, 1),
@@ -188,32 +192,35 @@ const SINGLETONS1U: &[(u8, u8)] = &[
     (0xe8, 2),
     (0xee, 32),
     (0xf0, 4),
-    (0xf9, 6),
+    (0xf8, 2),
+    (0xf9, 2),
     (0xfa, 2),
+    (0xfb, 1),
 ];
 #[rustfmt::skip]
 const SINGLETONS1L: &[u8] = &[
     0x0c, 0x27, 0x3b, 0x3e, 0x4e, 0x4f, 0x8f, 0x9e,
     0x9e, 0x9f, 0x06, 0x07, 0x09, 0x36, 0x3d, 0x3e,
     0x56, 0xf3, 0xd0, 0xd1, 0x04, 0x14, 0x18, 0x36,
-    0x37, 0x56, 0x57, 0xbd, 0x35, 0xce, 0xcf, 0xe0,
-    0x12, 0x87, 0x89, 0x8e, 0x9e, 0x04, 0x0d, 0x0e,
-    0x11, 0x12, 0x29, 0x31, 0x34, 0x3a, 0x45, 0x46,
-    0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65, 0x5a, 0x5c,
-    0xb6, 0xb7, 0x1b, 0x1c, 0xa8, 0xa9, 0xd8, 0xd9,
-    0x09, 0x37, 0x90, 0x91, 0xa8, 0x07, 0x0a, 0x3b,
-    0x3e, 0x66, 0x69, 0x8f, 0x92, 0x6f, 0x5f, 0xee,
-    0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27, 0x28, 0x55,
-    0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7, 0xa8, 0xad,
-    0xba, 0xbc, 0xc4, 0x06, 0x0b, 0x0c, 0x15, 0x1d,
-    0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7, 0xcc, 0xcd,
-    0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25, 0x3e, 0x3f,
-    0xc5, 0xc6, 0x04, 0x20, 0x23, 0x25, 0x26, 0x28,
-    0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c, 0x50, 0x53,
-    0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e, 0x60, 0x63,
-    0x65, 0x66, 0x6b, 0x73, 0x78, 0x7d, 0x7f, 0x8a,
-    0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0, 0x0c, 0x72,
-    0xa3, 0xa4, 0xcb, 0xcc, 0x6e, 0x6f,
+    0x37, 0x56, 0x57, 0x7f, 0xaa, 0xae, 0xaf, 0xbd,
+    0x35, 0xe0, 0x12, 0x87, 0x89, 0x8e, 0x9e, 0x04,
+    0x0d, 0x0e, 0x11, 0x12, 0x29, 0x31, 0x34, 0x3a,
+    0x45, 0x46, 0x49, 0x4a, 0x4e, 0x4f, 0x64, 0x65,
+    0x5c, 0xb6, 0xb7, 0x1b, 0x1c, 0x07, 0x08, 0x0a,
+    0x0b, 0x14, 0x17, 0x36, 0x39, 0x3a, 0xa8, 0xa9,
+    0xd8, 0xd9, 0x09, 0x37, 0x90, 0x91, 0xa8, 0x07,
+    0x0a, 0x3b, 0x3e, 0x66, 0x69, 0x8f, 0x92, 0x6f,
+    0x5f, 0xee, 0xef, 0x5a, 0x62, 0x9a, 0x9b, 0x27,
+    0x28, 0x55, 0x9d, 0xa0, 0xa1, 0xa3, 0xa4, 0xa7,
+    0xa8, 0xad, 0xba, 0xbc, 0xc4, 0x06, 0x0b, 0x0c,
+    0x15, 0x1d, 0x3a, 0x3f, 0x45, 0x51, 0xa6, 0xa7,
+    0xcc, 0xcd, 0xa0, 0x07, 0x19, 0x1a, 0x22, 0x25,
+    0x3e, 0x3f, 0xc5, 0xc6, 0x04, 0x20, 0x23, 0x25,
+    0x26, 0x28, 0x33, 0x38, 0x3a, 0x48, 0x4a, 0x4c,
+    0x50, 0x53, 0x55, 0x56, 0x58, 0x5a, 0x5c, 0x5e,
+    0x60, 0x63, 0x65, 0x66, 0x6b, 0x73, 0x78, 0x7d,
+    0x7f, 0x8a, 0xa4, 0xaa, 0xaf, 0xb0, 0xc0, 0xd0,
+    0xae, 0xaf, 0x79, 0xcc, 0x6e, 0x6f, 0x93,
 ];
 #[rustfmt::skip]
 const NORMAL0: &[u8] = &[
@@ -225,7 +232,7 @@ const NORMAL0: &[u8] = &[
     0x06, 0x11,
     0x81, 0xac, 0x0e,
     0x80, 0xab, 0x35,
-    0x1e, 0x15,
+    0x28, 0x0b,
     0x80, 0xe0, 0x03,
     0x19, 0x08,
     0x01, 0x04,
@@ -237,8 +244,8 @@ const NORMAL0: &[u8] = &[
     0x11, 0x0a,
     0x50, 0x0f,
     0x12, 0x07,
-    0x55, 0x08,
-    0x02, 0x04,
+    0x55, 0x07,
+    0x03, 0x04,
     0x1c, 0x0a,
     0x09, 0x03,
     0x08, 0x03,
@@ -292,7 +299,7 @@ const NORMAL0: &[u8] = &[
     0x0b, 0x03,
     0x80, 0xac, 0x06,
     0x0a, 0x06,
-    0x1f, 0x41,
+    0x21, 0x3f,
     0x4c, 0x04,
     0x2d, 0x03,
     0x74, 0x08,
@@ -315,21 +322,19 @@ const NORMAL0: &[u8] = &[
     0x3b, 0x07,
     0x02, 0x0e,
     0x18, 0x09,
-    0x80, 0xb0, 0x30,
+    0x80, 0xb3, 0x2d,
     0x74, 0x0c,
     0x80, 0xd6, 0x1a,
     0x0c, 0x05,
     0x80, 0xff, 0x05,
-    0x80, 0xb6, 0x05,
-    0x24, 0x0c,
-    0x9b, 0xc6, 0x0a,
-    0xd2, 0x30, 0x10,
+    0x80, 0xdf, 0x0c,
+    0xee, 0x0d, 0x03,
     0x84, 0x8d, 0x03,
     0x37, 0x09,
     0x81, 0x5c, 0x14,
     0x80, 0xb8, 0x08,
-    0x80, 0xc7, 0x30,
-    0x35, 0x04,
+    0x80, 0xcb, 0x2a,
+    0x38, 0x03,
     0x0a, 0x06,
     0x38, 0x08,
     0x46, 0x08,
@@ -341,7 +346,7 @@ const NORMAL0: &[u8] = &[
     0x80, 0x83, 0x18,
     0x1c, 0x0a,
     0x16, 0x09,
-    0x48, 0x08,
+    0x4c, 0x04,
     0x80, 0x8a, 0x06,
     0xab, 0xa4, 0x0c,
     0x17, 0x04,
@@ -365,7 +370,7 @@ const NORMAL1: &[u8] = &[
     0x7b, 0x05,
     0x03, 0x04,
     0x2d, 0x03,
-    0x65, 0x04,
+    0x66, 0x03,
     0x01, 0x2f,
     0x2e, 0x80, 0x82,
     0x1d, 0x03,
@@ -410,16 +415,17 @@ const NORMAL1: &[u8] = &[
     0x33, 0x07,
     0x2e, 0x08,
     0x0a, 0x81, 0x26,
-    0x1f, 0x80, 0x81,
+    0x52, 0x4e,
     0x28, 0x08,
-    0x2a, 0x80, 0x86,
+    0x2a, 0x56,
+    0x1c, 0x14,
     0x17, 0x09,
     0x4e, 0x04,
     0x1e, 0x0f,
     0x43, 0x0e,
     0x19, 0x07,
     0x0a, 0x06,
-    0x47, 0x09,
+    0x48, 0x08,
     0x27, 0x09,
     0x75, 0x0b,
     0x3f, 0x41,
@@ -430,7 +436,7 @@ const NORMAL1: &[u8] = &[
     0x01, 0x05,
     0x10, 0x03,
     0x05, 0x80, 0x8b,
-    0x60, 0x20,
+    0x62, 0x1e,
     0x48, 0x08,
     0x0a, 0x80, 0xa6,
     0x5e, 0x22,
@@ -443,7 +449,8 @@ const NORMAL1: &[u8] = &[
     0x10, 0x80, 0xc0,
     0x3c, 0x64,
     0x53, 0x0c,
-    0x01, 0x80, 0xa0,
+    0x48, 0x09,
+    0x0a, 0x46,
     0x45, 0x1b,
     0x48, 0x08,
     0x53, 0x1d,
@@ -456,7 +463,8 @@ const NORMAL1: &[u8] = &[
     0x0a, 0x06,
     0x39, 0x07,
     0x0a, 0x81, 0x36,
-    0x19, 0x80, 0xc7,
+    0x19, 0x80, 0xb7,
+    0x01, 0x0f,
     0x32, 0x0d,
     0x83, 0x9b, 0x66,
     0x75, 0x0b,
@@ -474,9 +482,11 @@ const NORMAL1: &[u8] = &[
     0x4b, 0x04,
     0x39, 0x07,
     0x11, 0x40,
-    0x04, 0x1c,
+    0x05, 0x0b,
+    0x02, 0x0e,
     0x97, 0xf8, 0x08,
-    0x82, 0xf3, 0xa5, 0x0d,
+    0x84, 0xd6, 0x2a,
+    0x09, 0xa2, 0xf7,
     0x81, 0x1f, 0x31,
     0x03, 0x11,
     0x04, 0x08,
@@ -515,17 +525,15 @@ const NORMAL1: &[u8] = &[
     0x2c, 0x04,
     0x64, 0x0c,
     0x56, 0x0a,
-    0x0d, 0x03,
-    0x5d, 0x03,
-    0x3d, 0x39,
+    0x80, 0xae, 0x38,
     0x1d, 0x0d,
     0x2c, 0x04,
     0x09, 0x07,
     0x02, 0x0e,
     0x06, 0x80, 0x9a,
-    0x83, 0xd6, 0x0a,
+    0x83, 0xd8, 0x08,
+    0x0d, 0x03,
     0x0d, 0x03,
-    0x0b, 0x05,
     0x74, 0x0c,
     0x59, 0x07,
     0x0c, 0x14,
@@ -533,12 +541,15 @@ const NORMAL1: &[u8] = &[
     0x38, 0x08,
     0x0a, 0x06,
     0x28, 0x08,
-    0x1e, 0x52,
-    0x77, 0x03,
-    0x31, 0x03,
-    0x80, 0xa6, 0x0c,
-    0x14, 0x04,
+    0x22, 0x4e,
+    0x81, 0x54, 0x0c,
+    0x15, 0x03,
     0x03, 0x05,
+    0x07, 0x09,
+    0x19, 0x07,
+    0x07, 0x09,
     0x03, 0x0d,
-    0x06, 0x85, 0x6a,
+    0x07, 0x29,
+    0x80, 0xcb, 0x25,
+    0x0a, 0x84, 0x06,
 ];
diff --git a/src/libcore/unicode/unicode_data.rs b/src/libcore/unicode/unicode_data.rs
index da4cd4e9b1da1..3e90028613ce1 100644
--- a/src/libcore/unicode/unicode_data.rs
+++ b/src/libcore/unicode/unicode_data.rs
@@ -1,128 +1,130 @@
 ///! This file is generated by src/tools/unicode-table-generator; do not edit manually!
 use super::range_search;
 
-pub const UNICODE_VERSION: (u32, u32, u32) = (12, 1, 0);
+pub const UNICODE_VERSION: (u32, u32, u32) = (13, 0, 0);
 
 #[rustfmt::skip]
 pub mod alphabetic {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (190, 37);
-    static BITSET_CHUNKS_MAP: [u8; 187] = [
-        6, 32, 10, 18, 19, 23, 21, 12, 7, 5, 0, 20, 14, 49, 49, 49, 49, 49, 49, 36, 49, 49, 49, 49,
-        49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 47, 49, 30, 8, 49, 49, 49, 49,
-        49, 49, 49, 49, 49, 49, 45, 0, 0, 0, 0, 0, 0, 0, 0, 4, 35, 17, 31, 16, 25, 24, 26, 13, 15,
-        44, 27, 0, 0, 49, 11, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 38, 1, 49, 49, 49, 49, 49, 48,
-        42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 28, 0, 0, 0, 0, 0, 29, 0, 0, 9, 0, 33, 2, 3, 0, 0,
-        0, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49,
-        49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, 41, 49, 49, 49,
-        43, 22, 49, 49, 49, 49, 40, 49, 49, 49, 49, 49, 49, 46,
+    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (196, 44);
+    static BITSET_CHUNKS_MAP: [u8; 196] = [
+        6, 32, 10, 18, 19, 23, 21, 12, 7, 5, 0, 20, 14, 50, 50, 50, 50, 50, 50, 37, 50, 50, 50, 50,
+        50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 49, 50, 30, 8, 50, 50, 50, 50,
+        50, 50, 50, 50, 50, 50, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 17, 31, 16, 25, 24, 26, 13, 15,
+        45, 27, 0, 0, 50, 11, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 50, 50, 50, 50, 50, 48,
+        50, 34, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 28, 0, 0, 0, 0, 0, 29, 0, 0, 9, 0, 33, 2, 3, 0, 0,
+        0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,
+        50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 42, 50, 50, 50,
+        43, 22, 50, 50, 50, 50, 41, 50, 50, 50, 50, 50, 50, 47, 0, 0, 0, 38, 0, 50, 50, 50, 50,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 50] = [
+    static BITSET_INDEX_CHUNKS: [[u8; 16]; 51] = [
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 248, 241, 38, 40],
-        [0, 0, 0, 0, 0, 0, 0, 0, 108, 133, 110, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 190, 200, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 248, 248, 248, 248, 248, 205, 248, 23, 134, 245, 68, 237],
-        [0, 0, 179, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 103, 99, 176, 248, 248, 248, 248, 248, 248, 248, 61, 0, 151, 217, 178],
-        [0, 145, 28, 0, 168, 221, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [48, 77, 248, 165, 201, 120, 184, 137, 91, 175, 143, 83, 206, 196, 248, 56],
-        [53, 0, 0, 0, 126, 15, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0],
-        [59, 54, 127, 199, 167, 186, 157, 114, 154, 84, 160, 115, 158, 66, 155, 21],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 254, 247, 39, 68],
+        [0, 0, 0, 0, 0, 0, 0, 0, 111, 135, 113, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 195, 205, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 254, 254, 254, 254, 254, 210, 254, 25, 136, 251, 71, 243],
+        [0, 0, 182, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 107, 103, 180, 254, 254, 254, 254, 254, 254, 254, 61, 0, 155, 222, 181],
+        [0, 148, 30, 0, 172, 226, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [48, 80, 254, 169, 206, 123, 189, 139, 95, 179, 145, 86, 211, 204, 254, 56],
+        [53, 0, 0, 0, 129, 17, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0],
+        [59, 54, 185, 203, 171, 191, 161, 117, 158, 87, 164, 118, 162, 67, 159, 23],
         [62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [91, 129, 164, 101, 248, 248, 248, 79, 248, 248, 248, 248, 230, 128, 135, 117],
-        [97, 0, 220, 144, 0, 0, 212, 44, 142, 240, 30, 97, 0, 0, 0, 0],
-        [116, 247, 219, 171, 188, 248, 104, 190, 0, 0, 0, 0, 0, 0, 0, 0],
-        [141, 185, 88, 0, 149, 213, 22, 0, 0, 0, 0, 89, 0, 0, 0, 0],
-        [147, 90, 35, 82, 98, 0, 153, 0, 85, 119, 29, 45, 86, 71, 18, 0],
-        [150, 32, 248, 107, 0, 81, 0, 0, 0, 0, 227, 17, 211, 105, 231, 19],
-        [162, 41, 161, 69, 163, 173, 123, 73, 106, 14, 124, 37, 1, 187, 121, 0],
-        [172, 240, 228, 170, 248, 248, 248, 248, 248, 229, 138, 235, 234, 24, 222, 125],
-        [208, 233, 248, 74, 204, 64, 140, 232, 63, 0, 0, 0, 0, 0, 0, 0],
-        [220, 97, 202, 86, 94, 78, 203, 9, 226, 80, 46, 0, 183, 11, 174, 67],
-        [231, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248],
-        [247, 248, 248, 248, 248, 248, 248, 248, 248, 209, 225, 95, 76, 75, 180, 25],
-        [248, 5, 96, 50, 72, 87, 248, 26, 132, 0, 198, 51, 159, 42, 0, 0],
-        [248, 8, 72, 72, 49, 0, 0, 0, 0, 0, 0, 0, 194, 5, 0, 89],
-        [248, 36, 248, 7, 0, 0, 139, 31, 143, 3, 93, 0, 55, 0, 0, 0],
-        [248, 62, 248, 248, 248, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [248, 118, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [248, 236, 166, 246, 136, 239, 248, 248, 248, 248, 215, 169, 182, 207, 214, 12],
-        [248, 248, 13, 130, 248, 248, 248, 248, 57, 146, 248, 65, 218, 248, 243, 177],
-        [248, 248, 191, 111, 197, 43, 0, 0, 248, 248, 248, 248, 91, 47, 0, 0],
-        [248, 248, 244, 248, 189, 223, 152, 70, 224, 210, 248, 148, 240, 242, 68, 100],
-        [248, 248, 248, 4, 248, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [248, 248, 248, 248, 35, 195, 248, 248, 248, 248, 248, 113, 0, 0, 0, 0],
-        [248, 248, 248, 248, 131, 240, 238, 109, 0, 181, 248, 122, 102, 216, 143, 27],
-        [248, 248, 248, 248, 248, 248, 86, 0, 248, 248, 248, 248, 248, 248, 248, 248],
-        [248, 248, 248, 248, 248, 248, 248, 248, 33, 0, 0, 0, 0, 0, 0, 0],
-        [248, 248, 248, 248, 248, 248, 248, 248, 97, 35, 0, 60, 65, 156, 16, 0],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 6, 0, 0, 0, 0, 0, 0],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 192, 248, 248, 248, 248, 248],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 20, 248, 248, 248, 248],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 72, 0, 0, 0, 0],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 81, 248, 248, 248],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 23, 0],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 193, 112],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 39],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 65],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 92],
-        [248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, 248],
+        [95, 131, 168, 105, 254, 254, 254, 82, 254, 254, 254, 254, 236, 130, 137, 120],
+        [101, 0, 225, 146, 151, 2, 217, 45, 144, 246, 32, 101, 0, 0, 0, 0],
+        [119, 253, 224, 175, 193, 254, 227, 195, 0, 0, 0, 0, 0, 0, 0, 0],
+        [143, 190, 91, 0, 153, 218, 24, 0, 0, 0, 0, 92, 0, 0, 66, 0],
+        [150, 94, 37, 85, 102, 0, 157, 0, 88, 122, 31, 46, 89, 74, 20, 0],
+        [154, 34, 254, 110, 0, 84, 0, 0, 0, 0, 233, 19, 216, 108, 237, 21],
+        [166, 42, 165, 72, 167, 177, 126, 76, 109, 16, 127, 38, 1, 192, 124, 0],
+        [176, 246, 234, 174, 254, 254, 254, 254, 254, 235, 140, 241, 240, 26, 228, 128],
+        [213, 239, 254, 77, 209, 64, 142, 238, 63, 0, 0, 0, 0, 0, 0, 0],
+        [225, 101, 207, 89, 98, 81, 208, 10, 232, 83, 147, 1, 188, 13, 178, 70],
+        [237, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254],
+        [253, 254, 254, 254, 254, 254, 254, 254, 254, 214, 231, 99, 79, 78, 183, 27],
+        [254, 6, 100, 50, 75, 90, 254, 28, 134, 0, 202, 51, 163, 43, 0, 0],
+        [254, 9, 75, 75, 49, 0, 0, 0, 0, 0, 69, 0, 199, 6, 195, 93],
+        [254, 41, 254, 8, 0, 0, 141, 33, 145, 4, 97, 0, 55, 0, 0, 0],
+        [254, 62, 254, 254, 254, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [254, 121, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [254, 242, 170, 252, 138, 245, 254, 254, 254, 254, 220, 173, 186, 212, 219, 14],
+        [254, 254, 15, 132, 254, 254, 254, 254, 57, 149, 254, 65, 223, 254, 249, 187],
+        [254, 254, 196, 114, 201, 44, 0, 0, 254, 254, 254, 254, 95, 47, 0, 0],
+        [254, 254, 250, 254, 194, 229, 156, 73, 230, 215, 254, 152, 246, 248, 71, 104],
+        [254, 254, 254, 5, 254, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [254, 254, 254, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [254, 254, 254, 254, 37, 200, 254, 254, 254, 254, 254, 116, 0, 0, 0, 0],
+        [254, 254, 254, 254, 133, 246, 244, 112, 0, 184, 254, 125, 106, 221, 145, 29],
+        [254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 254],
+        [254, 254, 254, 254, 254, 254, 254, 254, 35, 0, 0, 0, 0, 0, 0, 0],
+        [254, 254, 254, 254, 254, 254, 254, 254, 101, 37, 0, 60, 65, 160, 18, 0],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 7, 0, 0, 0, 0, 0, 0],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 197, 254, 254, 254, 254, 254],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 35, 254, 254, 254, 254],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 84, 254, 254, 254],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 11, 0, 0],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 25, 0],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 198, 115],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 40],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 96],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 125],
+        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254],
     ];
-    static BITSET: [u64; 249] = [
-        0, 1, 15, 17, 31, 63, 127, 179, 511, 1023, 2191, 4079, 4087, 8191, 8319, 16384, 65535,
-        131071, 262143, 4128527, 8388607, 8461767, 24870911, 67108863, 134217727, 276824575,
-        335544350, 486341884, 536805376, 536870911, 553648127, 1056964608, 1073692671, 1073741823,
-        1140785663, 2147483647, 2147485627, 4026540127, 4294934783, 8589934591, 47244640256,
-        64548249055, 68191066527, 68719476735, 115913785343, 137438953215, 549755813888,
-        1095220854783, 1099511627711, 1099511627775, 2199023190016, 2199023255551, 4398046511103,
-        8641373536127, 8791831609343, 8795690369023, 8796093022207, 13198434443263, 17592186044415,
-        35184321757183, 70368744112128, 88094074470339, 140737488355327, 140737488355328,
-        141836999983103, 281474976710655, 563017343310239, 1125625028935679, 1125899906842623,
-        1688915364814303, 2119858418286774, 2251795522912255, 2251799813685247, 3377704004976767,
-        3509778554814463, 3905461007941631, 4503595333443583, 4503599627370495, 8796093022142464,
-        9006649498927104, 9007192812290047, 9007199254740991, 15762594400829440, 17169970223906821,
-        17732925109967239, 18014398491652207, 18014398509481983, 20266198323101808,
-        36027697507139583, 36028792723996672, 36028792728190975, 36028797018963967,
-        72057594037927935, 90071992547409919, 143851303137705983, 144053615424700415,
-        144115188075855868, 144115188075855871, 288230371860938751, 297241973452963840,
-        301749971126844416, 319718190147960832, 576460743713488896, 576460743847706622,
-        576460748008488959, 576460752303359999, 576460752303423486, 576460752303423487,
-        790380184120328175, 1152640029630136575, 1152917029519358975, 1152921504591118335,
-        1152921504606845055, 1152921504606846975, 1153765996922689951, 2161727885562420159,
-        2251241253188403424, 2295745090394464220, 2305570330330005503, 2305843004918726656,
-        2305843004919250943, 2305843009196916483, 2305843009213693951, 3457638613854978028,
-        4323455298678290390, 4557642822898941951, 4575692405780512767, 4602678814877679616,
-        4611686017001275199, 4611686018360336384, 4611686018427322368, 4611686018427387903,
-        4656722014700830719, 6843210385291930244, 6881498031078244479, 6908521828386340863,
-        8935141660164089791, 8935423131384840192, 9168765891372858879, 9169328841326329855,
-        9187201948305063935, 9187343239835811327, 9216616637413720063, 9223372036854775807,
-        9223372041149743103, 9223934986808197120, 10371930679322607615, 10502394331027995967,
-        11241233151490523135, 13006395723845991295, 13258596753222922239, 13609596598936928288,
-        13834776580305453567, 13907115649320091647, 14082190885810440174, 14123225865944680428,
-        16212958624174047247, 16412803692974677999, 16424062692043104238, 16424062692043104239,
-        16424062692043243502, 16424625641996804079, 16429129241624174575, 16717361816799141871,
-        16717361816799216127, 16788293510930366511, 17005555242810474495, 17293822569102704639,
-        17581979622616071300, 17870283321271910397, 17870283321406070975, 17870283321406128127,
-        17978369712463020031, 18158513764145585631, 18158781978395017215, 18194542490281852927,
-        18410715276682199039, 18410715276690587772, 18428729675200069631, 18428729675200069632,
-        18433233274827440127, 18437455399478099968, 18437736874452713471, 18442240474082181119,
+    static BITSET: [u64; 255] = [
+        0, 1, 7, 15, 17, 31, 63, 127, 179, 511, 1023, 2047, 2191, 4079, 4087, 8191, 8319, 16384,
+        65535, 131071, 262143, 4128527, 4194303, 8461767, 24870911, 67108863, 134217727, 276824575,
+        335593502, 486341884, 536805376, 536870911, 553648127, 1056964608, 1073692671, 1073741823,
+        1140785663, 2147483647, 4026540127, 4294934783, 8589934591, 15032387515, 64548249055,
+        68191066527, 68719476735, 115913785343, 137438953215, 1095220854783, 1099511627711,
+        1099511627775, 2199023190016, 2199023255551, 4398046511103, 8641373536127, 8791831609343,
+        8795690369023, 8796093022207, 13198434443263, 17592186044415, 35184321757183,
+        70368744112128, 88094074470339, 140737488355327, 140737488355328, 141836999983103,
+        281474976710655, 281474976710656, 563017343310239, 844472174772224, 875211255709695,
+        1125625028935679, 1125899906842623, 1688915364814303, 2119858418286774, 2251795522912255,
+        2251799813685247, 3377704004976767, 3509778554814463, 3905461007941631, 4503595333443583,
+        4503599627370495, 8796093022142464, 9006649498927104, 9007192812290047, 9007199254740991,
+        15762594400829440, 17169970223906821, 17732925109967239, 18014398491652207,
+        18014398509481983, 20266198323101936, 36027697507139583, 36028792723996672,
+        36028792723996703, 36028792728190975, 36028797018963967, 72057594037927935,
+        90071992547409919, 143851303137705983, 144053615424700415, 144115188075855868,
+        144115188075855871, 288230371860938751, 297241973452963840, 301749971126844416,
+        319718190147960832, 576460743713488896, 576460743847706622, 576460752303359999,
+        576460752303423486, 576460752303423487, 790380184120328175, 1152640029630136575,
+        1152917029519358975, 1152921504591118335, 1152921504606845055, 1152921504606846975,
+        1153765996922689951, 2161727885562420159, 2251241253188403424, 2295745090394464220,
+        2305570330330005503, 2305843004918726656, 2305843004919250943, 2305843009196916483,
+        2305843009213693951, 3457638613854978030, 4323455298678290390, 4557642822898941951,
+        4575692405780512767, 4611686017001275199, 4611686018360336384, 4611686018427322368,
+        4611686018427387903, 4656722014700830719, 6843210385291930244, 6881498031078244479,
+        6908521828386340863, 8935141660164089791, 8935423131384840192, 9168765891372858879,
+        9169328841326329855, 9187201948305063935, 9187343239835811327, 9216616637413720063,
+        9223372036854775807, 9223372041149743103, 9223372586610589696, 9223934986808197120,
+        10371930679322607615, 10502394331027995967, 11078855083321979519, 11241233151490523135,
+        13006395723845991295, 13258596753222922239, 13609596598936928288, 13834776580305453567,
+        13907115649320091647, 14082190885810440174, 14123225865944680428, 16212958624174047247,
+        16412803692974677999, 16424062692043104238, 16424062692043104239, 16424062692043243502,
+        16424625641996804079, 16429129241624174575, 16717361816799141887, 16717361816799216127,
+        16788293510930366511, 17005555242810474495, 17293822569102704639, 17581979622616071300,
+        17870283321271910397, 17870283321406070975, 17870283321406128127, 17978369712463020031,
+        18158513764145585631, 18158781978395017215, 18194542490281852927, 18410715276682199039,
+        18428729675200069631, 18428729675200069632, 18433233274827440127, 18437455399478099968,
+        18437736870159843328, 18437736874452713471, 18437736874454812668, 18442240474082181119,
         18444492273895866367, 18445618173802708993, 18446181192473632767, 18446216308128218879,
         18446462598732840928, 18446462598732840959, 18446462598732840960, 18446462599806582783,
         18446462615912710143, 18446462667452317695, 18446463149025525759, 18446463629525450752,
-        18446463698110251007, 18446463698244468735, 18446464796682337663, 18446466966713532416,
+        18446463698244468735, 18446464796682337663, 18446466966713532671, 18446466996645134335,
         18446466996779287551, 18446471394825862144, 18446471394825863167, 18446480190918885375,
         18446498607738650623, 18446532967477018623, 18446602782178705022, 18446603336221163519,
         18446603336221196287, 18446638520593285119, 18446673709243564031, 18446708893632430079,
         18446740770879700992, 18446741595513422027, 18446741874686295551, 18446743249075830783,
         18446743798965862398, 18446744056529672000, 18446744060816261120, 18446744068886102015,
-        18446744069414584320, 18446744069414601696, 18446744069414649855, 18446744069456527359,
-        18446744069548736512, 18446744069548802046, 18446744069683019775, 18446744069951455231,
-        18446744070421282815, 18446744070446333439, 18446744070475743231, 18446744070488326143,
-        18446744071553646463, 18446744071562067967, 18446744073696837631, 18446744073701162813,
-        18446744073707454463, 18446744073709027328, 18446744073709355007, 18446744073709419615,
-        18446744073709486080, 18446744073709520895, 18446744073709543424, 18446744073709550079,
-        18446744073709550595, 18446744073709551579, 18446744073709551599, 18446744073709551614,
-        18446744073709551615,
+        18446744069414584320, 18446744069414601696, 18446744069414617087, 18446744069414649855,
+        18446744069456527359, 18446744069548736512, 18446744069548802046, 18446744069683019775,
+        18446744069951455231, 18446744070421282815, 18446744070446333439, 18446744070475743231,
+        18446744070488326143, 18446744071553646463, 18446744071562067967, 18446744073696837631,
+        18446744073701162813, 18446744073707454463, 18446744073709027328, 18446744073709355007,
+        18446744073709419615, 18446744073709486080, 18446744073709520895, 18446744073709543424,
+        18446744073709550079, 18446744073709550595, 18446744073709551579, 18446744073709551599,
+        18446744073709551614, 18446744073709551615,
     ];
 
     pub fn lookup(c: char) -> bool {
@@ -141,82 +143,83 @@ pub mod case_ignorable {
     static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 33);
     static BITSET_CHUNKS_MAP: [u8; 125] = [
         25, 14, 21, 30, 28, 4, 17, 23, 22, 0, 0, 16, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 13, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0, 0, 0, 3, 6, 9, 0, 7, 11, 32, 31, 26, 29, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0,
-        10, 0, 8, 0, 20, 0, 12, 0, 1,
+        10, 0, 8, 0, 19, 0, 12, 0, 1,
     ];
     static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 47, 52],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 171, 2],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 88, 134, 38],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 102, 6, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 76, 26, 0, 146, 136, 79, 43, 117],
-        [0, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 58, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 165, 97, 75, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 48, 0, 114, 0, 0],
-        [0, 0, 0, 0, 0, 170, 68, 0, 0, 7, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0],
-        [0, 0, 0, 28, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 133, 0, 0, 0, 0, 15, 160, 45, 84, 51, 78, 12, 109],
-        [0, 0, 11, 0, 0, 30, 161, 90, 35, 80, 0, 69, 173, 13, 81, 129],
-        [0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 131, 0, 85, 0, 148, 0, 175, 73, 0, 0, 0, 0, 0, 0, 0],
-        [20, 4, 62, 0, 118, 0, 0, 0, 32, 154, 145, 0, 124, 89, 67, 86],
-        [25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [59, 0, 0, 150, 70, 24, 132, 60, 100, 122, 163, 99, 0, 46, 0, 66],
-        [63, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0],
-        [71, 33, 0, 178, 123, 83, 120, 137, 121, 98, 121, 167, 153, 55, 3, 18],
-        [72, 149, 36, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [104, 133, 0, 110, 174, 105, 177, 166, 0, 0, 0, 0, 0, 0, 155, 139],
-        [107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [111, 50, 106, 0, 0, 0, 0, 0, 0, 0, 172, 179, 179, 112, 9, 0],
-        [113, 0, 0, 0, 0, 0, 0, 49, 142, 34, 31, 0, 0, 0, 0, 0],
-        [116, 0, 42, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [140, 93, 37, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0],
-        [159, 0, 101, 0, 158, 10, 29, 0, 0, 0, 0, 91, 0, 0, 0, 0],
-        [162, 56, 153, 54, 125, 53, 0, 27, 115, 21, 126, 19, 108, 144, 127, 8],
-        [168, 41, 151, 5, 0, 0, 157, 39, 156, 1, 103, 0, 65, 0, 0, 0],
-        [169, 147, 130, 17, 96, 87, 143, 16, 138, 0, 0, 64, 125, 95, 0, 0],
-        [176, 179, 0, 0, 179, 179, 179, 77, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 47, 57],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 173, 3],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 90, 136, 38],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 104, 7, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 78, 27, 0, 148, 138, 81, 44, 119],
+        [0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 58, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 167, 99, 77, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 48, 0, 116, 0, 0],
+        [0, 0, 0, 0, 0, 172, 70, 0, 0, 8, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 67, 0, 0, 24, 0, 0],
+        [0, 0, 0, 29, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 135, 0, 0, 0, 0, 16, 162, 46, 86, 51, 80, 13, 111],
+        [0, 0, 12, 0, 0, 43, 163, 92, 35, 82, 0, 71, 175, 14, 83, 131],
+        [0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 133, 0, 87, 0, 150, 0, 178, 75, 0, 0, 0, 0, 0, 0, 0],
+        [20, 5, 61, 0, 120, 0, 0, 0, 32, 156, 176, 1, 126, 91, 69, 88],
+        [26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [62, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0],
+        [66, 0, 0, 152, 72, 25, 134, 59, 102, 124, 165, 101, 0, 64, 0, 68],
+        [73, 33, 0, 181, 125, 85, 122, 139, 123, 100, 123, 169, 155, 54, 4, 18],
+        [74, 151, 36, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [106, 135, 0, 112, 177, 107, 180, 168, 0, 0, 0, 0, 0, 0, 157, 142],
+        [109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [113, 50, 108, 0, 0, 0, 0, 0, 0, 0, 174, 182, 182, 114, 10, 0],
+        [115, 0, 0, 0, 141, 5, 0, 49, 145, 34, 31, 0, 0, 0, 0, 0],
+        [118, 0, 42, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [143, 95, 37, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0],
+        [161, 0, 103, 0, 160, 11, 30, 0, 0, 0, 0, 93, 0, 0, 0, 0],
+        [164, 55, 155, 53, 127, 52, 2, 28, 117, 21, 128, 19, 110, 147, 129, 9],
+        [170, 41, 153, 6, 0, 0, 159, 39, 158, 1, 105, 0, 65, 0, 0, 0],
+        [171, 149, 132, 17, 98, 89, 146, 23, 140, 0, 0, 63, 127, 97, 0, 0],
+        [179, 182, 0, 0, 182, 182, 182, 79, 0, 0, 0, 0, 0, 0, 0, 0],
     ];
-    static BITSET: [u64; 180] = [
-        0, 1, 3, 4, 8, 13, 15, 28, 64, 176, 191, 1016, 1792, 2047, 4080, 4096, 7680, 8192, 8193,
-        16192, 30720, 32704, 32768, 131008, 262016, 2097152, 2359296, 6030336, 8323072, 10682368,
-        33554432, 58719232, 159383552, 234881024, 243138688, 402587711, 536805376, 536879204,
-        546307648, 805306369, 1073741824, 1073741916, 2113929216, 3221225472, 3758096384,
-        4026531840, 4160749568, 4294934528, 4294967296, 4512022528, 5368709120, 17179869183,
-        47244640256, 51539615774, 51539619904, 51543810078, 51545914817, 66035122176, 412316860416,
-        412316862532, 412316893184, 1030792151040, 2199023255648, 8641373536127, 8763880767488,
-        17303886364672, 36421322670080, 65128884076547, 65970697670631, 68168642985984,
-        70093866270720, 70368739983360, 136957967529984, 140737488355328, 263882790666240,
-        281470547525648, 281470682333183, 281474976710655, 281474976710656, 281474976710657,
-        281479271675905, 562675075514368, 562949953355776, 563001509683710, 844424930131968,
-        985162418487296, 1023920203366400, 2251799813685248, 3377699721314304, 4494803534348292,
-        4503599627370678, 6755399441055744, 7881299349733376, 8444256867844096, 8725724278030336,
-        8760633772212225, 8989057312882695, 9042383626829823, 9851624185018758, 24822575045541890,
-        28848986089586688, 30958948903026688, 35747322042253312, 53805701016846336,
-        58529202969772032, 72066390130950143, 112767012056334336, 143833713099145216,
-        189151184399892480, 216172782113783808, 220713756545974272, 288301294651703296,
-        302022650010533887, 504262420777140224, 558446353793941504, 572520102629474304,
-        593978171557150752, 1008806350890729472, 1009933895770046464, 1152921504606846976,
-        1152921504606846978, 1152921504606846982, 1153202979583561736, 1441151880758558727,
-        1715871458028158991, 1729382256910270467, 2301902359539744768, 2305843009196908767,
-        2305843009213693952, 2612078987781865472, 2771965570646540291, 3458764513820540928,
-        3731232291276455943, 4539628424389459968, 4589168020290535424, 4611404543450677248,
-        4611686018494513280, 4611686069967003678, 4671217976001691648, 6917775322003857411,
-        7421334051581067264, 8070450532247928832, 8788774672813524990, 9205357638345293827,
-        9222809086901354496, 9223091111633879040, 9223372036854775808, 9223372036854775935,
-        9223512774343131136, 9224216320050987008, 9224497932466651184, 9653465801268658176,
-        9727775195120332910, 10376293541461622786, 11526998316797657088, 11529215046068469760,
-        12103423998558208000, 12699025049277956096, 13005832773892571136, 13798747783286489088,
-        13832665517980123136, 13835058055282032640, 13835058055282163729, 13951307220663664640,
-        17870283321406128128, 17906312118425092095, 18158513697557839871, 18158513749097456062,
-        18374686479671623680, 18374686479671623682, 18444496122186563584, 18445618173802708992,
-        18446462598732840960, 18446462598733004800, 18446726481523507200, 18446744069414584320,
-        18446744069414584322, 18446744073575333888, 18446744073709027328, 18446744073709551615,
+    static BITSET: [u64; 183] = [
+        0, 1, 2, 3, 4, 8, 13, 15, 28, 64, 176, 191, 1016, 1792, 2047, 4080, 4096, 8192, 8193,
+        16192, 30720, 32704, 32768, 40448, 131008, 262016, 2097152, 2359296, 6030336, 8323072,
+        10682368, 58719232, 159383552, 234881024, 243138688, 402587711, 536805376, 536879204,
+        546307648, 805306369, 1073741824, 1073741916, 2113929216, 2181038080, 3221225472,
+        3758096384, 4026531840, 4294934528, 4294967296, 4512022528, 5368709120, 17179869183,
+        51539615774, 51539619904, 51545907230, 51545914817, 66035122176, 115964116992, 412316860416,
+        412316893184, 1030792151040, 2199023255648, 8641373536127, 8763880767488, 15397323538432,
+        17303886364672, 18004502906948, 26388279066624, 36421322670080, 65128884076547,
+        65970697670631, 68168642985984, 70093866270720, 70368739983360, 136957967529984,
+        140737488355328, 263882790666240, 281470547525648, 281470682333183, 281474976710655,
+        281474976710656, 281474976710657, 281479271675905, 562675075514368, 562949953355776,
+        563001509683710, 844424930131968, 985162418487296, 1023920203366400, 2251799813685248,
+        3377699721314304, 4494803534348292, 4503599627370678, 6755399441055744, 7881299349733376,
+        8444256867844096, 8725724278030336, 8760633772212225, 8989057312882695, 9042383626829823,
+        9851624185018758, 24822575045541890, 28848986089586688, 30958948903026688,
+        35747322042253312, 53805701016846336, 58529202969772032, 72066390130950143,
+        112767012056334336, 143833713099145216, 189151184399892480, 216172782113783808,
+        220713756545974272, 288301294651703296, 302022650010533887, 504262420777140224,
+        558446353793941504, 572520102629474304, 593978171557150752, 1008806350890729472,
+        1009933895770046464, 1152921504606846976, 1152921504606846978, 1152921504606846982,
+        1153202979583561736, 1441151880758558727, 1715871458028158991, 1729382256910270467,
+        2301902359539744768, 2305843009196908767, 2305843009213693952, 2612078987781865472,
+        2771965570646540291, 3458764513820540928, 3731232291276455943, 4539628424389459968,
+        4589168020290535424, 4611404543450677248, 4611686018494513280, 4611686069967003678,
+        4671217976001691648, 6341068275337658368, 6917775322003857411, 7421334051581067264,
+        8070450532247928832, 8788774672813524990, 9205357638345293827, 9222809086901354496,
+        9223372036854775808, 9223372036854775935, 9223512774343131136, 9224216320050987008,
+        9224497932466651184, 9653465801268658176, 9727775195120332910, 10376293541461622786,
+        11526998316797657088, 11529215046068469760, 12103423998558208000, 12699025049277956096,
+        13005832773892571136, 13798747783286489088, 13832665517980123136, 13835058055282032640,
+        13835058055282163729, 13951307220663664640, 17870283321406128128, 17906312118425092095,
+        18158513697557839871, 18158513749097456062, 18374686479671623680, 18374686479671623682,
+        18444496122186563584, 18445618173802708992, 18446462598732840960, 18446462598733004800,
+        18446463148488654848, 18446726481523507200, 18446744069414584320, 18446744069414584322,
+        18446744073575333888, 18446744073709027328, 18446744073709551615,
     ];
 
     pub fn lookup(c: char) -> bool {
@@ -264,13 +267,13 @@ pub mod cased {
     static BITSET: [u64; 63] = [
         0, 15, 24, 511, 1023, 4087, 65535, 16253055, 134217726, 536805376, 1073741823, 4294967295,
         133143986179, 4398046511103, 36009005809663, 70368744177663, 2251799813685247,
-        3509778554814463, 144115188074807295, 297241973452963840, 504403158265495676,
+        3509778554814463, 144115188074807295, 297241973452963840, 531424756029720572,
         576460743713488896, 576460743847706622, 1152921504591118335, 2295745090394464220,
         4557642822898941951, 4611686017001275199, 6908521828386340863, 8935141660164089791,
         9223934986808197120, 13605092999309557792, 16717361816799216127, 16717361816799223999,
         17005555242810474495, 17446871633794956420, 17870283321271910397, 17870283321406128127,
         18410715276682199039, 18428729675200069631, 18428729675200069632, 18437736874452713471,
-        18446462598732840959, 18446462598732840960, 18446463698110251007, 18446466996779287551,
+        18446462598732840959, 18446462598732840960, 18446464797621878783, 18446466996779287551,
         18446603336221163519, 18446603336221196287, 18446741874686295551, 18446743249075830783,
         18446744056529672000, 18446744056529682432, 18446744069414584320, 18446744069414601696,
         18446744069422972927, 18446744070475743231, 18446744071562067967, 18446744073707454463,
@@ -317,72 +320,72 @@ pub mod grapheme_extend {
     static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 30);
     static BITSET_CHUNKS_MAP: [u8; 123] = [
         4, 15, 21, 27, 25, 3, 18, 23, 17, 0, 0, 14, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0, 0, 0, 2, 7, 10, 0, 8, 12, 29, 28, 24, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0,
-        11, 0, 9, 0, 20, 0, 13,
+        11, 0, 9, 0, 19, 0, 13,
     ];
     static BITSET_INDEX_CHUNKS: [[u8; 16]; 31] = [
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 18, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 70, 102, 29],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 138, 62, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 83, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 35, 66, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 35, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 117, 0, 0, 45, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 130, 78, 60, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 37, 0, 90, 0, 0],
-        [0, 0, 0, 0, 0, 129, 54, 0, 0, 3, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0],
-        [0, 0, 0, 19, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 67, 0, 114, 0, 137, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 7, 0, 0, 0, 125, 5, 24, 63, 0, 55, 135, 9, 64, 100],
-        [0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [10, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [12, 0, 48, 0, 92, 0, 0, 0, 25, 119, 113, 0, 96, 71, 53, 68],
-        [46, 0, 0, 116, 57, 17, 101, 44, 81, 94, 127, 80, 0, 0, 0, 52],
-        [49, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0],
-        [56, 26, 0, 136, 95, 43, 107, 105, 93, 79, 93, 132, 128, 42, 104, 20],
-        [59, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [85, 0, 0, 87, 0, 0, 0, 131, 0, 0, 0, 0, 0, 0, 0, 0],
-        [89, 0, 0, 0, 0, 0, 0, 38, 110, 27, 22, 0, 0, 0, 0, 0],
-        [109, 74, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0],
-        [124, 0, 82, 0, 123, 6, 21, 0, 0, 0, 0, 72, 0, 0, 0, 0],
-        [126, 40, 118, 39, 108, 41, 0, 34, 91, 14, 97, 13, 86, 112, 98, 4],
-        [133, 32, 120, 2, 0, 0, 122, 30, 121, 1, 84, 0, 51, 0, 0, 0],
-        [134, 115, 88, 0, 77, 69, 111, 11, 106, 0, 0, 50, 108, 76, 0, 0],
-        [137, 138, 0, 0, 138, 138, 138, 62, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 20, 46],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 74, 106, 31],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 66, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 87, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 37, 70, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 37, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 48, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 134, 82, 64, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 39, 0, 94, 0, 0],
+        [0, 0, 0, 0, 0, 133, 58, 0, 0, 5, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 55, 0, 0, 18, 0, 0],
+        [0, 0, 0, 21, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 71, 0, 118, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 9, 0, 0, 0, 129, 7, 26, 67, 0, 59, 140, 11, 68, 104],
+        [0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [12, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [13, 0, 50, 0, 96, 0, 0, 0, 27, 123, 139, 1, 100, 75, 57, 72],
+        [51, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0],
+        [54, 0, 0, 120, 61, 19, 105, 47, 85, 98, 131, 84, 0, 0, 0, 56],
+        [60, 28, 0, 141, 99, 45, 111, 109, 97, 83, 97, 136, 132, 44, 108, 22],
+        [63, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [89, 0, 0, 91, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0],
+        [93, 0, 0, 0, 113, 3, 0, 40, 115, 29, 24, 0, 0, 0, 0, 0],
+        [114, 78, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0],
+        [128, 0, 86, 0, 127, 8, 23, 0, 0, 0, 0, 76, 0, 0, 0, 0],
+        [130, 42, 122, 41, 112, 43, 2, 36, 95, 15, 101, 14, 90, 117, 102, 6],
+        [137, 34, 124, 4, 0, 0, 126, 32, 125, 1, 88, 0, 53, 0, 0, 0],
+        [138, 119, 92, 0, 81, 73, 116, 17, 110, 0, 0, 52, 112, 80, 0, 0],
+        [142, 143, 0, 0, 143, 143, 143, 66, 0, 0, 0, 0, 0, 0, 0, 0],
     ];
-    static BITSET: [u64; 139] = [
-        0, 1, 13, 28, 64, 182, 191, 1016, 2032, 2047, 4096, 7680, 14336, 16128, 32640, 32768,
-        131008, 262016, 491520, 8323072, 8396801, 10682368, 58719232, 100663296, 134152192,
+    static BITSET: [u64; 144] = [
+        0, 1, 2, 8, 13, 28, 64, 182, 191, 1016, 2032, 2047, 4096, 14336, 16128, 32640, 32768,
+        40448, 131008, 262016, 491520, 8323072, 8396801, 10682368, 58719232, 100663296, 134152192,
         159383552, 234881024, 243138688, 536879204, 537919040, 805306369, 1073741824, 1073741916,
         1610612736, 2153546752, 3221225472, 3758096384, 4294967296, 4512022528, 51545911364,
-        51545914817, 51548004382, 51552198686, 51556262398, 137438953472, 412316860416,
-        412316862532, 1030792151040, 2199023255648, 8641373536127, 8763880767488, 17303886364672,
-        36421322670080, 65128884076547, 65970697670631, 67755789254656, 69200441769984,
-        70093866270720, 263882790666240, 277076930199552, 281470547525648, 281470681808895,
-        281474976710655, 281479271675904, 562675075514368, 562949953355776, 844424930131968,
-        985162418487296, 1023920203366400, 2251799813685248, 3377699721314304, 4494803534348292,
-        6755399441055744, 7881299349733376, 8444256867844096, 8725724278030336, 8760633780600833,
-        8989057312882695, 9042383626829823, 9851624185018758, 18067175067615234, 28848986089586688,
-        30958948903026688, 35747322042253312, 53805701016846336, 58529202969772032,
-        189151184399892480, 220713756545974272, 466122561432846339, 504262420777140224,
-        558446353793941504, 572520102629474304, 1009933895770046464, 1152921504606846982,
-        1152921504606851080, 1441151880758558727, 1724878657282899983, 2301902359539744768,
-        2305843009196908767, 2305843009213693952, 2310337812748042240, 3731232291276455943,
-        4589168020290535424, 4609293481125347328, 4611686018427387908, 4611686069975392286,
-        4671217976001691648, 5764607523034234882, 6341068275337658371, 7421334051581067264,
-        8788774672813524990, 9205357638345293827, 9222809086901354496, 9223090561878065152,
-        9223372036854775808, 9223372036854775935, 9224497932466651184, 9727775195120332910,
-        10376293541461622786, 11526998316797657088, 11959590285459062784, 12103423998558208000,
-        12699165786766311424, 13005832773892571136, 13798747783286489088, 13835058055282032640,
-        13835058055282163729, 13951307220663664640, 14987979559889010690, 17872468738205286400,
-        17906312118425092095, 18158513697557839871, 18158513749097456062, 18374686479671623680,
-        18374686479671623682, 18446462598732972032, 18446744056529158144, 18446744069414584320,
-        18446744073709551615,
+        51545914817, 51548004382, 51554295838, 51556262398, 68719476736, 137438953472, 412316860416,
+        1030792151040, 2199023255648, 8641373536127, 8763880767488, 17303886364672, 18004502906948,
+        26388279066624, 36421322670080, 65128884076547, 65970697670631, 67755789254656,
+        69200441769984, 70093866270720, 263882790666240, 277076930199552, 281470547525648,
+        281470681808895, 281474976710655, 281479271675904, 562675075514368, 562949953355776,
+        844424930131968, 985162418487296, 1023920203366400, 2251799813685248, 3377699721314304,
+        4494803534348292, 6755399441055744, 7881299349733376, 8444256867844096, 8725724278030336,
+        8760633780600833, 8989057312882695, 9042383626829823, 9851624185018758, 18067175067615234,
+        28848986089586688, 30958948903026688, 35747322042253312, 53805701016846336,
+        58529202969772032, 189151184399892480, 220713756545974272, 466122561432846339,
+        504262420777140224, 558446353793941504, 572520102629474304, 1009933895770046464,
+        1152921504606846982, 1152921504606851080, 1441151880758558727, 1724878657282899983,
+        2301902359539744768, 2305843009196908767, 2305843009213693952, 2310337812748042240,
+        3731232291276455943, 4589168020290535424, 4609293481125347328, 4611686018427387908,
+        4611686069975392286, 4671217976001691648, 5764607523034234882, 6341068275337658371,
+        6341349750314369024, 7421334051581067264, 8788774672813524990, 9205357638345293827,
+        9222809086901354496, 9223372036854775808, 9223372036854775935, 9224497932466651184,
+        9727775195120332910, 10376293541461622786, 11526998316797657088, 11959590285459062784,
+        12103423998558208000, 12699165786766311424, 13005832773892571136, 13798747783286489088,
+        13835058055282032640, 13835058055282163729, 13951307220663664640, 14987979559889010690,
+        17872468738205286400, 17906312118425092095, 18158513697557839871, 18158513749097456062,
+        18374686479671623680, 18374686479671623682, 18446462598732840960, 18446462598732972032,
+        18446744056529158144, 18446744069414584320, 18446744073709551615,
     ];
 
     pub fn lookup(c: char) -> bool {
@@ -430,7 +433,7 @@ pub mod lowercase {
         133143986179, 274877905920, 1099509514240, 4398046445568, 17592185782272, 36009005809663,
         46912496118442, 187649984473770, 281474972516352, 2251799813685247, 2339875276368554,
         4503599560261632, 61925590106570972, 71777214282006783, 72057592964186127,
-        144115188074807295, 297241973452963840, 504403158265495560, 576460743713488896,
+        144115188074807295, 297241973452963840, 522417556774978824, 576460743713488896,
         1152921487426978047, 1152921504590069760, 1814856824841797631, 3607524039012697088,
         4362299189061746720, 4539628424389459968, 4601013482110844927, 4611405638684049471,
         4674456033467236607, 6172933889249159850, 9223934986808197120, 10663022717737544362,
@@ -439,7 +442,7 @@ pub mod lowercase {
         12298110845996498944, 15324248332066007893, 16596095761559859497, 16717361816799215616,
         16987577794709946364, 17293822586148356092, 18158513701852807104, 18410715274543104000,
         18428729675466407935, 18446462598732840960, 18446462598732858304, 18446462598737002495,
-        18446463698110251007, 18446673704966422527, 18446726481523572736, 18446739675663105535,
+        18446464797621878783, 18446673704966422527, 18446726481523572736, 18446739675663105535,
         18446739675663106031, 18446742974197923840, 18446744056529682432, 18446744069414584320,
         18446744073709529733, 18446744073709551615,
     ];
@@ -457,56 +460,57 @@ pub mod lowercase {
 
 #[rustfmt::skip]
 pub mod n {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 11);
-    static BITSET_CHUNKS_MAP: [u8; 124] = [
-        30, 7, 10, 24, 18, 3, 28, 20, 23, 27, 0, 15, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 2, 12, 17, 25, 16, 22, 19, 14, 21, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 6, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        4, 1, 0, 0, 9, 0, 13, 26,
+    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (127, 0);
+    static BITSET_CHUNKS_MAP: [u8; 127] = [
+        31, 8, 11, 25, 19, 4, 29, 21, 24, 28, 0, 16, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 3, 13, 18, 26, 17, 23, 20, 15, 22, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        5, 2, 0, 0, 10, 0, 14, 27, 12, 0, 1,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 33] = [
+    static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 48],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 42, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 21, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 46, 0, 0, 0, 2],
-        [0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 30, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 30, 0, 44, 0, 30, 0, 30, 0, 40, 0, 33],
-        [0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 36, 43, 4, 0, 0, 0, 0, 51, 22, 3, 0, 12],
-        [0, 0, 0, 6, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 61, 46, 0, 0, 0, 0, 59, 0, 0, 23, 9, 0, 0],
-        [0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 2, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 0],
-        [0, 14, 0, 14, 0, 0, 0, 0, 0, 14, 0, 2, 50, 0, 0, 0],
-        [0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 25, 0, 0, 0, 14, 24, 0, 0, 0, 0, 0, 0, 0, 0, 10],
-        [0, 31, 0, 46, 64, 0, 0, 38, 0, 0, 0, 46, 0, 0, 0, 0],
-        [0, 45, 2, 0, 0, 70, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 58, 0, 30, 0, 41, 0, 30, 0, 14, 0, 14, 35, 0, 0, 0],
-        [0, 62, 29, 60, 17, 0, 54, 69, 0, 56, 19, 27, 0, 63, 28, 0],
-        [0, 65, 37, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 68, 18, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 8, 0],
-        [14, 0, 0, 0, 0, 7, 0, 16, 0, 0, 15, 0, 0, 14, 46, 0],
-        [39, 0, 0, 14, 2, 0, 0, 47, 0, 14, 0, 0, 0, 0, 0, 46],
-        [46, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [49, 0, 0, 0, 0, 0, 11, 0, 24, 20, 66, 0, 0, 0, 0, 0],
-        [72, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 49],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 43, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 22, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 47, 0, 0, 0, 2],
+        [0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 31, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 31, 0, 45, 0, 31, 0, 31, 0, 41, 0, 34],
+        [0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 37, 44, 4, 0, 0, 0, 0, 52, 23, 3, 0, 13],
+        [0, 0, 0, 7, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 35, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 62, 47, 0, 0, 0, 0, 60, 0, 0, 24, 10, 0, 5],
+        [0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0],
+        [0, 15, 0, 15, 0, 0, 0, 0, 0, 15, 0, 2, 51, 0, 0, 0],
+        [0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 26, 0, 0, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 11],
+        [0, 32, 0, 47, 65, 0, 0, 39, 0, 0, 0, 47, 0, 0, 0, 0],
+        [0, 46, 2, 0, 0, 71, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 59, 0, 31, 0, 42, 0, 31, 0, 15, 0, 15, 36, 0, 0, 0],
+        [0, 63, 30, 61, 18, 0, 55, 70, 0, 57, 20, 28, 0, 64, 29, 0],
+        [0, 66, 38, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 69, 19, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 9, 0],
+        [15, 0, 0, 0, 0, 8, 0, 17, 0, 0, 16, 0, 0, 15, 47, 0],
+        [40, 0, 0, 15, 2, 0, 0, 48, 0, 15, 0, 0, 0, 0, 0, 47],
+        [47, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [50, 0, 0, 0, 0, 0, 12, 0, 25, 21, 67, 0, 0, 0, 0, 0],
+        [73, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
     ];
-    static BITSET: [u64; 73] = [
-        0, 999, 1023, 1026, 3072, 8191, 65408, 65472, 1048575, 1966080, 2097151, 3932160, 4063232,
-        8388607, 67043328, 67044351, 134152192, 264241152, 268435455, 3758096384, 4294901504,
-        17112694784, 64424509440, 549218942976, 4393751543808, 35184372023296, 140737488355327,
-        272678883688448, 279275953455104, 280925220896768, 281200098803712, 281474976448512,
-        492581209243648, 2251524935778304, 2251795518717952, 4503595332403200, 4503599627370368,
-        8708132091985919, 9007190731849728, 17732923532771328, 71212894229889024,
+    static BITSET: [u64; 74] = [
+        0, 999, 1023, 1026, 3072, 4064, 8191, 65408, 65472, 1048575, 1966080, 2097151, 3932160,
+        4063232, 8388607, 67043328, 67044351, 134152192, 264241152, 268435455, 3758096384,
+        4294901504, 17112694784, 64424509440, 549218942976, 4393751543808, 35184372023296,
+        140737488355327, 272678883688448, 279275953455104, 280925220896768, 281200098803712,
+        281474976448512, 492581209243648, 2251524935778304, 2251795518717952, 4503595332403200,
+        4503599627370368, 8708132091985919, 9007190731849728, 17732923532771328, 71212894229889024,
         144114915328655360, 144115183780888576, 144115188075855871, 284007976623144960,
         284008251501051904, 287948901175001088, 287948901242044416, 287953294926544896,
         504407547722072192, 1152640029630136320, 1152921496016912384, 2305840810190438400,
@@ -541,28 +545,28 @@ pub mod uppercase {
     static BITSET_INDEX_CHUNKS: [[u8; 16]; 17] = [
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 10, 0, 38, 46, 44, 2],
-        [0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 51, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 60, 62, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 54, 0, 0, 0, 0, 0, 43, 43, 40, 43, 56, 23, 34, 35],
-        [0, 0, 57, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 9, 0, 38, 46, 44, 28],
+        [0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 51, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 60, 62, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 54, 0, 0, 0, 0, 0, 43, 43, 40, 43, 56, 22, 34, 35],
+        [0, 0, 57, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 66, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 30],
-        [0, 11, 0, 12, 50, 37, 36, 45, 47, 6, 0, 0, 0, 49, 18, 53],
-        [15, 0, 60, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [22, 52, 43, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [25, 39, 42, 41, 59, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [58, 65, 29, 17, 48, 63, 31, 20, 55, 61, 64, 32, 28, 21, 16, 4],
+        [0, 0, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 30],
+        [0, 10, 0, 11, 50, 37, 36, 45, 47, 5, 0, 0, 0, 49, 17, 53],
+        [14, 0, 60, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [21, 52, 43, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [24, 39, 42, 41, 59, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [58, 65, 29, 16, 48, 63, 31, 19, 55, 61, 64, 32, 27, 20, 15, 3],
     ];
     static BITSET: [u64; 67] = [
-        0, 8, 116, 1023, 1024, 8383, 21882, 65535, 1048575, 8388607, 89478485, 134217726,
-        2139095039, 4294967295, 17179869183, 1099511627775, 2199023190016, 4398046445568,
-        17575006099264, 23456248059221, 70368743129088, 140737484161024, 140737488355327,
-        280378317225728, 281470681743392, 281474976710655, 1169903278445909, 2251799813685247,
-        9007198986305536, 17977448100528131, 18014398509481983, 288230371856744511,
+        0, 8, 1023, 1024, 8383, 21882, 65535, 1048575, 8388607, 89478485, 134217726, 2139095039,
+        4294967295, 17179869183, 1099511627775, 2199023190016, 4398046445568, 17575006099264,
+        23456248059221, 70368743129088, 140737484161024, 140737488355327, 280378317225728,
+        281470681743392, 281474976710655, 1169903278445909, 2251799813685247, 9007198986305536,
+        9007199254741748, 17977448100528131, 18014398509481983, 288230371856744511,
         576460735123554305, 576460743713488896, 1080863910568919040, 1080897995681042176,
         1274187559846268630, 3122495741643543722, 6148633210533183488, 6148914689804861440,
         6148914690880001365, 6148914691236506283, 6148914691236516865, 6148914691236517205,
@@ -1202,20 +1206,21 @@ pub mod conversions {
         ('\u{a7ba}', ['\u{a7bb}', '\u{0}', '\u{0}']), ('\u{a7bc}', ['\u{a7bd}', '\u{0}', '\u{0}']),
         ('\u{a7be}', ['\u{a7bf}', '\u{0}', '\u{0}']), ('\u{a7c2}', ['\u{a7c3}', '\u{0}', '\u{0}']),
         ('\u{a7c4}', ['\u{a794}', '\u{0}', '\u{0}']), ('\u{a7c5}', ['\u{282}', '\u{0}', '\u{0}']),
-        ('\u{a7c6}', ['\u{1d8e}', '\u{0}', '\u{0}']), ('\u{ff21}', ['\u{ff41}', '\u{0}', '\u{0}']),
-        ('\u{ff22}', ['\u{ff42}', '\u{0}', '\u{0}']), ('\u{ff23}', ['\u{ff43}', '\u{0}', '\u{0}']),
-        ('\u{ff24}', ['\u{ff44}', '\u{0}', '\u{0}']), ('\u{ff25}', ['\u{ff45}', '\u{0}', '\u{0}']),
-        ('\u{ff26}', ['\u{ff46}', '\u{0}', '\u{0}']), ('\u{ff27}', ['\u{ff47}', '\u{0}', '\u{0}']),
-        ('\u{ff28}', ['\u{ff48}', '\u{0}', '\u{0}']), ('\u{ff29}', ['\u{ff49}', '\u{0}', '\u{0}']),
-        ('\u{ff2a}', ['\u{ff4a}', '\u{0}', '\u{0}']), ('\u{ff2b}', ['\u{ff4b}', '\u{0}', '\u{0}']),
-        ('\u{ff2c}', ['\u{ff4c}', '\u{0}', '\u{0}']), ('\u{ff2d}', ['\u{ff4d}', '\u{0}', '\u{0}']),
-        ('\u{ff2e}', ['\u{ff4e}', '\u{0}', '\u{0}']), ('\u{ff2f}', ['\u{ff4f}', '\u{0}', '\u{0}']),
-        ('\u{ff30}', ['\u{ff50}', '\u{0}', '\u{0}']), ('\u{ff31}', ['\u{ff51}', '\u{0}', '\u{0}']),
-        ('\u{ff32}', ['\u{ff52}', '\u{0}', '\u{0}']), ('\u{ff33}', ['\u{ff53}', '\u{0}', '\u{0}']),
-        ('\u{ff34}', ['\u{ff54}', '\u{0}', '\u{0}']), ('\u{ff35}', ['\u{ff55}', '\u{0}', '\u{0}']),
-        ('\u{ff36}', ['\u{ff56}', '\u{0}', '\u{0}']), ('\u{ff37}', ['\u{ff57}', '\u{0}', '\u{0}']),
-        ('\u{ff38}', ['\u{ff58}', '\u{0}', '\u{0}']), ('\u{ff39}', ['\u{ff59}', '\u{0}', '\u{0}']),
-        ('\u{ff3a}', ['\u{ff5a}', '\u{0}', '\u{0}']),
+        ('\u{a7c6}', ['\u{1d8e}', '\u{0}', '\u{0}']), ('\u{a7c7}', ['\u{a7c8}', '\u{0}', '\u{0}']),
+        ('\u{a7c9}', ['\u{a7ca}', '\u{0}', '\u{0}']), ('\u{a7f5}', ['\u{a7f6}', '\u{0}', '\u{0}']),
+        ('\u{ff21}', ['\u{ff41}', '\u{0}', '\u{0}']), ('\u{ff22}', ['\u{ff42}', '\u{0}', '\u{0}']),
+        ('\u{ff23}', ['\u{ff43}', '\u{0}', '\u{0}']), ('\u{ff24}', ['\u{ff44}', '\u{0}', '\u{0}']),
+        ('\u{ff25}', ['\u{ff45}', '\u{0}', '\u{0}']), ('\u{ff26}', ['\u{ff46}', '\u{0}', '\u{0}']),
+        ('\u{ff27}', ['\u{ff47}', '\u{0}', '\u{0}']), ('\u{ff28}', ['\u{ff48}', '\u{0}', '\u{0}']),
+        ('\u{ff29}', ['\u{ff49}', '\u{0}', '\u{0}']), ('\u{ff2a}', ['\u{ff4a}', '\u{0}', '\u{0}']),
+        ('\u{ff2b}', ['\u{ff4b}', '\u{0}', '\u{0}']), ('\u{ff2c}', ['\u{ff4c}', '\u{0}', '\u{0}']),
+        ('\u{ff2d}', ['\u{ff4d}', '\u{0}', '\u{0}']), ('\u{ff2e}', ['\u{ff4e}', '\u{0}', '\u{0}']),
+        ('\u{ff2f}', ['\u{ff4f}', '\u{0}', '\u{0}']), ('\u{ff30}', ['\u{ff50}', '\u{0}', '\u{0}']),
+        ('\u{ff31}', ['\u{ff51}', '\u{0}', '\u{0}']), ('\u{ff32}', ['\u{ff52}', '\u{0}', '\u{0}']),
+        ('\u{ff33}', ['\u{ff53}', '\u{0}', '\u{0}']), ('\u{ff34}', ['\u{ff54}', '\u{0}', '\u{0}']),
+        ('\u{ff35}', ['\u{ff55}', '\u{0}', '\u{0}']), ('\u{ff36}', ['\u{ff56}', '\u{0}', '\u{0}']),
+        ('\u{ff37}', ['\u{ff57}', '\u{0}', '\u{0}']), ('\u{ff38}', ['\u{ff58}', '\u{0}', '\u{0}']),
+        ('\u{ff39}', ['\u{ff59}', '\u{0}', '\u{0}']), ('\u{ff3a}', ['\u{ff5a}', '\u{0}', '\u{0}']),
         ('\u{10400}', ['\u{10428}', '\u{0}', '\u{0}']),
         ('\u{10401}', ['\u{10429}', '\u{0}', '\u{0}']),
         ('\u{10402}', ['\u{1042a}', '\u{0}', '\u{0}']),
@@ -2052,51 +2057,52 @@ pub mod conversions {
         ('\u{a7b7}', ['\u{a7b6}', '\u{0}', '\u{0}']), ('\u{a7b9}', ['\u{a7b8}', '\u{0}', '\u{0}']),
         ('\u{a7bb}', ['\u{a7ba}', '\u{0}', '\u{0}']), ('\u{a7bd}', ['\u{a7bc}', '\u{0}', '\u{0}']),
         ('\u{a7bf}', ['\u{a7be}', '\u{0}', '\u{0}']), ('\u{a7c3}', ['\u{a7c2}', '\u{0}', '\u{0}']),
-        ('\u{ab53}', ['\u{a7b3}', '\u{0}', '\u{0}']), ('\u{ab70}', ['\u{13a0}', '\u{0}', '\u{0}']),
-        ('\u{ab71}', ['\u{13a1}', '\u{0}', '\u{0}']), ('\u{ab72}', ['\u{13a2}', '\u{0}', '\u{0}']),
-        ('\u{ab73}', ['\u{13a3}', '\u{0}', '\u{0}']), ('\u{ab74}', ['\u{13a4}', '\u{0}', '\u{0}']),
-        ('\u{ab75}', ['\u{13a5}', '\u{0}', '\u{0}']), ('\u{ab76}', ['\u{13a6}', '\u{0}', '\u{0}']),
-        ('\u{ab77}', ['\u{13a7}', '\u{0}', '\u{0}']), ('\u{ab78}', ['\u{13a8}', '\u{0}', '\u{0}']),
-        ('\u{ab79}', ['\u{13a9}', '\u{0}', '\u{0}']), ('\u{ab7a}', ['\u{13aa}', '\u{0}', '\u{0}']),
-        ('\u{ab7b}', ['\u{13ab}', '\u{0}', '\u{0}']), ('\u{ab7c}', ['\u{13ac}', '\u{0}', '\u{0}']),
-        ('\u{ab7d}', ['\u{13ad}', '\u{0}', '\u{0}']), ('\u{ab7e}', ['\u{13ae}', '\u{0}', '\u{0}']),
-        ('\u{ab7f}', ['\u{13af}', '\u{0}', '\u{0}']), ('\u{ab80}', ['\u{13b0}', '\u{0}', '\u{0}']),
-        ('\u{ab81}', ['\u{13b1}', '\u{0}', '\u{0}']), ('\u{ab82}', ['\u{13b2}', '\u{0}', '\u{0}']),
-        ('\u{ab83}', ['\u{13b3}', '\u{0}', '\u{0}']), ('\u{ab84}', ['\u{13b4}', '\u{0}', '\u{0}']),
-        ('\u{ab85}', ['\u{13b5}', '\u{0}', '\u{0}']), ('\u{ab86}', ['\u{13b6}', '\u{0}', '\u{0}']),
-        ('\u{ab87}', ['\u{13b7}', '\u{0}', '\u{0}']), ('\u{ab88}', ['\u{13b8}', '\u{0}', '\u{0}']),
-        ('\u{ab89}', ['\u{13b9}', '\u{0}', '\u{0}']), ('\u{ab8a}', ['\u{13ba}', '\u{0}', '\u{0}']),
-        ('\u{ab8b}', ['\u{13bb}', '\u{0}', '\u{0}']), ('\u{ab8c}', ['\u{13bc}', '\u{0}', '\u{0}']),
-        ('\u{ab8d}', ['\u{13bd}', '\u{0}', '\u{0}']), ('\u{ab8e}', ['\u{13be}', '\u{0}', '\u{0}']),
-        ('\u{ab8f}', ['\u{13bf}', '\u{0}', '\u{0}']), ('\u{ab90}', ['\u{13c0}', '\u{0}', '\u{0}']),
-        ('\u{ab91}', ['\u{13c1}', '\u{0}', '\u{0}']), ('\u{ab92}', ['\u{13c2}', '\u{0}', '\u{0}']),
-        ('\u{ab93}', ['\u{13c3}', '\u{0}', '\u{0}']), ('\u{ab94}', ['\u{13c4}', '\u{0}', '\u{0}']),
-        ('\u{ab95}', ['\u{13c5}', '\u{0}', '\u{0}']), ('\u{ab96}', ['\u{13c6}', '\u{0}', '\u{0}']),
-        ('\u{ab97}', ['\u{13c7}', '\u{0}', '\u{0}']), ('\u{ab98}', ['\u{13c8}', '\u{0}', '\u{0}']),
-        ('\u{ab99}', ['\u{13c9}', '\u{0}', '\u{0}']), ('\u{ab9a}', ['\u{13ca}', '\u{0}', '\u{0}']),
-        ('\u{ab9b}', ['\u{13cb}', '\u{0}', '\u{0}']), ('\u{ab9c}', ['\u{13cc}', '\u{0}', '\u{0}']),
-        ('\u{ab9d}', ['\u{13cd}', '\u{0}', '\u{0}']), ('\u{ab9e}', ['\u{13ce}', '\u{0}', '\u{0}']),
-        ('\u{ab9f}', ['\u{13cf}', '\u{0}', '\u{0}']), ('\u{aba0}', ['\u{13d0}', '\u{0}', '\u{0}']),
-        ('\u{aba1}', ['\u{13d1}', '\u{0}', '\u{0}']), ('\u{aba2}', ['\u{13d2}', '\u{0}', '\u{0}']),
-        ('\u{aba3}', ['\u{13d3}', '\u{0}', '\u{0}']), ('\u{aba4}', ['\u{13d4}', '\u{0}', '\u{0}']),
-        ('\u{aba5}', ['\u{13d5}', '\u{0}', '\u{0}']), ('\u{aba6}', ['\u{13d6}', '\u{0}', '\u{0}']),
-        ('\u{aba7}', ['\u{13d7}', '\u{0}', '\u{0}']), ('\u{aba8}', ['\u{13d8}', '\u{0}', '\u{0}']),
-        ('\u{aba9}', ['\u{13d9}', '\u{0}', '\u{0}']), ('\u{abaa}', ['\u{13da}', '\u{0}', '\u{0}']),
-        ('\u{abab}', ['\u{13db}', '\u{0}', '\u{0}']), ('\u{abac}', ['\u{13dc}', '\u{0}', '\u{0}']),
-        ('\u{abad}', ['\u{13dd}', '\u{0}', '\u{0}']), ('\u{abae}', ['\u{13de}', '\u{0}', '\u{0}']),
-        ('\u{abaf}', ['\u{13df}', '\u{0}', '\u{0}']), ('\u{abb0}', ['\u{13e0}', '\u{0}', '\u{0}']),
-        ('\u{abb1}', ['\u{13e1}', '\u{0}', '\u{0}']), ('\u{abb2}', ['\u{13e2}', '\u{0}', '\u{0}']),
-        ('\u{abb3}', ['\u{13e3}', '\u{0}', '\u{0}']), ('\u{abb4}', ['\u{13e4}', '\u{0}', '\u{0}']),
-        ('\u{abb5}', ['\u{13e5}', '\u{0}', '\u{0}']), ('\u{abb6}', ['\u{13e6}', '\u{0}', '\u{0}']),
-        ('\u{abb7}', ['\u{13e7}', '\u{0}', '\u{0}']), ('\u{abb8}', ['\u{13e8}', '\u{0}', '\u{0}']),
-        ('\u{abb9}', ['\u{13e9}', '\u{0}', '\u{0}']), ('\u{abba}', ['\u{13ea}', '\u{0}', '\u{0}']),
-        ('\u{abbb}', ['\u{13eb}', '\u{0}', '\u{0}']), ('\u{abbc}', ['\u{13ec}', '\u{0}', '\u{0}']),
-        ('\u{abbd}', ['\u{13ed}', '\u{0}', '\u{0}']), ('\u{abbe}', ['\u{13ee}', '\u{0}', '\u{0}']),
-        ('\u{abbf}', ['\u{13ef}', '\u{0}', '\u{0}']), ('\u{fb00}', ['F', 'F', '\u{0}']),
-        ('\u{fb01}', ['F', 'I', '\u{0}']), ('\u{fb02}', ['F', 'L', '\u{0}']),
-        ('\u{fb03}', ['F', 'F', 'I']), ('\u{fb04}', ['F', 'F', 'L']),
-        ('\u{fb05}', ['S', 'T', '\u{0}']), ('\u{fb06}', ['S', 'T', '\u{0}']),
-        ('\u{fb13}', ['\u{544}', '\u{546}', '\u{0}']),
+        ('\u{a7c8}', ['\u{a7c7}', '\u{0}', '\u{0}']), ('\u{a7ca}', ['\u{a7c9}', '\u{0}', '\u{0}']),
+        ('\u{a7f6}', ['\u{a7f5}', '\u{0}', '\u{0}']), ('\u{ab53}', ['\u{a7b3}', '\u{0}', '\u{0}']),
+        ('\u{ab70}', ['\u{13a0}', '\u{0}', '\u{0}']), ('\u{ab71}', ['\u{13a1}', '\u{0}', '\u{0}']),
+        ('\u{ab72}', ['\u{13a2}', '\u{0}', '\u{0}']), ('\u{ab73}', ['\u{13a3}', '\u{0}', '\u{0}']),
+        ('\u{ab74}', ['\u{13a4}', '\u{0}', '\u{0}']), ('\u{ab75}', ['\u{13a5}', '\u{0}', '\u{0}']),
+        ('\u{ab76}', ['\u{13a6}', '\u{0}', '\u{0}']), ('\u{ab77}', ['\u{13a7}', '\u{0}', '\u{0}']),
+        ('\u{ab78}', ['\u{13a8}', '\u{0}', '\u{0}']), ('\u{ab79}', ['\u{13a9}', '\u{0}', '\u{0}']),
+        ('\u{ab7a}', ['\u{13aa}', '\u{0}', '\u{0}']), ('\u{ab7b}', ['\u{13ab}', '\u{0}', '\u{0}']),
+        ('\u{ab7c}', ['\u{13ac}', '\u{0}', '\u{0}']), ('\u{ab7d}', ['\u{13ad}', '\u{0}', '\u{0}']),
+        ('\u{ab7e}', ['\u{13ae}', '\u{0}', '\u{0}']), ('\u{ab7f}', ['\u{13af}', '\u{0}', '\u{0}']),
+        ('\u{ab80}', ['\u{13b0}', '\u{0}', '\u{0}']), ('\u{ab81}', ['\u{13b1}', '\u{0}', '\u{0}']),
+        ('\u{ab82}', ['\u{13b2}', '\u{0}', '\u{0}']), ('\u{ab83}', ['\u{13b3}', '\u{0}', '\u{0}']),
+        ('\u{ab84}', ['\u{13b4}', '\u{0}', '\u{0}']), ('\u{ab85}', ['\u{13b5}', '\u{0}', '\u{0}']),
+        ('\u{ab86}', ['\u{13b6}', '\u{0}', '\u{0}']), ('\u{ab87}', ['\u{13b7}', '\u{0}', '\u{0}']),
+        ('\u{ab88}', ['\u{13b8}', '\u{0}', '\u{0}']), ('\u{ab89}', ['\u{13b9}', '\u{0}', '\u{0}']),
+        ('\u{ab8a}', ['\u{13ba}', '\u{0}', '\u{0}']), ('\u{ab8b}', ['\u{13bb}', '\u{0}', '\u{0}']),
+        ('\u{ab8c}', ['\u{13bc}', '\u{0}', '\u{0}']), ('\u{ab8d}', ['\u{13bd}', '\u{0}', '\u{0}']),
+        ('\u{ab8e}', ['\u{13be}', '\u{0}', '\u{0}']), ('\u{ab8f}', ['\u{13bf}', '\u{0}', '\u{0}']),
+        ('\u{ab90}', ['\u{13c0}', '\u{0}', '\u{0}']), ('\u{ab91}', ['\u{13c1}', '\u{0}', '\u{0}']),
+        ('\u{ab92}', ['\u{13c2}', '\u{0}', '\u{0}']), ('\u{ab93}', ['\u{13c3}', '\u{0}', '\u{0}']),
+        ('\u{ab94}', ['\u{13c4}', '\u{0}', '\u{0}']), ('\u{ab95}', ['\u{13c5}', '\u{0}', '\u{0}']),
+        ('\u{ab96}', ['\u{13c6}', '\u{0}', '\u{0}']), ('\u{ab97}', ['\u{13c7}', '\u{0}', '\u{0}']),
+        ('\u{ab98}', ['\u{13c8}', '\u{0}', '\u{0}']), ('\u{ab99}', ['\u{13c9}', '\u{0}', '\u{0}']),
+        ('\u{ab9a}', ['\u{13ca}', '\u{0}', '\u{0}']), ('\u{ab9b}', ['\u{13cb}', '\u{0}', '\u{0}']),
+        ('\u{ab9c}', ['\u{13cc}', '\u{0}', '\u{0}']), ('\u{ab9d}', ['\u{13cd}', '\u{0}', '\u{0}']),
+        ('\u{ab9e}', ['\u{13ce}', '\u{0}', '\u{0}']), ('\u{ab9f}', ['\u{13cf}', '\u{0}', '\u{0}']),
+        ('\u{aba0}', ['\u{13d0}', '\u{0}', '\u{0}']), ('\u{aba1}', ['\u{13d1}', '\u{0}', '\u{0}']),
+        ('\u{aba2}', ['\u{13d2}', '\u{0}', '\u{0}']), ('\u{aba3}', ['\u{13d3}', '\u{0}', '\u{0}']),
+        ('\u{aba4}', ['\u{13d4}', '\u{0}', '\u{0}']), ('\u{aba5}', ['\u{13d5}', '\u{0}', '\u{0}']),
+        ('\u{aba6}', ['\u{13d6}', '\u{0}', '\u{0}']), ('\u{aba7}', ['\u{13d7}', '\u{0}', '\u{0}']),
+        ('\u{aba8}', ['\u{13d8}', '\u{0}', '\u{0}']), ('\u{aba9}', ['\u{13d9}', '\u{0}', '\u{0}']),
+        ('\u{abaa}', ['\u{13da}', '\u{0}', '\u{0}']), ('\u{abab}', ['\u{13db}', '\u{0}', '\u{0}']),
+        ('\u{abac}', ['\u{13dc}', '\u{0}', '\u{0}']), ('\u{abad}', ['\u{13dd}', '\u{0}', '\u{0}']),
+        ('\u{abae}', ['\u{13de}', '\u{0}', '\u{0}']), ('\u{abaf}', ['\u{13df}', '\u{0}', '\u{0}']),
+        ('\u{abb0}', ['\u{13e0}', '\u{0}', '\u{0}']), ('\u{abb1}', ['\u{13e1}', '\u{0}', '\u{0}']),
+        ('\u{abb2}', ['\u{13e2}', '\u{0}', '\u{0}']), ('\u{abb3}', ['\u{13e3}', '\u{0}', '\u{0}']),
+        ('\u{abb4}', ['\u{13e4}', '\u{0}', '\u{0}']), ('\u{abb5}', ['\u{13e5}', '\u{0}', '\u{0}']),
+        ('\u{abb6}', ['\u{13e6}', '\u{0}', '\u{0}']), ('\u{abb7}', ['\u{13e7}', '\u{0}', '\u{0}']),
+        ('\u{abb8}', ['\u{13e8}', '\u{0}', '\u{0}']), ('\u{abb9}', ['\u{13e9}', '\u{0}', '\u{0}']),
+        ('\u{abba}', ['\u{13ea}', '\u{0}', '\u{0}']), ('\u{abbb}', ['\u{13eb}', '\u{0}', '\u{0}']),
+        ('\u{abbc}', ['\u{13ec}', '\u{0}', '\u{0}']), ('\u{abbd}', ['\u{13ed}', '\u{0}', '\u{0}']),
+        ('\u{abbe}', ['\u{13ee}', '\u{0}', '\u{0}']), ('\u{abbf}', ['\u{13ef}', '\u{0}', '\u{0}']),
+        ('\u{fb00}', ['F', 'F', '\u{0}']), ('\u{fb01}', ['F', 'I', '\u{0}']),
+        ('\u{fb02}', ['F', 'L', '\u{0}']), ('\u{fb03}', ['F', 'F', 'I']),
+        ('\u{fb04}', ['F', 'F', 'L']), ('\u{fb05}', ['S', 'T', '\u{0}']),
+        ('\u{fb06}', ['S', 'T', '\u{0}']), ('\u{fb13}', ['\u{544}', '\u{546}', '\u{0}']),
         ('\u{fb14}', ['\u{544}', '\u{535}', '\u{0}']),
         ('\u{fb15}', ['\u{544}', '\u{53b}', '\u{0}']),
         ('\u{fb16}', ['\u{54e}', '\u{546}', '\u{0}']),

From 349fcb9ef60f28eca485b9da5ba588fe558bae74 Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Mon, 24 Feb 2020 10:09:44 -0800
Subject: [PATCH 08/26] tidy: Use cargo_metadata for license checks.

---
 Cargo.lock                 |  13 +-
 src/bootstrap/test.rs      |   3 -
 src/tools/tidy/Cargo.toml  |   3 +-
 src/tools/tidy/src/deps.rs | 535 ++++++++++++++++---------------------
 src/tools/tidy/src/main.rs |   5 +-
 5 files changed, 241 insertions(+), 318 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index aefd40bfaa59e..1aefebab5a792 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -386,9 +386,9 @@ dependencies = [
 
 [[package]]
 name = "cargo_metadata"
-version = "0.9.0"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d2d1617e838936c0d2323a65cc151e03ae19a7678dd24f72bccf27119b90a5d"
+checksum = "46e3374c604fb39d1a2f35ed5e4a4e30e60d01fab49446e08f1b3e9a90aef202"
 dependencies = [
  "semver",
  "serde",
@@ -450,7 +450,7 @@ dependencies = [
 name = "clippy"
 version = "0.0.212"
 dependencies = [
- "cargo_metadata 0.9.0",
+ "cargo_metadata 0.9.1",
  "clippy-mini-macro-test",
  "clippy_lints",
  "compiletest_rs",
@@ -474,7 +474,7 @@ version = "0.2.0"
 name = "clippy_lints"
 version = "0.0.212"
 dependencies = [
- "cargo_metadata 0.9.0",
+ "cargo_metadata 0.9.1",
  "if_chain",
  "itertools 0.8.0",
  "lazy_static 1.4.0",
@@ -2124,7 +2124,7 @@ name = "miri"
 version = "0.1.0"
 dependencies = [
  "byteorder",
- "cargo_metadata 0.9.0",
+ "cargo_metadata 0.9.1",
  "colored",
  "compiletest_rs",
  "directories",
@@ -4818,10 +4818,9 @@ dependencies = [
 name = "tidy"
 version = "0.1.0"
 dependencies = [
+ "cargo_metadata 0.9.1",
  "lazy_static 1.4.0",
  "regex",
- "serde",
- "serde_json",
  "walkdir",
 ]
 
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index 1b9e9a889483c..e2bcb8d673af2 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -726,9 +726,6 @@ impl Step for Tidy {
         let mut cmd = builder.tool_cmd(Tool::Tidy);
         cmd.arg(builder.src.join("src"));
         cmd.arg(&builder.initial_cargo);
-        if !builder.config.vendor {
-            cmd.arg("--no-vendor");
-        }
         if builder.is_verbose() {
             cmd.arg("--verbose");
         }
diff --git a/src/tools/tidy/Cargo.toml b/src/tools/tidy/Cargo.toml
index 43cae31f33f1f..f984e5b61a5fd 100644
--- a/src/tools/tidy/Cargo.toml
+++ b/src/tools/tidy/Cargo.toml
@@ -5,8 +5,7 @@ authors = ["Alex Crichton <alex@alexcrichton.com>"]
 edition = "2018"
 
 [dependencies]
+cargo_metadata = "0.9.1"
 regex = "1"
-serde = { version = "1.0.8", features = ["derive"] }
-serde_json = "1.0.2"
 lazy_static = "1"
 walkdir = "2"
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 7a20a96130ccb..f1750ba1a2bc4 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -1,12 +1,8 @@
-//! Checks the licenses of third-party dependencies by inspecting vendors.
+//! Checks the licenses of third-party dependencies.
 
-use std::collections::{BTreeSet, HashMap, HashSet};
-use std::fs;
+use cargo_metadata::{Metadata, Package, PackageId};
+use std::collections::{BTreeSet, HashSet};
 use std::path::Path;
-use std::process::Command;
-
-use serde::Deserialize;
-use serde_json;
 
 const LICENSES: &[&str] = &[
     "MIT/Apache-2.0",
@@ -32,6 +28,7 @@ const EXCEPTIONS: &[&str] = &[
     "arrayref",           // BSD-2-Clause, mdbook via handlebars via pest
     "thread-id",          // Apache-2.0, mdbook
     "toml-query",         // MPL-2.0, mdbook
+    "toml-query_derive",  // MPL-2.0, mdbook
     "is-match",           // MPL-2.0, mdbook
     "cssparser",          // MPL-2.0, rustdoc
     "smallvec",           // MPL-2.0, rustdoc
@@ -63,223 +60,179 @@ const EXCEPTIONS: &[&str] = &[
 ];
 
 /// Which crates to check against the whitelist?
-const WHITELIST_CRATES: &[CrateVersion<'_>] =
-    &[CrateVersion("rustc", "0.0.0"), CrateVersion("rustc_codegen_llvm", "0.0.0")];
+const WHITELIST_CRATES: &[&str] = &["rustc", "rustc_codegen_llvm"];
 
 /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
-const WHITELIST: &[Crate<'_>] = &[
-    Crate("adler32"),
-    Crate("aho-corasick"),
-    Crate("annotate-snippets"),
-    Crate("ansi_term"),
-    Crate("arrayvec"),
-    Crate("atty"),
-    Crate("autocfg"),
-    Crate("backtrace"),
-    Crate("backtrace-sys"),
-    Crate("bitflags"),
-    Crate("build_const"),
-    Crate("byteorder"),
-    Crate("c2-chacha"),
-    Crate("cc"),
-    Crate("cfg-if"),
-    Crate("chalk-engine"),
-    Crate("chalk-macros"),
-    Crate("cloudabi"),
-    Crate("cmake"),
-    Crate("compiler_builtins"),
-    Crate("crc"),
-    Crate("crc32fast"),
-    Crate("crossbeam-deque"),
-    Crate("crossbeam-epoch"),
-    Crate("crossbeam-queue"),
-    Crate("crossbeam-utils"),
-    Crate("datafrog"),
-    Crate("dlmalloc"),
-    Crate("either"),
-    Crate("ena"),
-    Crate("env_logger"),
-    Crate("filetime"),
-    Crate("flate2"),
-    Crate("fortanix-sgx-abi"),
-    Crate("fuchsia-zircon"),
-    Crate("fuchsia-zircon-sys"),
-    Crate("getopts"),
-    Crate("getrandom"),
-    Crate("hashbrown"),
-    Crate("humantime"),
-    Crate("indexmap"),
-    Crate("itertools"),
-    Crate("jobserver"),
-    Crate("kernel32-sys"),
-    Crate("lazy_static"),
-    Crate("libc"),
-    Crate("libz-sys"),
-    Crate("lock_api"),
-    Crate("log"),
-    Crate("log_settings"),
-    Crate("measureme"),
-    Crate("memchr"),
-    Crate("memmap"),
-    Crate("memoffset"),
-    Crate("miniz-sys"),
-    Crate("miniz_oxide"),
-    Crate("miniz_oxide_c_api"),
-    Crate("nodrop"),
-    Crate("num_cpus"),
-    Crate("owning_ref"),
-    Crate("parking_lot"),
-    Crate("parking_lot_core"),
-    Crate("pkg-config"),
-    Crate("polonius-engine"),
-    Crate("ppv-lite86"),
-    Crate("proc-macro2"),
-    Crate("punycode"),
-    Crate("quick-error"),
-    Crate("quote"),
-    Crate("rand"),
-    Crate("rand_chacha"),
-    Crate("rand_core"),
-    Crate("rand_hc"),
-    Crate("rand_isaac"),
-    Crate("rand_pcg"),
-    Crate("rand_xorshift"),
-    Crate("redox_syscall"),
-    Crate("redox_termios"),
-    Crate("regex"),
-    Crate("regex-syntax"),
-    Crate("remove_dir_all"),
-    Crate("rustc-demangle"),
-    Crate("rustc-hash"),
-    Crate("rustc-rayon"),
-    Crate("rustc-rayon-core"),
-    Crate("rustc_version"),
-    Crate("scoped-tls"),
-    Crate("scopeguard"),
-    Crate("semver"),
-    Crate("semver-parser"),
-    Crate("serde"),
-    Crate("serde_derive"),
-    Crate("smallvec"),
-    Crate("stable_deref_trait"),
-    Crate("syn"),
-    Crate("synstructure"),
-    Crate("tempfile"),
-    Crate("termcolor"),
-    Crate("terminon"),
-    Crate("termion"),
-    Crate("termize"),
-    Crate("thread_local"),
-    Crate("ucd-util"),
-    Crate("unicode-normalization"),
-    Crate("unicode-script"),
-    Crate("unicode-security"),
-    Crate("unicode-width"),
-    Crate("unicode-xid"),
-    Crate("unreachable"),
-    Crate("utf8-ranges"),
-    Crate("vcpkg"),
-    Crate("version_check"),
-    Crate("void"),
-    Crate("wasi"),
-    Crate("winapi"),
-    Crate("winapi-build"),
-    Crate("winapi-i686-pc-windows-gnu"),
-    Crate("winapi-util"),
-    Crate("winapi-x86_64-pc-windows-gnu"),
-    Crate("wincolor"),
-    Crate("hermit-abi"),
+const WHITELIST: &[&str] = &[
+    "adler32",
+    "aho-corasick",
+    "annotate-snippets",
+    "ansi_term",
+    "arrayvec",
+    "atty",
+    "autocfg",
+    "backtrace",
+    "backtrace-sys",
+    "bitflags",
+    "build_const",
+    "byteorder",
+    "c2-chacha",
+    "cc",
+    "cfg-if",
+    "chalk-engine",
+    "chalk-macros",
+    "cloudabi",
+    "cmake",
+    "compiler_builtins",
+    "crc",
+    "crc32fast",
+    "crossbeam-deque",
+    "crossbeam-epoch",
+    "crossbeam-queue",
+    "crossbeam-utils",
+    "datafrog",
+    "dlmalloc",
+    "either",
+    "ena",
+    "env_logger",
+    "filetime",
+    "flate2",
+    "fortanix-sgx-abi",
+    "fuchsia-zircon",
+    "fuchsia-zircon-sys",
+    "getopts",
+    "getrandom",
+    "hashbrown",
+    "humantime",
+    "indexmap",
+    "itertools",
+    "jobserver",
+    "kernel32-sys",
+    "lazy_static",
+    "libc",
+    "libz-sys",
+    "lock_api",
+    "log",
+    "log_settings",
+    "measureme",
+    "memchr",
+    "memmap",
+    "memoffset",
+    "miniz-sys",
+    "miniz_oxide",
+    "miniz_oxide_c_api",
+    "nodrop",
+    "num_cpus",
+    "owning_ref",
+    "parking_lot",
+    "parking_lot_core",
+    "pkg-config",
+    "polonius-engine",
+    "ppv-lite86",
+    "proc-macro2",
+    "punycode",
+    "quick-error",
+    "quote",
+    "rand",
+    "rand_chacha",
+    "rand_core",
+    "rand_hc",
+    "rand_isaac",
+    "rand_pcg",
+    "rand_xorshift",
+    "redox_syscall",
+    "redox_termios",
+    "regex",
+    "regex-syntax",
+    "remove_dir_all",
+    "rustc-demangle",
+    "rustc-hash",
+    "rustc-rayon",
+    "rustc-rayon-core",
+    "rustc_version",
+    "scoped-tls",
+    "scopeguard",
+    "semver",
+    "semver-parser",
+    "serde",
+    "serde_derive",
+    "smallvec",
+    "stable_deref_trait",
+    "syn",
+    "synstructure",
+    "tempfile",
+    "termcolor",
+    "terminon",
+    "termion",
+    "termize",
+    "thread_local",
+    "ucd-util",
+    "unicode-normalization",
+    "unicode-script",
+    "unicode-security",
+    "unicode-width",
+    "unicode-xid",
+    "unreachable",
+    "utf8-ranges",
+    "vcpkg",
+    "version_check",
+    "void",
+    "wasi",
+    "winapi",
+    "winapi-build",
+    "winapi-i686-pc-windows-gnu",
+    "winapi-util",
+    "winapi-x86_64-pc-windows-gnu",
+    "wincolor",
+    "hermit-abi",
 ];
 
-// Some types for Serde to deserialize the output of `cargo metadata` to.
-
-#[derive(Deserialize)]
-struct Output {
-    resolve: Resolve,
-}
-
-#[derive(Deserialize)]
-struct Resolve {
-    nodes: Vec<ResolveNode>,
-}
-
-#[derive(Deserialize)]
-struct ResolveNode {
-    id: String,
-    dependencies: Vec<String>,
-}
-
-/// A unique identifier for a crate.
-#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
-struct Crate<'a>(&'a str); // (name)
-
-#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
-struct CrateVersion<'a>(&'a str, &'a str); // (name, version)
-
-impl Crate<'_> {
-    pub fn id_str(&self) -> String {
-        format!("{} ", self.0)
-    }
-}
-
-impl<'a> CrateVersion<'a> {
-    /// Returns the struct and whether or not the dependency is in-tree.
-    pub fn from_str(s: &'a str) -> (Self, bool) {
-        let mut parts = s.split(' ');
-        let name = parts.next().unwrap();
-        let version = parts.next().unwrap();
-        let path = parts.next().unwrap();
-
-        let is_path_dep = path.starts_with("(path+");
-
-        (CrateVersion(name, version), is_path_dep)
-    }
-
-    pub fn id_str(&self) -> String {
-        format!("{} {}", self.0, self.1)
-    }
-}
-
-impl<'a> From<CrateVersion<'a>> for Crate<'a> {
-    fn from(cv: CrateVersion<'a>) -> Crate<'a> {
-        Crate(cv.0)
-    }
+/// Dependency checks.
+///
+/// `path` is path to the `src` directory, `cargo` is path to the cargo executable.
+pub fn check(path: &Path, cargo: &Path, bad: &mut bool) {
+    let mut cmd = cargo_metadata::MetadataCommand::new();
+    cmd.cargo_path(cargo)
+        .manifest_path(path.parent().unwrap().join("Cargo.toml"))
+        .features(cargo_metadata::CargoOpt::AllFeatures);
+    let metadata = t!(cmd.exec());
+    check_exceptions(&metadata, bad);
+    check_whitelist(&metadata, bad);
+    check_crate_duplicate(&metadata, bad);
 }
 
-/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
+/// Check that all licenses are in the valid list in `LICENSES`.
 ///
-/// Specifically, this checks that the license is correct.
-pub fn check(path: &Path, bad: &mut bool) {
-    // Check licences.
-    let path = path.join("../vendor");
-    assert!(path.exists(), "vendor directory missing");
-    let mut saw_dir = false;
-    for dir in t!(path.read_dir()) {
-        saw_dir = true;
-        let dir = t!(dir);
-
-        // Skip our exceptions.
-        let is_exception = EXCEPTIONS.iter().any(|exception| {
-            dir.path().to_str().unwrap().contains(&format!("vendor/{}", exception))
-        });
-        if is_exception {
+/// Packages listed in `EXCEPTIONS` are allowed for tools.
+fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
+    for pkg in &metadata.packages {
+        if pkg.source.is_none() {
+            // No need to check local packages.
             continue;
         }
-
-        let toml = dir.path().join("Cargo.toml");
-        *bad = !check_license(&toml) || *bad;
+        if EXCEPTIONS.contains(&pkg.name.as_str()) {
+            continue;
+        }
+        let license = match &pkg.license {
+            Some(license) => license,
+            None => {
+                println!("dependency `{}` does not define a license expression", pkg.id,);
+                *bad = true;
+                continue;
+            }
+        };
+        if !LICENSES.contains(&license.as_str()) {
+            println!("invalid license `{}` in `{}`", license, pkg.id);
+            *bad = true;
+        }
     }
-    assert!(saw_dir, "no vendored source");
 }
 
 /// Checks the dependency of `WHITELIST_CRATES` at the given path. Changes `bad` to `true` if a
 /// check failed.
 ///
 /// Specifically, this checks that the dependencies are on the `WHITELIST`.
-pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
-    // Get dependencies from Cargo metadata.
-    let resolve = get_deps(path, cargo);
-
+fn check_whitelist(metadata: &Metadata, bad: &mut bool) {
     // Get the whitelist in a convenient form.
     let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();
 
@@ -287,122 +240,59 @@ pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
     let mut visited = BTreeSet::new();
     let mut unapproved = BTreeSet::new();
     for &krate in WHITELIST_CRATES.iter() {
-        let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false);
+        let pkg = pkg_from_name(metadata, krate);
+        let mut bad = check_crate_whitelist(&whitelist, metadata, &mut visited, pkg);
         unapproved.append(&mut bad);
     }
 
     if !unapproved.is_empty() {
         println!("Dependencies not on the whitelist:");
         for dep in unapproved {
-            println!("* {}", dep.id_str());
+            println!("* {}", dep);
         }
         *bad = true;
     }
-
-    check_crate_duplicate(&resolve, bad);
-}
-
-fn check_license(path: &Path) -> bool {
-    if !path.exists() {
-        panic!("{} does not exist", path.display());
-    }
-    let contents = t!(fs::read_to_string(&path));
-
-    let mut found_license = false;
-    for line in contents.lines() {
-        if !line.starts_with("license") {
-            continue;
-        }
-        let license = extract_license(line);
-        if !LICENSES.contains(&&*license) {
-            println!("invalid license {} in {}", license, path.display());
-            return false;
-        }
-        found_license = true;
-        break;
-    }
-    if !found_license {
-        println!("no license in {}", path.display());
-        return false;
-    }
-
-    true
-}
-
-fn extract_license(line: &str) -> String {
-    let first_quote = line.find('"');
-    let last_quote = line.rfind('"');
-    if let (Some(f), Some(l)) = (first_quote, last_quote) {
-        let license = &line[f + 1..l];
-        license.into()
-    } else {
-        "bad-license-parse".into()
-    }
-}
-
-/// Gets the dependencies of the crate at the given path using `cargo metadata`.
-fn get_deps(path: &Path, cargo: &Path) -> Resolve {
-    // Run `cargo metadata` to get the set of dependencies.
-    let output = Command::new(cargo)
-        .arg("metadata")
-        .arg("--format-version")
-        .arg("1")
-        .arg("--manifest-path")
-        .arg(path.join("../Cargo.toml"))
-        .output()
-        .expect("Unable to run `cargo metadata`")
-        .stdout;
-    let output = String::from_utf8_lossy(&output);
-    let output: Output = serde_json::from_str(&output).unwrap();
-
-    output.resolve
 }
 
 /// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
 /// the whitelist. Returns a list of illegal dependencies.
 fn check_crate_whitelist<'a>(
-    whitelist: &'a HashSet<Crate<'_>>,
-    resolve: &'a Resolve,
-    visited: &mut BTreeSet<CrateVersion<'a>>,
-    krate: CrateVersion<'a>,
-    must_be_on_whitelist: bool,
-) -> BTreeSet<Crate<'a>> {
+    whitelist: &'a HashSet<&'static str>,
+    metadata: &'a Metadata,
+    visited: &mut BTreeSet<&'a PackageId>,
+    krate: &'a Package,
+) -> BTreeSet<&'a PackageId> {
     // This will contain bad deps.
     let mut unapproved = BTreeSet::new();
 
     // Check if we have already visited this crate.
-    if visited.contains(&krate) {
+    if visited.contains(&krate.id) {
         return unapproved;
     }
 
-    visited.insert(krate);
+    visited.insert(&krate.id);
 
     // If this path is in-tree, we don't require it to be on the whitelist.
-    if must_be_on_whitelist {
+    if krate.source.is_some() {
         // If this dependency is not on `WHITELIST`, add to bad set.
-        if !whitelist.contains(&krate.into()) {
-            unapproved.insert(krate.into());
+        if !whitelist.contains(krate.name.as_str()) {
+            unapproved.insert(&krate.id);
         }
     }
 
-    // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!).
-    let to_check = resolve
-        .nodes
-        .iter()
-        .find(|n| n.id.starts_with(&krate.id_str()))
-        .expect("crate does not exist");
+    // Do a DFS in the crate graph.
+    let to_check = deps_of(metadata, &krate.id);
 
-    for dep in to_check.dependencies.iter() {
-        let (krate, is_path_dep) = CrateVersion::from_str(dep);
-
-        let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
+    for dep in to_check {
+        let mut bad = check_crate_whitelist(whitelist, metadata, visited, dep);
         unapproved.append(&mut bad);
     }
 
     unapproved
 }
 
-fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
+/// Prevents multiple versions of some expensive crates.
+fn check_crate_duplicate(metadata: &Metadata, bad: &mut bool) {
     const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[
         // These two crates take quite a long time to build, so don't allow two versions of them
         // to accidentally sneak into our dependency graph, in order to ensure we keep our CI times
@@ -410,19 +300,60 @@ fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
         "cargo",
         "rustc-ap-syntax",
     ];
-    let mut name_to_id: HashMap<_, Vec<_>> = HashMap::new();
-    for node in resolve.nodes.iter() {
-        name_to_id.entry(node.id.split_whitespace().next().unwrap()).or_default().push(&node.id);
-    }
 
-    for name in FORBIDDEN_TO_HAVE_DUPLICATES {
-        if name_to_id[name].len() <= 1 {
-            continue;
-        }
-        println!("crate `{}` is duplicated in `Cargo.lock`", name);
-        for id in name_to_id[name].iter() {
-            println!("  * {}", id);
+    for &name in FORBIDDEN_TO_HAVE_DUPLICATES {
+        let matches: Vec<_> = metadata.packages.iter().filter(|pkg| pkg.name == name).collect();
+        match matches.len() {
+            0 => {
+                println!(
+                    "crate `{}` is missing, update `check_crate_duplicate` \
+                    if it is no longer used",
+                    name
+                );
+                *bad = true;
+            }
+            1 => {}
+            _ => {
+                println!(
+                    "crate `{}` is duplicated in `Cargo.lock`, \
+                    it is too expensive to build multiple times, \
+                    so make sure only one version appears across all dependencies",
+                    name
+                );
+                for pkg in matches {
+                    println!("  * {}", pkg.id);
+                }
+                *bad = true;
+            }
         }
-        *bad = true;
     }
 }
+
+/// Returns a list of dependencies for the given package.
+fn deps_of<'a>(metadata: &'a Metadata, pkg_id: &'a PackageId) -> Vec<&'a Package> {
+    let node = metadata
+        .resolve
+        .as_ref()
+        .unwrap()
+        .nodes
+        .iter()
+        .find(|n| &n.id == pkg_id)
+        .unwrap_or_else(|| panic!("could not find `{}` in resolve", pkg_id));
+    node.deps
+        .iter()
+        .map(|dep| {
+            metadata.packages.iter().find(|pkg| pkg.id == dep.pkg).unwrap_or_else(|| {
+                panic!("could not find dep `{}` for pkg `{}` in resolve", dep.pkg, pkg_id)
+            })
+        })
+        .collect()
+}
+
+/// Finds a package with the given name.
+fn pkg_from_name<'a>(metadata: &'a Metadata, name: &'static str) -> &'a Package {
+    let mut i = metadata.packages.iter().filter(|p| p.name == name);
+    let result =
+        i.next().unwrap_or_else(|| panic!("could not find package `{}` in package list", name));
+    assert!(i.next().is_none(), "more than one package found for `{}`", name);
+    result
+}
diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index 909529d730784..e2856c690550a 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -30,10 +30,7 @@ fn main() {
     pal::check(&path, &mut bad);
     unstable_book::check(&path, collected, &mut bad);
     unit_tests::check(&path, &mut bad);
-    if !args.iter().any(|s| *s == "--no-vendor") {
-        deps::check(&path, &mut bad);
-    }
-    deps::check_whitelist(&path, &cargo, &mut bad);
+    deps::check(&path, &cargo, &mut bad);
     extdeps::check(&path, &mut bad);
     ui_tests::check(&path, &mut bad);
     error_codes_check::check(&path, &mut bad);

From 3f45da02910b91783dc5313622d2338f34696760 Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Mon, 24 Feb 2020 10:38:29 -0800
Subject: [PATCH 09/26] tidy: Check if exceptions are no longer used.

---
 src/tools/tidy/src/deps.rs | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index f1750ba1a2bc4..c3bb313202f58 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -26,18 +26,14 @@ const EXCEPTIONS: &[&str] = &[
     "openssl",            // BSD+advertising clause, cargo, mdbook
     "pest",               // MPL2, mdbook via handlebars
     "arrayref",           // BSD-2-Clause, mdbook via handlebars via pest
-    "thread-id",          // Apache-2.0, mdbook
     "toml-query",         // MPL-2.0, mdbook
     "toml-query_derive",  // MPL-2.0, mdbook
     "is-match",           // MPL-2.0, mdbook
-    "cssparser",          // MPL-2.0, rustdoc
     "smallvec",           // MPL-2.0, rustdoc
     "rdrand",             // ISC, mdbook, rustfmt
     "fuchsia-cprng",      // BSD-3-Clause, mdbook, rustfmt
     "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
     "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
-    "cssparser-macros",   // MPL-2.0, rustdoc
-    "selectors",          // MPL-2.0, rustdoc
     "clippy_lints",       // MPL-2.0, rls
     "colored",            // MPL-2.0, rustfmt
     "ordslice",           // Apache-2.0, rls
@@ -74,7 +70,6 @@ const WHITELIST: &[&str] = &[
     "backtrace",
     "backtrace-sys",
     "bitflags",
-    "build_const",
     "byteorder",
     "c2-chacha",
     "cc",
@@ -84,7 +79,6 @@ const WHITELIST: &[&str] = &[
     "cloudabi",
     "cmake",
     "compiler_builtins",
-    "crc",
     "crc32fast",
     "crossbeam-deque",
     "crossbeam-epoch",
@@ -118,12 +112,9 @@ const WHITELIST: &[&str] = &[
     "memchr",
     "memmap",
     "memoffset",
-    "miniz-sys",
     "miniz_oxide",
-    "miniz_oxide_c_api",
     "nodrop",
     "num_cpus",
-    "owning_ref",
     "parking_lot",
     "parking_lot_core",
     "pkg-config",
@@ -162,7 +153,6 @@ const WHITELIST: &[&str] = &[
     "synstructure",
     "tempfile",
     "termcolor",
-    "terminon",
     "termion",
     "termize",
     "thread_local",
@@ -172,11 +162,9 @@ const WHITELIST: &[&str] = &[
     "unicode-security",
     "unicode-width",
     "unicode-xid",
-    "unreachable",
     "utf8-ranges",
     "vcpkg",
     "version_check",
-    "void",
     "wasi",
     "winapi",
     "winapi-build",
@@ -205,6 +193,18 @@ pub fn check(path: &Path, cargo: &Path, bad: &mut bool) {
 ///
 /// Packages listed in `EXCEPTIONS` are allowed for tools.
 fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
+    // Check that the EXCEPTIONS list does not have unused entries.
+    for exception in EXCEPTIONS {
+        if !metadata.packages.iter().any(|p| p.name == *exception) {
+            println!(
+                "could not find exception package `{}`\n\
+                Remove from EXCEPTIONS list if it is no longer used.",
+                exception
+            );
+            *bad = true;
+        }
+    }
+    // Check if any package does not have a valid license.
     for pkg in &metadata.packages {
         if pkg.source.is_none() {
             // No need to check local packages.
@@ -233,6 +233,17 @@ fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
 ///
 /// Specifically, this checks that the dependencies are on the `WHITELIST`.
 fn check_whitelist(metadata: &Metadata, bad: &mut bool) {
+    // Check that the WHITELIST does not have unused entries.
+    for name in WHITELIST {
+        if !metadata.packages.iter().any(|p| p.name == *name) {
+            println!(
+                "could not find whitelisted package `{}`\n\
+                Remove from WHITELIST list if it is no longer used.",
+                name
+            );
+            *bad = true;
+        }
+    }
     // Get the whitelist in a convenient form.
     let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();
 

From be10f14329c63710ca1b47e3ebebffed59a175f4 Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Mon, 24 Feb 2020 12:54:32 -0800
Subject: [PATCH 10/26] tidy: check the licenses don't change

---
 src/tools/tidy/src/deps.rs | 96 ++++++++++++++++++++++++--------------
 1 file changed, 60 insertions(+), 36 deletions(-)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index c3bb313202f58..f6ad58a60112f 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -21,38 +21,33 @@ const LICENSES: &[&str] = &[
 /// should be considered bugs. Exceptions are only allowed in Rust
 /// tooling. It is _crucial_ that no exception crates be dependencies
 /// of the Rust runtime (std/test).
-const EXCEPTIONS: &[&str] = &[
-    "mdbook",             // MPL2, mdbook
-    "openssl",            // BSD+advertising clause, cargo, mdbook
-    "pest",               // MPL2, mdbook via handlebars
-    "arrayref",           // BSD-2-Clause, mdbook via handlebars via pest
-    "toml-query",         // MPL-2.0, mdbook
-    "toml-query_derive",  // MPL-2.0, mdbook
-    "is-match",           // MPL-2.0, mdbook
-    "smallvec",           // MPL-2.0, rustdoc
-    "rdrand",             // ISC, mdbook, rustfmt
-    "fuchsia-cprng",      // BSD-3-Clause, mdbook, rustfmt
-    "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
-    "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
-    "clippy_lints",       // MPL-2.0, rls
-    "colored",            // MPL-2.0, rustfmt
-    "ordslice",           // Apache-2.0, rls
-    "cloudabi",           // BSD-2-Clause, (rls -> crossbeam-channel 0.2 -> rand 0.5)
-    "ryu",                // Apache-2.0, rls/cargo/... (because of serde)
-    "bytesize",           // Apache-2.0, cargo
-    "im-rc",              // MPL-2.0+, cargo
-    "adler32",            // BSD-3-Clause AND Zlib, cargo dep that isn't used
-    "constant_time_eq",   // CC0-1.0, rustfmt
-    "utf8parse",          // Apache-2.0 OR MIT, cargo via strip-ansi-escapes
-    "vte",                // Apache-2.0 OR MIT, cargo via strip-ansi-escapes
-    "sized-chunks",       // MPL-2.0+, cargo via im-rc
-    "bitmaps",            // MPL-2.0+, cargo via im-rc
+const EXCEPTIONS: &[(&str, &str)] = &[
+    ("mdbook", "MPL-2.0"),                  // mdbook
+    ("openssl", "Apache-2.0"),              // cargo, mdbook
+    ("arrayref", "BSD-2-Clause"),           // mdbook via handlebars via pest
+    ("toml-query", "MPL-2.0"),              // mdbook
+    ("toml-query_derive", "MPL-2.0"),       // mdbook
+    ("is-match", "MPL-2.0"),                // mdbook
+    ("rdrand", "ISC"),                      // mdbook, rustfmt
+    ("fuchsia-cprng", "BSD-3-Clause"),      // mdbook, rustfmt
+    ("fuchsia-zircon-sys", "BSD-3-Clause"), // rustdoc, rustc, cargo
+    ("fuchsia-zircon", "BSD-3-Clause"),     // rustdoc, rustc, cargo (jobserver & tempdir)
+    ("colored", "MPL-2.0"),                 // rustfmt
+    ("ordslice", "Apache-2.0"),             // rls
+    ("cloudabi", "BSD-2-Clause"),           // (rls -> crossbeam-channel 0.2 -> rand 0.5)
+    ("ryu", "Apache-2.0 OR BSL-1.0"),       // rls/cargo/... (because of serde)
+    ("bytesize", "Apache-2.0"),             // cargo
+    ("im-rc", "MPL-2.0+"),                  // cargo
+    ("adler32", "BSD-3-Clause AND Zlib"),   // cargo dep that isn't used
+    ("constant_time_eq", "CC0-1.0"),        // rustfmt
+    ("sized-chunks", "MPL-2.0+"),           // cargo via im-rc
+    ("bitmaps", "MPL-2.0+"),                // cargo via im-rc
     // FIXME: this dependency violates the documentation comment above:
-    "fortanix-sgx-abi",   // MPL-2.0+, libstd but only for `sgx` target
-    "dunce",              // CC0-1.0 mdbook-linkcheck
-    "codespan-reporting", // Apache-2.0 mdbook-linkcheck
-    "codespan",           // Apache-2.0 mdbook-linkcheck
-    "crossbeam-channel",  // MIT/Apache-2.0 AND BSD-2-Clause, cargo
+    ("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target
+    ("dunce", "CC0-1.0"),            // mdbook-linkcheck
+    ("codespan-reporting", "Apache-2.0"), // mdbook-linkcheck
+    ("codespan", "Apache-2.0"),      // mdbook-linkcheck
+    ("crossbeam-channel", "MIT/Apache-2.0 AND BSD-2-Clause"), // cargo
 ];
 
 /// Which crates to check against the whitelist?
@@ -193,24 +188,53 @@ pub fn check(path: &Path, cargo: &Path, bad: &mut bool) {
 ///
 /// Packages listed in `EXCEPTIONS` are allowed for tools.
 fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
-    // Check that the EXCEPTIONS list does not have unused entries.
-    for exception in EXCEPTIONS {
-        if !metadata.packages.iter().any(|p| p.name == *exception) {
+    // Validate the EXCEPTIONS list hasn't changed.
+    for (name, license) in EXCEPTIONS {
+        // Check that the package actually exists.
+        if !metadata.packages.iter().any(|p| p.name == *name) {
             println!(
                 "could not find exception package `{}`\n\
                 Remove from EXCEPTIONS list if it is no longer used.",
-                exception
+                name
             );
             *bad = true;
         }
+        // Check that the license hasn't changed.
+        for pkg in metadata.packages.iter().filter(|p| p.name == *name) {
+            if pkg.name == "fuchsia-cprng" {
+                // This package doesn't declare a license expression. Manual
+                // inspection of the license file is necessary, which appears
+                // to be BSD-3-Clause.
+                assert!(pkg.license.is_none());
+                continue;
+            }
+            match &pkg.license {
+                None => {
+                    println!(
+                        "dependency exception `{}` does not declare a license expression",
+                        pkg.id
+                    );
+                    *bad = true;
+                }
+                Some(pkg_license) => {
+                    if pkg_license.as_str() != *license {
+                        println!("dependency exception `{}` license has changed", name);
+                        println!("    previously `{}` now `{}`", license, pkg_license);
+                        println!("    update EXCEPTIONS for the new license");
+                        *bad = true;
+                    }
+                }
+            }
+        }
     }
+    let exception_names: Vec<_> = EXCEPTIONS.iter().map(|(name, _license)| *name).collect();
     // Check if any package does not have a valid license.
     for pkg in &metadata.packages {
         if pkg.source.is_none() {
             // No need to check local packages.
             continue;
         }
-        if EXCEPTIONS.contains(&pkg.name.as_str()) {
+        if exception_names.contains(&pkg.name.as_str()) {
             continue;
         }
         let license = match &pkg.license {

From bc738f239348dd62f18d40fdc001757ec00cfea5 Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Mon, 24 Feb 2020 13:47:49 -0800
Subject: [PATCH 11/26] tidy: Verify the runtime crates don't have license
 exceptions.

---
 src/tools/tidy/src/deps.rs | 63 ++++++++++++++++++++++++++++++++++----
 1 file changed, 57 insertions(+), 6 deletions(-)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index f6ad58a60112f..aa4d86e2a8459 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -1,6 +1,6 @@
 //! Checks the licenses of third-party dependencies.
 
-use cargo_metadata::{Metadata, Package, PackageId};
+use cargo_metadata::{Metadata, Package, PackageId, Resolve};
 use std::collections::{BTreeSet, HashSet};
 use std::path::Path;
 
@@ -50,6 +50,10 @@ const EXCEPTIONS: &[(&str, &str)] = &[
     ("crossbeam-channel", "MIT/Apache-2.0 AND BSD-2-Clause"), // cargo
 ];
 
+/// These are the root crates that are part of the runtime. The licenses for
+/// these and all their dependencies *must not* be in the exception list.
+const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "panic_abort", "panic_unwind"];
+
 /// Which crates to check against the whitelist?
 const WHITELIST_CRATES: &[&str] = &["rustc", "rustc_codegen_llvm"];
 
@@ -227,14 +231,17 @@ fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
             }
         }
     }
+
     let exception_names: Vec<_> = EXCEPTIONS.iter().map(|(name, _license)| *name).collect();
+    let runtime_ids = compute_runtime_crates(metadata);
+
     // Check if any package does not have a valid license.
     for pkg in &metadata.packages {
         if pkg.source.is_none() {
             // No need to check local packages.
             continue;
         }
-        if exception_names.contains(&pkg.name.as_str()) {
+        if !runtime_ids.contains(&pkg.id) && exception_names.contains(&pkg.name.as_str()) {
             continue;
         }
         let license = match &pkg.license {
@@ -246,6 +253,13 @@ fn check_exceptions(metadata: &Metadata, bad: &mut bool) {
             }
         };
         if !LICENSES.contains(&license.as_str()) {
+            if pkg.name == "fortanix-sgx-abi" {
+                // This is a specific exception because SGX is considered
+                // "third party". See
+                // https://github.com/rust-lang/rust/issues/62620 for more. In
+                // general, these should never be added.
+                continue;
+            }
             println!("invalid license `{}` in `{}`", license, pkg.id);
             *bad = true;
         }
@@ -366,10 +380,8 @@ fn check_crate_duplicate(metadata: &Metadata, bad: &mut bool) {
 
 /// Returns a list of dependencies for the given package.
 fn deps_of<'a>(metadata: &'a Metadata, pkg_id: &'a PackageId) -> Vec<&'a Package> {
-    let node = metadata
-        .resolve
-        .as_ref()
-        .unwrap()
+    let resolve = metadata.resolve.as_ref().unwrap();
+    let node = resolve
         .nodes
         .iter()
         .find(|n| &n.id == pkg_id)
@@ -392,3 +404,42 @@ fn pkg_from_name<'a>(metadata: &'a Metadata, name: &'static str) -> &'a Package
     assert!(i.next().is_none(), "more than one package found for `{}`", name);
     result
 }
+
+/// Finds all the packages that are in the rust runtime.
+fn compute_runtime_crates<'a>(metadata: &'a Metadata) -> HashSet<&'a PackageId> {
+    let resolve = metadata.resolve.as_ref().unwrap();
+    let mut result = HashSet::new();
+    for name in RUNTIME_CRATES {
+        let id = &pkg_from_name(metadata, name).id;
+        normal_deps_of_r(resolve, id, &mut result);
+    }
+    result
+}
+
+/// Recursively find all normal dependencies.
+fn normal_deps_of_r<'a>(
+    resolve: &'a Resolve,
+    pkg_id: &'a PackageId,
+    result: &mut HashSet<&'a PackageId>,
+) {
+    if !result.insert(pkg_id) {
+        return;
+    }
+    let node = resolve
+        .nodes
+        .iter()
+        .find(|n| &n.id == pkg_id)
+        .unwrap_or_else(|| panic!("could not find `{}` in resolve", pkg_id));
+    // Don't care about dev-dependencies.
+    // Build dependencies *shouldn't* matter unless they do some kind of
+    // codegen. For now we'll assume they don't.
+    let deps = node.deps.iter().filter(|node_dep| {
+        node_dep
+            .dep_kinds
+            .iter()
+            .any(|kind_info| kind_info.kind == cargo_metadata::DependencyKind::Normal)
+    });
+    for dep in deps {
+        normal_deps_of_r(resolve, &dep.pkg, result);
+    }
+}

From 3c48c89e7b9cea0b70a138571dec708fd09ba72f Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Mon, 24 Feb 2020 13:52:08 -0800
Subject: [PATCH 12/26] tidy: Add some clarifying comments in license checks.

---
 src/tools/tidy/src/deps.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index aa4d86e2a8459..886bdf6fbbf89 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -4,6 +4,8 @@ use cargo_metadata::{Metadata, Package, PackageId, Resolve};
 use std::collections::{BTreeSet, HashSet};
 use std::path::Path;
 
+/// These are licenses that are allowed for all crates, including the runtime,
+/// rustc, tools, etc.
 const LICENSES: &[&str] = &[
     "MIT/Apache-2.0",
     "MIT / Apache-2.0",
@@ -58,6 +60,9 @@ const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "panic_abort", "panic_
 const WHITELIST_CRATES: &[&str] = &["rustc", "rustc_codegen_llvm"];
 
 /// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
+///
+/// This list is here to provide a speed-bump to adding a new dependency to
+/// rustc. Please check with the compiler team before adding an entry.
 const WHITELIST: &[&str] = &[
     "adler32",
     "aho-corasick",

From 2e46faa3cacc111d77294a7c1a1d8337cd018e80 Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Tue, 25 Feb 2020 07:28:46 -0800
Subject: [PATCH 13/26] tidy: Add `test` to RUNTIME_CRATES.

---
 src/tools/tidy/src/deps.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 886bdf6fbbf89..1aee4d7cda43c 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -54,7 +54,7 @@ const EXCEPTIONS: &[(&str, &str)] = &[
 
 /// These are the root crates that are part of the runtime. The licenses for
 /// these and all their dependencies *must not* be in the exception list.
-const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "panic_abort", "panic_unwind"];
+const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "test", "panic_abort", "panic_unwind"];
 
 /// Which crates to check against the whitelist?
 const WHITELIST_CRATES: &[&str] = &["rustc", "rustc_codegen_llvm"];

From 9a53cf37777f9fe88d58d2a53c10f3d880ccfcc0 Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Tue, 25 Feb 2020 07:30:33 -0800
Subject: [PATCH 14/26] tidy: Sort WHITELIST.

---
 src/tools/tidy/src/deps.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 1aee4d7cda43c..115787a7eedad 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -101,6 +101,7 @@ const WHITELIST: &[&str] = &[
     "getopts",
     "getrandom",
     "hashbrown",
+    "hermit-abi",
     "humantime",
     "indexmap",
     "itertools",
@@ -176,7 +177,6 @@ const WHITELIST: &[&str] = &[
     "winapi-util",
     "winapi-x86_64-pc-windows-gnu",
     "wincolor",
-    "hermit-abi",
 ];
 
 /// Dependency checks.

From ed0158d7aabbd768aba12c7e3892b934b9dfa12c Mon Sep 17 00:00:00 2001
From: Eric Huss <eric@huss.org>
Date: Thu, 12 Mar 2020 19:23:30 -0700
Subject: [PATCH 15/26] tidy: Remove chalk-engine/chalk-macros.

Removed in #69247 while this PR was waiting to merge.
---
 src/tools/tidy/src/deps.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 115787a7eedad..1ffc415fb2497 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -78,8 +78,6 @@ const WHITELIST: &[&str] = &[
     "c2-chacha",
     "cc",
     "cfg-if",
-    "chalk-engine",
-    "chalk-macros",
     "cloudabi",
     "cmake",
     "compiler_builtins",

From 041d97f4fdd625ca193e29f389f754904441aa04 Mon Sep 17 00:00:00 2001
From: iximeow <me@iximeow.net>
Date: Thu, 12 Mar 2020 19:53:13 -0700
Subject: [PATCH 16/26] unix: Set a guard page at the end of signal stacks

This mitigates possible issues when signal stacks overflow, which could
manifest as segfaults or in unlucky circumstances possible clobbering of
other memory values as stack overflows tend to enable.
---
 src/libstd/sys/unix/stack_overflow.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/libstd/sys/unix/stack_overflow.rs b/src/libstd/sys/unix/stack_overflow.rs
index 9e8be55075578..db42e11683ad8 100644
--- a/src/libstd/sys/unix/stack_overflow.rs
+++ b/src/libstd/sys/unix/stack_overflow.rs
@@ -45,7 +45,7 @@ mod imp {
     use libc::{mmap, munmap};
     use libc::{sigaction, sighandler_t, SA_ONSTACK, SA_SIGINFO, SIGBUS, SIG_DFL};
     use libc::{sigaltstack, SIGSTKSZ, SS_DISABLE};
-    use libc::{MAP_ANON, MAP_PRIVATE, PROT_READ, PROT_WRITE, SIGSEGV};
+    use libc::{MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE, SIGSEGV};
 
     use crate::sys_common::thread_info;
 
@@ -137,11 +137,16 @@ mod imp {
     }
 
     unsafe fn get_stackp() -> *mut libc::c_void {
+        let page_size = crate::sys::unix::os::page_size();
         let stackp =
-            mmap(ptr::null_mut(), SIGSTKSZ, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+            mmap(ptr::null_mut(), SIGSTKSZ + page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
         if stackp == MAP_FAILED {
             panic!("failed to allocate an alternative stack");
         }
+        let guard_result = libc::mprotect(stackp, page_size, PROT_NONE);
+        if guard_result != 0 {
+            panic!("failed to set up alternative stack guard page");
+        }
         stackp
     }
 

From 0ca2ed364662351da9e8a1f4ba54e027e1f420df Mon Sep 17 00:00:00 2001
From: iximeow <me@iximeow.net>
Date: Thu, 12 Mar 2020 21:17:10 -0700
Subject: [PATCH 17/26] return a pointer to the end of the valid part of the
 sigstack, no further

also unmap the whole thing when cleaning up, rather than leaving a spare
page floating around.
---
 src/libstd/sys/unix/stack_overflow.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/libstd/sys/unix/stack_overflow.rs b/src/libstd/sys/unix/stack_overflow.rs
index db42e11683ad8..c4d771d75d967 100644
--- a/src/libstd/sys/unix/stack_overflow.rs
+++ b/src/libstd/sys/unix/stack_overflow.rs
@@ -47,6 +47,7 @@ mod imp {
     use libc::{sigaltstack, SIGSTKSZ, SS_DISABLE};
     use libc::{MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE, SIGSEGV};
 
+    use crate::sys::unix::os::page_size;
     use crate::sys_common::thread_info;
 
     #[cfg(any(target_os = "linux", target_os = "android"))]
@@ -137,17 +138,16 @@ mod imp {
     }
 
     unsafe fn get_stackp() -> *mut libc::c_void {
-        let page_size = crate::sys::unix::os::page_size();
         let stackp =
-            mmap(ptr::null_mut(), SIGSTKSZ + page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+            mmap(ptr::null_mut(), SIGSTKSZ + page_size(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
         if stackp == MAP_FAILED {
             panic!("failed to allocate an alternative stack");
         }
-        let guard_result = libc::mprotect(stackp, page_size, PROT_NONE);
+        let guard_result = libc::mprotect(stackp, page_size(), PROT_NONE);
         if guard_result != 0 {
             panic!("failed to set up alternative stack guard page");
         }
-        stackp
+        stackp.add(page_size())
     }
 
     #[cfg(any(
@@ -195,7 +195,9 @@ mod imp {
                 ss_size: SIGSTKSZ,
             };
             sigaltstack(&stack, ptr::null_mut());
-            munmap(handler._data, SIGSTKSZ);
+            // We know from `get_stackp` that the alternate stack we installed is part of a mapping
+            // that started one page earlier, so walk back a page and unmap from there.
+            munmap(handler._data.sub(page_size()), SIGSTKSZ + page_size());
         }
     }
 }

From 28eeea630faf1e7514da96c5eedd67e330fe8571 Mon Sep 17 00:00:00 2001
From: iximeow <me@iximeow.net>
Date: Thu, 12 Mar 2020 22:21:36 -0700
Subject: [PATCH 18/26] fix formatting

---
 src/libstd/sys/unix/stack_overflow.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/libstd/sys/unix/stack_overflow.rs b/src/libstd/sys/unix/stack_overflow.rs
index c4d771d75d967..2626ca37cf8b8 100644
--- a/src/libstd/sys/unix/stack_overflow.rs
+++ b/src/libstd/sys/unix/stack_overflow.rs
@@ -138,8 +138,14 @@ mod imp {
     }
 
     unsafe fn get_stackp() -> *mut libc::c_void {
-        let stackp =
-            mmap(ptr::null_mut(), SIGSTKSZ + page_size(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
+        let stackp = mmap(
+            ptr::null_mut(),
+            SIGSTKSZ + page_size(),
+            PROT_READ | PROT_WRITE,
+            MAP_PRIVATE | MAP_ANON,
+            -1,
+            0,
+        );
         if stackp == MAP_FAILED {
             panic!("failed to allocate an alternative stack");
         }

From 38fa3783ce635814b7e3814ab26d52b653fc0ab0 Mon Sep 17 00:00:00 2001
From: Jonas Schievink <jonasschievink@gmail.com>
Date: Sat, 14 Mar 2020 15:46:57 +0100
Subject: [PATCH 19/26] Swap inserts to keep the original ordering

---
 src/librustc_mir/transform/generator.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs
index b386582947f1a..0001bb0536365 100644
--- a/src/librustc_mir/transform/generator.rs
+++ b/src/librustc_mir/transform/generator.rs
@@ -1099,17 +1099,17 @@ fn create_generator_resume_function<'tcx>(
     // Panic when resumed on the returned or poisoned state
     let generator_kind = body.generator_kind.unwrap();
 
-    if can_return {
+    if can_unwind {
         cases.insert(
             1,
-            (RETURNED, insert_panic_block(tcx, body, ResumedAfterReturn(generator_kind))),
+            (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))),
         );
     }
 
-    if can_unwind {
+    if can_return {
         cases.insert(
             1,
-            (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(generator_kind))),
+            (RETURNED, insert_panic_block(tcx, body, ResumedAfterReturn(generator_kind))),
         );
     }
 

From d4ea327f24ff3ba0f4caa9782693dd438429a565 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Sat, 14 Mar 2020 18:17:08 +0100
Subject: [PATCH 20/26] Make ignore and compile fail signs more visible

---
 src/librustdoc/html/static/rustdoc.css      | 13 ++++++++++---
 src/librustdoc/html/static/themes/dark.css  |  9 +++++----
 src/librustdoc/html/static/themes/light.css | 12 ++++++------
 3 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/librustdoc/html/static/rustdoc.css b/src/librustdoc/html/static/rustdoc.css
index f05db6c218524..8887bca3c5962 100644
--- a/src/librustdoc/html/static/rustdoc.css
+++ b/src/librustdoc/html/static/rustdoc.css
@@ -1032,7 +1032,7 @@ h3 > .collapse-toggle, h4 > .collapse-toggle {
 
 .information {
 	position: absolute;
-	left: -20px;
+	left: -25px;
 	margin-top: 7px;
 	z-index: 1;
 }
@@ -1047,12 +1047,13 @@ h3 > .collapse-toggle, h4 > .collapse-toggle {
 	width: 120px;
 	display: none;
 	text-align: center;
-	padding: 5px 3px;
+	padding: 5px 3px 3px 3px;
 	border-radius: 6px;
 	margin-left: 5px;
 	top: -5px;
 	left: 105%;
 	z-index: 10;
+	font-size: 16px;
 }
 
 .tooltip:hover .tooltiptext {
@@ -1063,14 +1064,20 @@ h3 > .collapse-toggle, h4 > .collapse-toggle {
 	content: " ";
 	position: absolute;
 	top: 50%;
-	left: 11px;
+	left: 16px;
 	margin-top: -5px;
 	border-width: 5px;
 	border-style: solid;
 }
 
+.tooltip.compile_fail, .tooltip.ignore {
+	font-weight: bold;
+	font-size: 20px;
+}
+
 .tooltip .tooltiptext {
 	border: 1px solid;
+	font-weight: normal;
 }
 
 pre.rust {
diff --git a/src/librustdoc/html/static/themes/dark.css b/src/librustdoc/html/static/themes/dark.css
index e7041d54c6bfd..ff32a0fa09e4d 100644
--- a/src/librustdoc/html/static/themes/dark.css
+++ b/src/librustdoc/html/static/themes/dark.css
@@ -254,7 +254,7 @@ a.test-arrow:hover{
 }
 
 pre.compile_fail {
-	border-left: 2px solid rgba(255,0,0,.6);
+	border-left: 2px solid rgba(255,0,0,.8);
 }
 
 pre.compile_fail:hover, .information:hover + pre.compile_fail {
@@ -270,7 +270,7 @@ pre.ignore:hover, .information:hover + pre.ignore {
 }
 
 .tooltip.compile_fail {
-	color: rgba(255,0,0,.6);
+	color: rgba(255,0,0,.8);
 }
 
 .information > .compile_fail:hover {
@@ -282,7 +282,7 @@ pre.ignore:hover, .information:hover + pre.ignore {
 }
 
 .information > .ignore:hover {
-	color: rgba(255,142,0,1);
+	color: #ff9200;
 }
 
 .search-failed a {
@@ -290,8 +290,9 @@ pre.ignore:hover, .information:hover + pre.ignore {
 }
 
 .tooltip .tooltiptext {
-	background-color: black;
+	background-color: #000;
 	color: #fff;
+	border-color: #000;
 }
 
 .tooltip .tooltiptext::after {
diff --git a/src/librustdoc/html/static/themes/light.css b/src/librustdoc/html/static/themes/light.css
index a1efef6701fd1..2b2819f7126b4 100644
--- a/src/librustdoc/html/static/themes/light.css
+++ b/src/librustdoc/html/static/themes/light.css
@@ -248,7 +248,7 @@ a.test-arrow:hover{
 }
 
 pre.compile_fail {
-	border-left: 2px solid rgba(255,0,0,.4);
+	border-left: 2px solid rgba(255,0,0,.5);
 }
 
 pre.compile_fail:hover, .information:hover + pre.compile_fail {
@@ -256,7 +256,7 @@ pre.compile_fail:hover, .information:hover + pre.compile_fail {
 }
 
 pre.ignore {
-	border-left: 2px solid rgba(255,142,0,.4);
+	border-left: 2px solid rgba(255,142,0,.6);
 }
 
 pre.ignore:hover, .information:hover + pre.ignore {
@@ -264,7 +264,7 @@ pre.ignore:hover, .information:hover + pre.ignore {
 }
 
 .tooltip.compile_fail {
-	color: rgba(255,0,0,.3);
+	color: rgba(255,0,0,.5);
 }
 
 .information > .compile_fail:hover {
@@ -272,11 +272,11 @@ pre.ignore:hover, .information:hover + pre.ignore {
 }
 
 .tooltip.ignore {
-	color: rgba(255,142,0,.3);
+	color: rgba(255,142,0,.6);
 }
 
 .information > .ignore:hover {
-	color: rgba(255,142,0,1);
+	color: #ff9200;
 }
 
 .search-failed a {
@@ -284,7 +284,7 @@ pre.ignore:hover, .information:hover + pre.ignore {
 }
 
 .tooltip .tooltiptext {
-	background-color: black;
+	background-color: #000;
 	color: #fff;
 }
 

From 8216a32d32e705ecbc7c2329f7637fbd259a190d Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Sat, 14 Mar 2020 18:17:20 +0100
Subject: [PATCH 21/26] remove compilation warning

---
 src/librustdoc/clean/inline.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs
index 7cb870ae702fb..f600b3308e882 100644
--- a/src/librustdoc/clean/inline.rs
+++ b/src/librustdoc/clean/inline.rs
@@ -12,7 +12,6 @@ use rustc_hir::Mutability;
 use rustc_metadata::creader::LoadedMacro;
 use rustc_mir::const_eval::is_min_const_fn;
 use rustc_span::hygiene::MacroKind;
-use rustc_span::symbol::sym;
 use rustc_span::Span;
 
 use crate::clean::{self, GetDefId, ToSource, TypeKind};

From 51ea260ee7ee896701ef7296d7f63e7ef455b383 Mon Sep 17 00:00:00 2001
From: Eduard-Mihai Burtescu <edy.burt@gmail.com>
Date: Sun, 9 Feb 2020 02:39:14 +0200
Subject: [PATCH 22/26] rustc: don't resolve Instances which would produce
 malformed shims.

---
 src/librustc/ty/instance.rs | 15 ++++++++
 src/librustc_mir/shim.rs    | 33 ++++++++++-------
 src/librustc_ty/instance.rs | 70 ++++++++++++++++++++++++++-----------
 3 files changed, 86 insertions(+), 32 deletions(-)

diff --git a/src/librustc/ty/instance.rs b/src/librustc/ty/instance.rs
index 445df76cd32be..07acb7abfdbb8 100644
--- a/src/librustc/ty/instance.rs
+++ b/src/librustc/ty/instance.rs
@@ -39,6 +39,11 @@ pub enum InstanceDef<'tcx> {
 
     /// `<fn() as FnTrait>::call_*`
     /// `DefId` is `FnTrait::call_*`.
+    ///
+    /// NB: the (`fn` pointer) type must be monomorphic for MIR shims to work.
+    // FIXME(eddyb) support generating shims for a "shallow type",
+    // e.g. `fn(_, _) -> _` instead of requiring a fully monomorphic
+    // `fn(Foo, Bar) -> Baz` etc.
     FnPtrShim(DefId, Ty<'tcx>),
 
     /// `<dyn Trait as Trait>::fn`, "direct calls" of which are implicitly
@@ -57,9 +62,19 @@ pub enum InstanceDef<'tcx> {
     /// The `DefId` is for `core::ptr::drop_in_place`.
     /// The `Option<Ty<'tcx>>` is either `Some(T)`, or `None` for empty drop
     /// glue.
+    ///
+    /// NB: the type must be monomorphic for MIR shims to work.
+    // FIXME(eddyb) support generating shims for a "shallow type",
+    // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
+    // `Foo<Bar>` or `[String]` etc.
     DropGlue(DefId, Option<Ty<'tcx>>),
 
     ///`<T as Clone>::clone` shim.
+    ///
+    /// NB: the type must be monomorphic for MIR shims to work.
+    // FIXME(eddyb) support generating shims for a "shallow type",
+    // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
+    // `Foo<Bar>` or `[String]` etc.
     CloneShim(DefId, Ty<'tcx>),
 }
 
diff --git a/src/librustc_mir/shim.rs b/src/librustc_mir/shim.rs
index 1f7db2861a2eb..c1d969a4b5163 100644
--- a/src/librustc_mir/shim.rs
+++ b/src/librustc_mir/shim.rs
@@ -2,13 +2,13 @@ use rustc::mir::*;
 use rustc::ty::layout::VariantIdx;
 use rustc::ty::query::Providers;
 use rustc::ty::subst::{InternalSubsts, Subst};
-use rustc::ty::{self, Ty, TyCtxt};
+use rustc::ty::{self, Ty, TyCtxt, TypeFoldable};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 
 use rustc_index::vec::{Idx, IndexVec};
 
-use rustc_span::{sym, Span};
+use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
 
 use std::fmt;
@@ -39,6 +39,11 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx
             None,
         ),
         ty::InstanceDef::FnPtrShim(def_id, ty) => {
+            // FIXME(eddyb) support generating shims for a "shallow type",
+            // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
+            // `Foo<Bar>` or `[String]` etc.
+            assert!(!ty.needs_subst());
+
             let trait_ = tcx.trait_of_item(def_id).unwrap();
             let adjustment = match tcx.fn_trait_kind_from_lang_item(trait_) {
                 Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
@@ -81,17 +86,21 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx
                 None,
             )
         }
-        ty::InstanceDef::DropGlue(def_id, ty) => build_drop_shim(tcx, def_id, ty),
+        ty::InstanceDef::DropGlue(def_id, ty) => {
+            // FIXME(eddyb) support generating shims for a "shallow type",
+            // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
+            // `Foo<Bar>` or `[String]` etc.
+            assert!(!ty.needs_subst());
+
+            build_drop_shim(tcx, def_id, ty)
+        }
         ty::InstanceDef::CloneShim(def_id, ty) => {
-            let name = tcx.item_name(def_id);
-            if name == sym::clone {
-                build_clone_shim(tcx, def_id, ty)
-            } else if name == sym::clone_from {
-                debug!("make_shim({:?}: using default trait implementation", instance);
-                return tcx.optimized_mir(def_id);
-            } else {
-                bug!("builtin clone shim {:?} not supported", instance)
-            }
+            // FIXME(eddyb) support generating shims for a "shallow type",
+            // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
+            // `Foo<Bar>` or `[String]` etc.
+            assert!(!ty.needs_subst());
+
+            build_clone_shim(tcx, def_id, ty)
         }
         ty::InstanceDef::Virtual(..) => {
             bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance)
diff --git a/src/librustc_ty/instance.rs b/src/librustc_ty/instance.rs
index 10cc2c0e3033c..a5abe7b6413cc 100644
--- a/src/librustc_ty/instance.rs
+++ b/src/librustc_ty/instance.rs
@@ -1,6 +1,7 @@
 use rustc::ty::subst::SubstsRef;
 use rustc::ty::{self, Instance, TyCtxt, TypeFoldable};
 use rustc_hir::def_id::DefId;
+use rustc_span::sym;
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits;
 
@@ -31,21 +32,26 @@ pub fn resolve_instance<'tcx>(
                 debug!(" => intrinsic");
                 ty::InstanceDef::Intrinsic(def_id)
             }
-            _ => {
-                if Some(def_id) == tcx.lang_items().drop_in_place_fn() {
-                    let ty = substs.type_at(0);
-                    if ty.needs_drop(tcx, param_env.with_reveal_all()) {
-                        debug!(" => nontrivial drop glue");
-                        ty::InstanceDef::DropGlue(def_id, Some(ty))
-                    } else {
-                        debug!(" => trivial drop glue");
-                        ty::InstanceDef::DropGlue(def_id, None)
+            ty::FnDef(def_id, substs) if Some(def_id) == tcx.lang_items().drop_in_place_fn() => {
+                let ty = substs.type_at(0);
+
+                if ty.needs_drop(tcx, param_env) {
+                    // `DropGlue` requires a monomorphic aka concrete type.
+                    if ty.needs_subst() {
+                        return None;
                     }
+
+                    debug!(" => nontrivial drop glue");
+                    ty::InstanceDef::DropGlue(def_id, Some(ty))
                 } else {
-                    debug!(" => free item");
-                    ty::InstanceDef::Item(def_id)
+                    debug!(" => trivial drop glue");
+                    ty::InstanceDef::DropGlue(def_id, None)
                 }
             }
+            _ => {
+                debug!(" => free item");
+                ty::InstanceDef::Item(def_id)
+            }
         };
         Some(Instance { def, substs })
     };
@@ -113,20 +119,44 @@ fn resolve_associated_item<'tcx>(
                 trait_closure_kind,
             ))
         }
-        traits::VtableFnPointer(ref data) => Some(Instance {
-            def: ty::InstanceDef::FnPtrShim(trait_item.def_id, data.fn_ty),
-            substs: rcvr_substs,
-        }),
+        traits::VtableFnPointer(ref data) => {
+            // `FnPtrShim` requires a monomorphic aka concrete type.
+            if data.fn_ty.needs_subst() {
+                return None;
+            }
+
+            Some(Instance {
+                def: ty::InstanceDef::FnPtrShim(trait_item.def_id, data.fn_ty),
+                substs: rcvr_substs,
+            })
+        }
         traits::VtableObject(ref data) => {
             let index = traits::get_vtable_index_of_object_method(tcx, data, def_id);
             Some(Instance { def: ty::InstanceDef::Virtual(def_id, index), substs: rcvr_substs })
         }
         traits::VtableBuiltin(..) => {
-            if tcx.lang_items().clone_trait().is_some() {
-                Some(Instance {
-                    def: ty::InstanceDef::CloneShim(def_id, trait_ref.self_ty()),
-                    substs: rcvr_substs,
-                })
+            if Some(trait_ref.def_id) == tcx.lang_items().clone_trait() {
+                // FIXME(eddyb) use lang items for methods instead of names.
+                let name = tcx.item_name(def_id);
+                if name == sym::clone {
+                    let self_ty = trait_ref.self_ty();
+
+                    // `CloneShim` requires a monomorphic aka concrete type.
+                    if self_ty.needs_subst() {
+                        return None;
+                    }
+
+                    Some(Instance {
+                        def: ty::InstanceDef::CloneShim(def_id, self_ty),
+                        substs: rcvr_substs,
+                    })
+                } else {
+                    assert_eq!(name, sym::clone_from);
+
+                    // Use the default `fn clone_from` from `trait Clone`.
+                    let substs = tcx.erase_regions(&rcvr_substs);
+                    Some(ty::Instance::new(def_id, substs))
+                }
             } else {
                 None
             }

From bc8ff3fe8e8cebc9e4855aaf44c19c887f8f4be4 Mon Sep 17 00:00:00 2001
From: Ana-Maria Mihalache <mihalacheana.maria@yahoo.com>
Date: Wed, 11 Mar 2020 15:20:31 +0000
Subject: [PATCH 23/26] rustc: tweak comments on InstanceDef.

---
 src/librustc/ty/instance.rs | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/librustc/ty/instance.rs b/src/librustc/ty/instance.rs
index 07acb7abfdbb8..c9ad8707a7454 100644
--- a/src/librustc/ty/instance.rs
+++ b/src/librustc/ty/instance.rs
@@ -40,10 +40,9 @@ pub enum InstanceDef<'tcx> {
     /// `<fn() as FnTrait>::call_*`
     /// `DefId` is `FnTrait::call_*`.
     ///
-    /// NB: the (`fn` pointer) type must be monomorphic for MIR shims to work.
-    // FIXME(eddyb) support generating shims for a "shallow type",
-    // e.g. `fn(_, _) -> _` instead of requiring a fully monomorphic
-    // `fn(Foo, Bar) -> Baz` etc.
+    /// NB: the (`fn` pointer) type must currently be monomorphic to avoid double substitution
+    /// problems with the MIR shim bodies. `Instance::resolve` enforces this.
+    // FIXME(#69925) support polymorphic MIR shim bodies properly instead.
     FnPtrShim(DefId, Ty<'tcx>),
 
     /// `<dyn Trait as Trait>::fn`, "direct calls" of which are implicitly
@@ -63,18 +62,16 @@ pub enum InstanceDef<'tcx> {
     /// The `Option<Ty<'tcx>>` is either `Some(T)`, or `None` for empty drop
     /// glue.
     ///
-    /// NB: the type must be monomorphic for MIR shims to work.
-    // FIXME(eddyb) support generating shims for a "shallow type",
-    // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
-    // `Foo<Bar>` or `[String]` etc.
+    /// NB: the type must currently be monomorphic to avoid double substitution
+    /// problems with the MIR shim bodies. `Instance::resolve` enforces this.
+    // FIXME(#69925) support polymorphic MIR shim bodies properly instead.
     DropGlue(DefId, Option<Ty<'tcx>>),
 
     ///`<T as Clone>::clone` shim.
     ///
-    /// NB: the type must be monomorphic for MIR shims to work.
-    // FIXME(eddyb) support generating shims for a "shallow type",
-    // e.g. `Foo<_>` or `[_]` instead of requiring a fully monomorphic
-    // `Foo<Bar>` or `[String]` etc.
+    /// NB: the type must currently be monomorphic to avoid double substitution
+    /// problems with the MIR shim bodies. `Instance::resolve` enforces this.
+    // FIXME(#69925) support polymorphic MIR shim bodies properly instead.
     CloneShim(DefId, Ty<'tcx>),
 }
 

From af0d6fc0070065df30af99fe8f1995548d81e903 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= <tomasz.miasko@gmail.com>
Date: Tue, 17 Mar 2020 00:00:00 +0000
Subject: [PATCH 24/26] Use copy bound in atomic operations to generate simpler
 MIR

---
 src/libcore/sync/atomic.rs | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/libcore/sync/atomic.rs b/src/libcore/sync/atomic.rs
index 3ba15968f8933..220f221cdd36d 100644
--- a/src/libcore/sync/atomic.rs
+++ b/src/libcore/sync/atomic.rs
@@ -2259,7 +2259,7 @@ fn strongest_failure_ordering(order: Ordering) -> Ordering {
 }
 
 #[inline]
-unsafe fn atomic_store<T>(dst: *mut T, val: T, order: Ordering) {
+unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
     match order {
         Release => intrinsics::atomic_store_rel(dst, val),
         Relaxed => intrinsics::atomic_store_relaxed(dst, val),
@@ -2270,7 +2270,7 @@ unsafe fn atomic_store<T>(dst: *mut T, val: T, order: Ordering) {
 }
 
 #[inline]
-unsafe fn atomic_load<T>(dst: *const T, order: Ordering) -> T {
+unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_load_acq(dst),
         Relaxed => intrinsics::atomic_load_relaxed(dst),
@@ -2282,7 +2282,7 @@ unsafe fn atomic_load<T>(dst: *const T, order: Ordering) -> T {
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_swap<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_xchg_acq(dst, val),
         Release => intrinsics::atomic_xchg_rel(dst, val),
@@ -2295,7 +2295,7 @@ unsafe fn atomic_swap<T>(dst: *mut T, val: T, order: Ordering) -> T {
 /// Returns the previous value (like __sync_fetch_and_add).
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_add<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_add<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_xadd_acq(dst, val),
         Release => intrinsics::atomic_xadd_rel(dst, val),
@@ -2308,7 +2308,7 @@ unsafe fn atomic_add<T>(dst: *mut T, val: T, order: Ordering) -> T {
 /// Returns the previous value (like __sync_fetch_and_sub).
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_sub<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_sub<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_xsub_acq(dst, val),
         Release => intrinsics::atomic_xsub_rel(dst, val),
@@ -2320,7 +2320,7 @@ unsafe fn atomic_sub<T>(dst: *mut T, val: T, order: Ordering) -> T {
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_compare_exchange<T>(
+unsafe fn atomic_compare_exchange<T: Copy>(
     dst: *mut T,
     old: T,
     new: T,
@@ -2346,7 +2346,7 @@ unsafe fn atomic_compare_exchange<T>(
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_compare_exchange_weak<T>(
+unsafe fn atomic_compare_exchange_weak<T: Copy>(
     dst: *mut T,
     old: T,
     new: T,
@@ -2372,7 +2372,7 @@ unsafe fn atomic_compare_exchange_weak<T>(
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_and<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_and<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_and_acq(dst, val),
         Release => intrinsics::atomic_and_rel(dst, val),
@@ -2384,7 +2384,7 @@ unsafe fn atomic_and<T>(dst: *mut T, val: T, order: Ordering) -> T {
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_nand<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_nand<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_nand_acq(dst, val),
         Release => intrinsics::atomic_nand_rel(dst, val),
@@ -2396,7 +2396,7 @@ unsafe fn atomic_nand<T>(dst: *mut T, val: T, order: Ordering) -> T {
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_or<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_or<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_or_acq(dst, val),
         Release => intrinsics::atomic_or_rel(dst, val),
@@ -2408,7 +2408,7 @@ unsafe fn atomic_or<T>(dst: *mut T, val: T, order: Ordering) -> T {
 
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_xor<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_xor_acq(dst, val),
         Release => intrinsics::atomic_xor_rel(dst, val),
@@ -2421,7 +2421,7 @@ unsafe fn atomic_xor<T>(dst: *mut T, val: T, order: Ordering) -> T {
 /// returns the max value (signed comparison)
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_max<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_max<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_max_acq(dst, val),
         Release => intrinsics::atomic_max_rel(dst, val),
@@ -2434,7 +2434,7 @@ unsafe fn atomic_max<T>(dst: *mut T, val: T, order: Ordering) -> T {
 /// returns the min value (signed comparison)
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_min<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_min<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_min_acq(dst, val),
         Release => intrinsics::atomic_min_rel(dst, val),
@@ -2447,7 +2447,7 @@ unsafe fn atomic_min<T>(dst: *mut T, val: T, order: Ordering) -> T {
 /// returns the max value (unsigned comparison)
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_umax<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_umax<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_umax_acq(dst, val),
         Release => intrinsics::atomic_umax_rel(dst, val),
@@ -2460,7 +2460,7 @@ unsafe fn atomic_umax<T>(dst: *mut T, val: T, order: Ordering) -> T {
 /// returns the min value (unsigned comparison)
 #[inline]
 #[cfg(target_has_atomic = "8")]
-unsafe fn atomic_umin<T>(dst: *mut T, val: T, order: Ordering) -> T {
+unsafe fn atomic_umin<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     match order {
         Acquire => intrinsics::atomic_umin_acq(dst, val),
         Release => intrinsics::atomic_umin_rel(dst, val),

From d5b6a20557743911ff9f90af5d8ad24c699570d3 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Thu, 12 Mar 2020 13:03:48 -0700
Subject: [PATCH 25/26] std: Don't abort process when printing panics in tests

This commit fixes an issue when using `set_print` and friends, notably
used by libtest, to avoid aborting the process if printing panics. This
previously panicked due to borrowing a mutable `RefCell` twice, and this
is worked around by borrowing these cells for less time, instead
taking out and removing contents temporarily.

Closes #69558
---
 src/libstd/io/stdio.rs                   | 12 ++++++++----
 src/test/ui/panic-while-printing.rs      | 24 ++++++++++++++++++++++++
 src/test/ui/test-panic-while-printing.rs | 24 ++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 4 deletions(-)
 create mode 100644 src/test/ui/panic-while-printing.rs
 create mode 100644 src/test/ui/test-panic-while-printing.rs

diff --git a/src/libstd/io/stdio.rs b/src/libstd/io/stdio.rs
index d410faca30d9e..0fb0757792ed7 100644
--- a/src/libstd/io/stdio.rs
+++ b/src/libstd/io/stdio.rs
@@ -792,10 +792,14 @@ fn print_to<T>(
 {
     let result = local_s
         .try_with(|s| {
-            if let Ok(mut borrowed) = s.try_borrow_mut() {
-                if let Some(w) = borrowed.as_mut() {
-                    return w.write_fmt(args);
-                }
+            // Note that we completely remove a local sink to write to in case
+            // our printing recursively panics/prints, so the recursive
+            // panic/print goes to the global sink instead of our local sink.
+            let prev = s.borrow_mut().take();
+            if let Some(mut w) = prev {
+                let result = w.write_fmt(args);
+                *s.borrow_mut() = Some(w);
+                return result;
             }
             global_s().write_fmt(args)
         })
diff --git a/src/test/ui/panic-while-printing.rs b/src/test/ui/panic-while-printing.rs
new file mode 100644
index 0000000000000..7e9fa16b0847a
--- /dev/null
+++ b/src/test/ui/panic-while-printing.rs
@@ -0,0 +1,24 @@
+// run-pass
+// ignore-emscripten no subprocess support
+
+#![feature(set_stdio)]
+
+use std::fmt;
+use std::fmt::{Display, Formatter};
+use std::io::set_panic;
+
+pub struct A;
+
+impl Display for A {
+    fn fmt(&self, _f: &mut Formatter<'_>) -> fmt::Result {
+        panic!();
+    }
+}
+
+fn main() {
+    set_panic(Some(Box::new(Vec::new())));
+    assert!(std::panic::catch_unwind(|| {
+        eprintln!("{}", A);
+    })
+    .is_err());
+}
diff --git a/src/test/ui/test-panic-while-printing.rs b/src/test/ui/test-panic-while-printing.rs
new file mode 100644
index 0000000000000..23f45407c1af3
--- /dev/null
+++ b/src/test/ui/test-panic-while-printing.rs
@@ -0,0 +1,24 @@
+// compile-flags:--test
+// run-pass
+// ignore-emscripten no subprocess support
+
+use std::fmt;
+use std::fmt::{Display, Formatter};
+
+pub struct A(Vec<u32>);
+
+impl Display for A {
+    fn fmt(&self, _f: &mut Formatter<'_>) -> fmt::Result {
+        self.0[0];
+        Ok(())
+    }
+}
+
+#[test]
+fn main() {
+    let result = std::panic::catch_unwind(|| {
+        let a = A(vec![]);
+        eprintln!("{}", a);
+    });
+    assert!(result.is_err());
+}

From 5e2856122a42a9e148d9d583561f7b07037bd3bc Mon Sep 17 00:00:00 2001
From: Aaron Hill <aa1ronham@gmail.com>
Date: Fri, 7 Feb 2020 14:02:24 -0500
Subject: [PATCH 26/26] Properly handle Spans that reference imported
 SourceFiles

Previously, metadata encoding used DUMMY_SP to represent any spans that
referenced an 'imported' SourceFile - e.g. a SourceFile from an upstream
dependency. These leads to sub-optimal error messages in certain cases
(see the included test).

This PR changes how we encode and decode spans in crate metadata. We
encode spans in one of two ways:

* 'Local' spans, which reference non-imported SourceFiles, are encoded
  exactly as before.
* 'Foreign' spans, which reference imported SourceFiles, are encoded
  with the CrateNum of their 'originating' crate. Additionally, their
'lo' and 'high' values are rebased on top of the 'originating' crate,
which allows them to be used with the SourceMap data encoded for that
crate.

The `ExternalSource` enum is renamed to `ExternalSourceKind`. There is
now a struct called `ExternalSource`, which holds an
`ExternalSourceKind` along with the original line number information for
the file. This is used during `Span` serialization to rebase spans onto
their 'owning' crate.
---
 src/librustc/hir/map/collector.rs             |  4 +-
 src/librustc/ich/impls_syntax.rs              |  8 +-
 src/librustc_metadata/rmeta/decoder.rs        | 91 +++++++++++++++++--
 src/librustc_metadata/rmeta/encoder.rs        | 61 ++++++++++---
 src/librustc_metadata/rmeta/mod.rs            |  5 +-
 src/librustc_span/lib.rs                      | 61 ++++++++-----
 src/librustc_span/source_map.rs               | 12 ++-
 .../ui/span/auxiliary/transitive_dep_three.rs |  9 ++
 .../ui/span/auxiliary/transitive_dep_two.rs   |  3 +
 src/test/ui/span/transitive-dep-span.rs       | 13 +++
 src/test/ui/span/transitive-dep-span.stderr   | 19 ++++
 11 files changed, 234 insertions(+), 52 deletions(-)
 create mode 100644 src/test/ui/span/auxiliary/transitive_dep_three.rs
 create mode 100644 src/test/ui/span/auxiliary/transitive_dep_two.rs
 create mode 100644 src/test/ui/span/transitive-dep-span.rs
 create mode 100644 src/test/ui/span/transitive-dep-span.stderr

diff --git a/src/librustc/hir/map/collector.rs b/src/librustc/hir/map/collector.rs
index e8233c0446d1f..8b276d0a762c3 100644
--- a/src/librustc/hir/map/collector.rs
+++ b/src/librustc/hir/map/collector.rs
@@ -10,7 +10,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::svh::Svh;
 use rustc_hir as hir;
 use rustc_hir::def_id::CRATE_DEF_INDEX;
-use rustc_hir::def_id::{CrateNum, DefIndex, LOCAL_CRATE};
+use rustc_hir::def_id::{DefIndex, LOCAL_CRATE};
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::*;
 use rustc_index::vec::{Idx, IndexVec};
@@ -175,7 +175,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> {
             .source_map
             .files()
             .iter()
-            .filter(|source_file| CrateNum::from_u32(source_file.crate_of_origin) == LOCAL_CRATE)
+            .filter(|source_file| source_file.cnum == LOCAL_CRATE)
             .map(|source_file| source_file.name_hash)
             .collect();
 
diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs
index daff8a0f1825e..c5a4b53b10df8 100644
--- a/src/librustc/ich/impls_syntax.rs
+++ b/src/librustc/ich/impls_syntax.rs
@@ -5,7 +5,6 @@ use crate::ich::StableHashingContext;
 
 use rustc_ast::ast;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_hir::def_id::{CrateNum, DefId, CRATE_DEF_INDEX};
 use rustc_span::SourceFile;
 
 use smallvec::SmallVec;
@@ -59,7 +58,7 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
             name_hash,
             name_was_remapped,
             unmapped_path: _,
-            crate_of_origin,
+            cnum,
             // Do not hash the source as it is not encoded
             src: _,
             src_hash,
@@ -75,9 +74,6 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
         (name_hash as u64).hash_stable(hcx, hasher);
         name_was_remapped.hash_stable(hcx, hasher);
 
-        DefId { krate: CrateNum::from_u32(crate_of_origin), index: CRATE_DEF_INDEX }
-            .hash_stable(hcx, hasher);
-
         src_hash.hash_stable(hcx, hasher);
 
         // We only hash the relative position within this source_file
@@ -101,6 +97,8 @@ impl<'a> HashStable<StableHashingContext<'a>> for SourceFile {
         for &char_pos in normalized_pos.iter() {
             stable_normalized_pos(char_pos, start_pos).hash_stable(hcx, hasher);
         }
+
+        cnum.hash_stable(hcx, hasher);
     }
 }
 
diff --git a/src/librustc_metadata/rmeta/decoder.rs b/src/librustc_metadata/rmeta/decoder.rs
index cf0f881605866..9cad086b4e863 100644
--- a/src/librustc_metadata/rmeta/decoder.rs
+++ b/src/librustc_metadata/rmeta/decoder.rs
@@ -386,7 +386,7 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
             return Ok(DUMMY_SP);
         }
 
-        debug_assert_eq!(tag, TAG_VALID_SPAN);
+        debug_assert!(tag == TAG_VALID_SPAN_LOCAL || tag == TAG_VALID_SPAN_FOREIGN);
 
         let lo = BytePos::decode(self)?;
         let len = BytePos::decode(self)?;
@@ -398,7 +398,68 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
             bug!("Cannot decode Span without Session.")
         };
 
-        let imported_source_files = self.cdata().imported_source_files(&sess.source_map());
+        // There are two possibilities here:
+        // 1. This is a 'local span', which is located inside a `SourceFile`
+        // that came from this crate. In this case, we use the source map data
+        // encoded in this crate. This branch should be taken nearly all of the time.
+        // 2. This is a 'foreign span', which is located inside a `SourceFile`
+        // that came from a *different* crate (some crate upstream of the one
+        // whose metadata we're looking at). For example, consider this dependency graph:
+        //
+        // A -> B -> C
+        //
+        // Suppose that we're currently compiling crate A, and start deserializing
+        // metadata from crate B. When we deserialize a Span from crate B's metadata,
+        // there are two posibilites:
+        //
+        // 1. The span references a file from crate B. This makes it a 'local' span,
+        // which means that we can use crate B's serialized source map information.
+        // 2. The span references a file from crate C. This makes it a 'foreign' span,
+        // which means we need to use Crate *C* (not crate B) to determine the source
+        // map information. We only record source map information for a file in the
+        // crate that 'owns' it, so deserializing a Span may require us to look at
+        // a transitive dependency.
+        //
+        // When we encode a foreign span, we adjust its 'lo' and 'high' values
+        // to be based on the *foreign* crate (e.g. crate C), not the crate
+        // we are writing metadata for (e.g. crate B). This allows us to
+        // treat the 'local' and 'foreign' cases almost identically during deserialization:
+        // we can call `imported_source_files` for the proper crate, and binary search
+        // through the returned slice using our span.
+        let imported_source_files = if tag == TAG_VALID_SPAN_LOCAL {
+            self.cdata().imported_source_files(sess.source_map())
+        } else {
+            // FIXME: We don't decode dependencies of proc-macros.
+            // Remove this once #69976 is merged
+            if self.cdata().root.is_proc_macro_crate() {
+                debug!(
+                    "SpecializedDecoder<Span>::specialized_decode: skipping span for proc-macro crate {:?}",
+                    self.cdata().cnum
+                );
+                // Decode `CrateNum` as u32 - using `CrateNum::decode` will ICE
+                // since we don't have `cnum_map` populated.
+                // This advances the decoder position so that we can continue
+                // to read metadata.
+                let _ = u32::decode(self)?;
+                return Ok(DUMMY_SP);
+            }
+            // tag is TAG_VALID_SPAN_FOREIGN, checked by `debug_assert` above
+            let cnum = CrateNum::decode(self)?;
+            debug!(
+                "SpecializedDecoder<Span>::specialized_decode: loading source files from cnum {:?}",
+                cnum
+            );
+
+            // Decoding 'foreign' spans should be rare enough that it's
+            // not worth it to maintain a per-CrateNum cache for `last_source_file_index`.
+            // We just set it to 0, to ensure that we don't try to access something out
+            // of bounds for our initial 'guess'
+            self.last_source_file_index = 0;
+
+            let foreign_data = self.cdata().cstore.get_crate_data(cnum);
+            foreign_data.imported_source_files(sess.source_map())
+        };
+
         let source_file = {
             // Optimize for the case that most spans within a translated item
             // originate from the same source_file.
@@ -412,16 +473,32 @@ impl<'a, 'tcx> SpecializedDecoder<Span> for DecodeContext<'a, 'tcx> {
                     .binary_search_by_key(&lo, |source_file| source_file.original_start_pos)
                     .unwrap_or_else(|index| index - 1);
 
-                self.last_source_file_index = index;
+                // Don't try to cache the index for foreign spans,
+                // as this would require a map from CrateNums to indices
+                if tag == TAG_VALID_SPAN_LOCAL {
+                    self.last_source_file_index = index;
+                }
                 &imported_source_files[index]
             }
         };
 
         // Make sure our binary search above is correct.
-        debug_assert!(lo >= source_file.original_start_pos && lo <= source_file.original_end_pos);
+        debug_assert!(
+            lo >= source_file.original_start_pos && lo <= source_file.original_end_pos,
+            "Bad binary search: lo={:?} source_file.original_start_pos={:?} source_file.original_end_pos={:?}",
+            lo,
+            source_file.original_start_pos,
+            source_file.original_end_pos
+        );
 
         // Make sure we correctly filtered out invalid spans during encoding
-        debug_assert!(hi >= source_file.original_start_pos && hi <= source_file.original_end_pos);
+        debug_assert!(
+            hi >= source_file.original_start_pos && hi <= source_file.original_end_pos,
+            "Bad binary search: hi={:?} source_file.original_start_pos={:?} source_file.original_end_pos={:?}",
+            hi,
+            source_file.original_start_pos,
+            source_file.original_end_pos
+        );
 
         let lo =
             (lo + source_file.translated_source_file.start_pos) - source_file.original_start_pos;
@@ -1425,14 +1502,16 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                     let local_version = local_source_map.new_imported_source_file(
                         name,
                         name_was_remapped,
-                        self.cnum.as_u32(),
                         src_hash,
                         name_hash,
                         source_length,
+                        self.cnum,
                         lines,
                         multibyte_chars,
                         non_narrow_chars,
                         normalized_pos,
+                        start_pos,
+                        end_pos,
                     );
                     debug!(
                         "CrateMetaData::imported_source_files alloc \
diff --git a/src/librustc_metadata/rmeta/encoder.rs b/src/librustc_metadata/rmeta/encoder.rs
index 90179520a6233..98528018d9e80 100644
--- a/src/librustc_metadata/rmeta/encoder.rs
+++ b/src/librustc_metadata/rmeta/encoder.rs
@@ -1,6 +1,7 @@
 use crate::rmeta::table::FixedSizeEncoding;
 use crate::rmeta::*;
 
+use log::{debug, trace};
 use rustc::hir::map::definitions::DefPathTable;
 use rustc::hir::map::Map;
 use rustc::middle::cstore::{EncodedMetadata, ForeignModule, LinkagePreference, NativeLibrary};
@@ -29,9 +30,7 @@ use rustc_serialize::{opaque, Encodable, Encoder, SpecializedEncoder};
 use rustc_session::config::{self, CrateType};
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::{self, FileName, SourceFile, Span};
-
-use log::{debug, trace};
+use rustc_span::{self, ExternalSource, FileName, SourceFile, Span};
 use std::hash::Hash;
 use std::num::NonZeroUsize;
 use std::path::Path;
@@ -165,20 +164,56 @@ impl<'tcx> SpecializedEncoder<Span> for EncodeContext<'tcx> {
             return TAG_INVALID_SPAN.encode(self);
         }
 
-        // HACK(eddyb) there's no way to indicate which crate a Span is coming
-        // from right now, so decoding would fail to find the SourceFile if
-        // it's not local to the crate the Span is found in.
-        if self.source_file_cache.is_imported() {
-            return TAG_INVALID_SPAN.encode(self);
-        }
+        // There are two possible cases here:
+        // 1. This span comes from a 'foreign' crate - e.g. some crate upstream of the
+        // crate we are writing metadata for. When the metadata for *this* crate gets
+        // deserialized, the deserializer will need to know which crate it originally came
+        // from. We use `TAG_VALID_SPAN_FOREIGN` to indicate that a `CrateNum` should
+        // be deserialized after the rest of the span data, which tells the deserializer
+        // which crate contains the source map information.
+        // 2. This span comes from our own crate. No special hamdling is needed - we just
+        // write `TAG_VALID_SPAN_LOCAL` to let the deserializer know that it should use
+        // our own source map information.
+        let (tag, lo, hi) = if self.source_file_cache.is_imported() {
+            // To simplify deserialization, we 'rebase' this span onto the crate it originally came from
+            // (the crate that 'owns' the file it references. These rebased 'lo' and 'hi' values
+            // are relative to the source map information for the 'foreign' crate whose CrateNum
+            // we write into the metadata. This allows `imported_source_files` to binary
+            // search through the 'foreign' crate's source map information, using the
+            // deserialized 'lo' and 'hi' values directly.
+            //
+            // All of this logic ensures that the final result of deserialization is a 'normal'
+            // Span that can be used without any additional trouble.
+            let external_start_pos = {
+                // Introduce a new scope so that we drop the 'lock()' temporary
+                match &*self.source_file_cache.external_src.lock() {
+                    ExternalSource::Foreign { original_start_pos, .. } => *original_start_pos,
+                    src => panic!("Unexpected external source {:?}", src),
+                }
+            };
+            let lo = (span.lo - self.source_file_cache.start_pos) + external_start_pos;
+            let hi = (span.hi - self.source_file_cache.start_pos) + external_start_pos;
 
-        TAG_VALID_SPAN.encode(self)?;
-        span.lo.encode(self)?;
+            (TAG_VALID_SPAN_FOREIGN, lo, hi)
+        } else {
+            (TAG_VALID_SPAN_LOCAL, span.lo, span.hi)
+        };
+
+        tag.encode(self)?;
+        lo.encode(self)?;
 
         // Encode length which is usually less than span.hi and profits more
         // from the variable-length integer encoding that we use.
-        let len = span.hi - span.lo;
-        len.encode(self)
+        let len = hi - lo;
+        len.encode(self)?;
+
+        if tag == TAG_VALID_SPAN_FOREIGN {
+            // This needs to be two lines to avoid holding the `self.source_file_cache`
+            // while calling `cnum.encode(self)`
+            let cnum = self.source_file_cache.cnum;
+            cnum.encode(self)?;
+        }
+        Ok(())
 
         // Don't encode the expansion context.
     }
diff --git a/src/librustc_metadata/rmeta/mod.rs b/src/librustc_metadata/rmeta/mod.rs
index defa306b6d6fe..05d834e5dee12 100644
--- a/src/librustc_metadata/rmeta/mod.rs
+++ b/src/librustc_metadata/rmeta/mod.rs
@@ -405,5 +405,6 @@ struct GeneratorData<'tcx> {
 }
 
 // Tags used for encoding Spans:
-const TAG_VALID_SPAN: u8 = 0;
-const TAG_INVALID_SPAN: u8 = 1;
+const TAG_VALID_SPAN_LOCAL: u8 = 0;
+const TAG_VALID_SPAN_FOREIGN: u8 = 1;
+const TAG_INVALID_SPAN: u8 = 2;
diff --git a/src/librustc_span/lib.rs b/src/librustc_span/lib.rs
index 1f6d10f4e8f6a..dbc180114f1c1 100644
--- a/src/librustc_span/lib.rs
+++ b/src/librustc_span/lib.rs
@@ -27,7 +27,7 @@ pub mod hygiene;
 use hygiene::Transparency;
 pub use hygiene::{DesugaringKind, ExpnData, ExpnId, ExpnKind, MacroKind, SyntaxContext};
 pub mod def_id;
-use def_id::DefId;
+use def_id::{CrateNum, DefId, LOCAL_CRATE};
 mod span_encoding;
 pub use span_encoding::{Span, DUMMY_SP};
 
@@ -839,30 +839,42 @@ pub struct NormalizedPos {
     pub diff: u32,
 }
 
-/// The state of the lazy external source loading mechanism of a `SourceFile`.
-#[derive(PartialEq, Eq, Clone)]
+#[derive(PartialEq, Eq, Clone, Debug)]
 pub enum ExternalSource {
+    /// No external source has to be loaded, since the `SourceFile` represents a local crate.
+    Unneeded,
+    Foreign {
+        kind: ExternalSourceKind,
+        /// This SourceFile's byte-offset within the source_map of its original crate
+        original_start_pos: BytePos,
+        /// The end of this SourceFile within the source_map of its original crate
+        original_end_pos: BytePos,
+    },
+}
+
+/// The state of the lazy external source loading mechanism of a `SourceFile`.
+#[derive(PartialEq, Eq, Clone, Debug)]
+pub enum ExternalSourceKind {
     /// The external source has been loaded already.
     Present(String),
     /// No attempt has been made to load the external source.
     AbsentOk,
     /// A failed attempt has been made to load the external source.
     AbsentErr,
-    /// No external source has to be loaded, since the `SourceFile` represents a local crate.
     Unneeded,
 }
 
 impl ExternalSource {
     pub fn is_absent(&self) -> bool {
-        match *self {
-            ExternalSource::Present(_) => false,
+        match self {
+            ExternalSource::Foreign { kind: ExternalSourceKind::Present(_), .. } => false,
             _ => true,
         }
     }
 
     pub fn get_source(&self) -> Option<&str> {
-        match *self {
-            ExternalSource::Present(ref src) => Some(src),
+        match self {
+            ExternalSource::Foreign { kind: ExternalSourceKind::Present(ref src), .. } => Some(src),
             _ => None,
         }
     }
@@ -883,8 +895,6 @@ pub struct SourceFile {
     /// The unmapped path of the file that the source came from.
     /// Set to `None` if the `SourceFile` was imported from an external crate.
     pub unmapped_path: Option<FileName>,
-    /// Indicates which crate this `SourceFile` was imported from.
-    pub crate_of_origin: u32,
     /// The complete source code.
     pub src: Option<Lrc<String>>,
     /// The source code's hash.
@@ -906,6 +916,8 @@ pub struct SourceFile {
     pub normalized_pos: Vec<NormalizedPos>,
     /// A hash of the filename, used for speeding up hashing in incremental compilation.
     pub name_hash: u128,
+    /// Indicates which crate this `SourceFile` was imported from.
+    pub cnum: CrateNum,
 }
 
 impl Encodable for SourceFile {
@@ -972,7 +984,8 @@ impl Encodable for SourceFile {
             s.emit_struct_field("multibyte_chars", 6, |s| self.multibyte_chars.encode(s))?;
             s.emit_struct_field("non_narrow_chars", 7, |s| self.non_narrow_chars.encode(s))?;
             s.emit_struct_field("name_hash", 8, |s| self.name_hash.encode(s))?;
-            s.emit_struct_field("normalized_pos", 9, |s| self.normalized_pos.encode(s))
+            s.emit_struct_field("normalized_pos", 9, |s| self.normalized_pos.encode(s))?;
+            s.emit_struct_field("cnum", 10, |s| self.cnum.encode(s))
         })
     }
 }
@@ -1022,24 +1035,24 @@ impl Decodable for SourceFile {
             let name_hash: u128 = d.read_struct_field("name_hash", 8, |d| Decodable::decode(d))?;
             let normalized_pos: Vec<NormalizedPos> =
                 d.read_struct_field("normalized_pos", 9, |d| Decodable::decode(d))?;
+            let cnum: CrateNum = d.read_struct_field("cnum", 10, |d| Decodable::decode(d))?;
             Ok(SourceFile {
                 name,
                 name_was_remapped,
                 unmapped_path: None,
-                // `crate_of_origin` has to be set by the importer.
-                // This value matches up with `rustc_hir::def_id::INVALID_CRATE`.
-                // That constant is not available here, unfortunately.
-                crate_of_origin: std::u32::MAX - 1,
                 start_pos,
                 end_pos,
                 src: None,
                 src_hash,
-                external_src: Lock::new(ExternalSource::AbsentOk),
+                // Unused - the metadata decoder will construct
+                // a new SourceFile, filling in `external_src` properly
+                external_src: Lock::new(ExternalSource::Unneeded),
                 lines,
                 multibyte_chars,
                 non_narrow_chars,
                 normalized_pos,
                 name_hash,
+                cnum,
             })
         })
     }
@@ -1081,7 +1094,6 @@ impl SourceFile {
             name,
             name_was_remapped,
             unmapped_path: Some(unmapped_path),
-            crate_of_origin: 0,
             src: Some(Lrc::new(src)),
             src_hash,
             external_src: Lock::new(ExternalSource::Unneeded),
@@ -1092,6 +1104,7 @@ impl SourceFile {
             non_narrow_chars,
             normalized_pos,
             name_hash,
+            cnum: LOCAL_CRATE,
         }
     }
 
@@ -1109,21 +1122,27 @@ impl SourceFile {
     where
         F: FnOnce() -> Option<String>,
     {
-        if *self.external_src.borrow() == ExternalSource::AbsentOk {
+        if matches!(
+            *self.external_src.borrow(),
+            ExternalSource::Foreign { kind: ExternalSourceKind::AbsentOk, .. }
+        ) {
             let src = get_src();
             let mut external_src = self.external_src.borrow_mut();
             // Check that no-one else have provided the source while we were getting it
-            if *external_src == ExternalSource::AbsentOk {
+            if let ExternalSource::Foreign {
+                kind: src_kind @ ExternalSourceKind::AbsentOk, ..
+            } = &mut *external_src
+            {
                 if let Some(src) = src {
                     let mut hasher: StableHasher = StableHasher::new();
                     hasher.write(src.as_bytes());
 
                     if hasher.finish::<u128>() == self.src_hash {
-                        *external_src = ExternalSource::Present(src);
+                        *src_kind = ExternalSourceKind::Present(src);
                         return true;
                     }
                 } else {
-                    *external_src = ExternalSource::AbsentErr;
+                    *src_kind = ExternalSourceKind::AbsentErr;
                 }
 
                 false
diff --git a/src/librustc_span/source_map.rs b/src/librustc_span/source_map.rs
index 353f7b3f52bc3..7dd9e2f6316b4 100644
--- a/src/librustc_span/source_map.rs
+++ b/src/librustc_span/source_map.rs
@@ -296,14 +296,16 @@ impl SourceMap {
         &self,
         filename: FileName,
         name_was_remapped: bool,
-        crate_of_origin: u32,
         src_hash: u128,
         name_hash: u128,
         source_len: usize,
+        cnum: CrateNum,
         mut file_local_lines: Vec<BytePos>,
         mut file_local_multibyte_chars: Vec<MultiByteChar>,
         mut file_local_non_narrow_chars: Vec<NonNarrowChar>,
         mut file_local_normalized_pos: Vec<NormalizedPos>,
+        original_start_pos: BytePos,
+        original_end_pos: BytePos,
     ) -> Lrc<SourceFile> {
         let start_pos = self
             .allocate_address_space(source_len)
@@ -332,10 +334,13 @@ impl SourceMap {
             name: filename,
             name_was_remapped,
             unmapped_path: None,
-            crate_of_origin,
             src: None,
             src_hash,
-            external_src: Lock::new(ExternalSource::AbsentOk),
+            external_src: Lock::new(ExternalSource::Foreign {
+                kind: ExternalSourceKind::AbsentOk,
+                original_start_pos,
+                original_end_pos,
+            }),
             start_pos,
             end_pos,
             lines: file_local_lines,
@@ -343,6 +348,7 @@ impl SourceMap {
             non_narrow_chars: file_local_non_narrow_chars,
             normalized_pos: file_local_normalized_pos,
             name_hash,
+            cnum,
         });
 
         let mut files = self.files.borrow_mut();
diff --git a/src/test/ui/span/auxiliary/transitive_dep_three.rs b/src/test/ui/span/auxiliary/transitive_dep_three.rs
new file mode 100644
index 0000000000000..99b51625ac3ec
--- /dev/null
+++ b/src/test/ui/span/auxiliary/transitive_dep_three.rs
@@ -0,0 +1,9 @@
+#[macro_export]
+macro_rules! define_parse_error {
+    () => {
+        #[macro_export]
+        macro_rules! parse_error {
+            () => { parse error }
+        }
+    }
+}
diff --git a/src/test/ui/span/auxiliary/transitive_dep_two.rs b/src/test/ui/span/auxiliary/transitive_dep_two.rs
new file mode 100644
index 0000000000000..5110c42765b6d
--- /dev/null
+++ b/src/test/ui/span/auxiliary/transitive_dep_two.rs
@@ -0,0 +1,3 @@
+extern crate transitive_dep_three;
+
+transitive_dep_three::define_parse_error!();
diff --git a/src/test/ui/span/transitive-dep-span.rs b/src/test/ui/span/transitive-dep-span.rs
new file mode 100644
index 0000000000000..b445d389c561a
--- /dev/null
+++ b/src/test/ui/span/transitive-dep-span.rs
@@ -0,0 +1,13 @@
+// Tests that we properly serialize/deserialize spans from transitive dependencies
+// (e.g. imported SourceFiles)
+//
+// The order of these next lines is important, since we need
+// transitive_dep_two.rs to be able to reference transitive_dep_three.rs
+//
+// aux-build: transitive_dep_three.rs
+// aux-build: transitive_dep_two.rs
+// compile-flags: -Z macro-backtrace
+
+extern crate transitive_dep_two;
+
+transitive_dep_two::parse_error!(); //~ ERROR expected one of
diff --git a/src/test/ui/span/transitive-dep-span.stderr b/src/test/ui/span/transitive-dep-span.stderr
new file mode 100644
index 0000000000000..68d8911a4351c
--- /dev/null
+++ b/src/test/ui/span/transitive-dep-span.stderr
@@ -0,0 +1,19 @@
+error: expected one of `!` or `::`, found `error`
+  --> $DIR/auxiliary/transitive_dep_three.rs:6:27
+   |
+LL | /         macro_rules! parse_error {
+LL | |             () => { parse error }
+   | |                           ^^^^^ expected one of `!` or `::`
+LL | |         }
+   | |_________- in this expansion of `transitive_dep_two::parse_error!`
+   | 
+  ::: $DIR/transitive-dep-span.rs:13:1
+   |
+LL |   transitive_dep_two::parse_error!();
+   |   -----------------------------------
+   |   |
+   |   in this macro invocation
+   |   in this macro invocation
+
+error: aborting due to previous error
+