Auto merge of #80226 - ThePuzzlemaker:issue-80004-fix, r=jyn514,petro…

…chenkov

Highlight edition-specific keywords correctly in code blocks, accounting for code block edition modifiers

Previously, edition-specific keywords (such as `async` and `await`) were not highlighted in code blocks, regardless of what edition was set. With this PR, this issue is fixed.

Now, the following behavior happens:
- When a code block is explicitly set to edition X, keywords from edition X are highlighted
- When a code block is explicitly set to a version that does not contain those keywords from edition X (e.g. edition Y), keywords from edition X are **not** highlighted
- When a code block has no explicit edition, keywords from the edition passed via `--edition` to rustdoc are highlighted

For example, a project set with `edition = "2015"` in its `Cargo.toml` would not highlight `async`/`await` unless the code block was set to `edition2018`. Additionally, a project set with `edition = "2018"` in its `Cargo.toml` *would* highlight `async`/`await` unless the code block was set to a version that did not contain those keywords (e.g. `edition2015`).

This PR fixes #80004.

r? `@jyn514`

src/librustdoc/html/highlight.rs

@@ -12,7 +12,7 @@ use std::iter::Peekable;
 
 use rustc_lexer::{LiteralKind, TokenKind};
 use rustc_span::edition::Edition;
-use rustc_span::symbol::Ident;
+use rustc_span::symbol::Symbol;
 use rustc_span::with_default_session_globals;
 
 /// Highlights `src`, returning the HTML output.
@@ -21,6 +21,7 @@ crate fn render_with_highlighting(
     class: Option<&str>,
     playground_button: Option<&str>,
     tooltip: Option<(Option<Edition>, &str)>,
+    edition: Edition,
 ) -> String {
     debug!("highlighting: ================\n{}\n==============", src);
     let mut out = String::with_capacity(src.len());
@@ -39,7 +40,7 @@ crate fn render_with_highlighting(
     }
 
     write_header(&mut out, class);
-    write_code(&mut out, &src);
+    write_code(&mut out, &src, edition);
     write_footer(&mut out, playground_button);
 
     out
@@ -50,10 +51,10 @@ fn write_header(out: &mut String, class: Option<&str>) {
         .unwrap()
 }
 
-fn write_code(out: &mut String, src: &str) {
+fn write_code(out: &mut String, src: &str, edition: Edition) {
     // This replace allows to fix how the code source with DOS backline characters is displayed.
     let src = src.replace("\r\n", "\n");
-    Classifier::new(&src).highlight(&mut |highlight| {
+    Classifier::new(&src, edition).highlight(&mut |highlight| {
         match highlight {
             Highlight::Token { text, class } => string(out, Escape(text), class),
             Highlight::EnterSpan { class } => enter_span(out, class),
@@ -144,12 +145,19 @@ struct Classifier<'a> {
     in_attribute: bool,
     in_macro: bool,
     in_macro_nonterminal: bool,
+    edition: Edition,
 }
 
 impl<'a> Classifier<'a> {
-    fn new(src: &str) -> Classifier<'_> {
+    fn new(src: &str, edition: Edition) -> Classifier<'_> {
         let tokens = TokenIter { src }.peekable();
-        Classifier { tokens, in_attribute: false, in_macro: false, in_macro_nonterminal: false }
+        Classifier {
+            tokens,
+            in_attribute: false,
+            in_macro: false,
+            in_macro_nonterminal: false,
+            edition,
+        }
     }
 
     /// Exhausts the `Classifier` writing the output into `sink`.
@@ -301,7 +309,7 @@ impl<'a> Classifier<'a> {
                 "Option" | "Result" => Class::PreludeTy,
                 "Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
                 // Keywords are also included in the identifier set.
-                _ if Ident::from_str(text).is_reserved() => Class::KeyWord,
+                _ if Symbol::intern(text).is_reserved(|| self.edition) => Class::KeyWord,
                 _ if self.in_macro_nonterminal => {
                     self.in_macro_nonterminal = false;
                     Class::MacroNonTerminal

src/librustdoc/html/highlight/tests.rs

@@ -1,5 +1,6 @@
 use super::write_code;
 use expect_test::expect_file;
+use rustc_span::edition::Edition;
 
 const STYLE: &str = r#"
 <style>
@@ -18,7 +19,7 @@ fn test_html_highlighting() {
     let src = include_str!("fixtures/sample.rs");
     let html = {
         let mut out = String::new();
-        write_code(&mut out, src);
+        write_code(&mut out, src, Edition::Edition2018);
         format!("{}<pre><code>{}</code></pre>\n", STYLE, out)
     };
     expect_file!["fixtures/sample.html"].assert_eq(&html);
@@ -30,6 +31,6 @@ fn test_dos_backline() {
     println!(\"foo\");\r\n\
 }\r\n";
     let mut html = String::new();
-    write_code(&mut html, src);
+    write_code(&mut html, src, Edition::Edition2018);
     expect_file!["fixtures/dos_line.html"].assert_eq(&html);
 }

src/librustdoc/html/markdown.rs

@@ -303,6 +303,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
             )),
             playground_button.as_deref(),
             tooltip,
+            edition,
         ));
         Some(Event::Html(s.into()))
     }

src/librustdoc/html/render/mod.rs

@@ -4747,6 +4747,7 @@ fn item_macro(w: &mut Buffer, cx: &Context<'_>, it: &clean::Item, t: &clean::Mac
             Some("macro"),
             None,
             None,
+            it.source.span().edition(),
         ))
     });
     document(w, cx, it, None)

src/librustdoc/html/sources.rs

@@ -8,6 +8,7 @@ use crate::html::layout;
 use crate::html::render::{SharedContext, BASIC_KEYWORDS};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_session::Session;
+use rustc_span::edition::Edition;
 use rustc_span::source_map::FileName;
 use std::ffi::OsStr;
 use std::fs;
@@ -132,7 +133,7 @@ impl SourceCollector<'_, '_> {
             &self.scx.layout,
             &page,
             "",
-            |buf: &mut _| print_src(buf, contents),
+            |buf: &mut _| print_src(buf, contents, self.scx.edition),
             &self.scx.style_files,
         );
         self.scx.fs.write(&cur, v.as_bytes())?;
@@ -170,7 +171,7 @@ where
 
 /// Wrapper struct to render the source code of a file. This will do things like
 /// adding line numbers to the left-hand side.
-fn print_src(buf: &mut Buffer, s: String) {
+fn print_src(buf: &mut Buffer, s: String, edition: Edition) {
     let lines = s.lines().count();
     let mut cols = 0;
     let mut tmp = lines;
@@ -183,5 +184,5 @@ fn print_src(buf: &mut Buffer, s: String) {
         write!(buf, "<span id=\"{0}\">{0:1$}</span>\n", i, cols);
     }
     write!(buf, "</pre>");
-    write!(buf, "{}", highlight::render_with_highlighting(s, None, None, None));
+    write!(buf, "{}", highlight::render_with_highlighting(s, None, None, None, edition));
 }