Rollup merge of rust-lang#80226 - ThePuzzlemaker:issue-80004-fix, r=j…

…yn514,petrochenkov

Highlight edition-specific keywords correctly in code blocks, accounting for code block edition modifiers

Previously, edition-specific keywords (such as `async` and `await`) were not highlighted in code blocks, regardless of what edition was set. With this PR, this issue is fixed.

Now, the following behavior happens:
- When a code block is explicitly set to edition X, keywords from edition X are highlighted
- When a code block is explicitly set to a version that does not contain those keywords from edition X (e.g. edition Y), keywords from edition X are **not** highlighted
- When a code block has no explicit edition, keywords from the edition passed via `--edition` to rustdoc are highlighted

For example, a project set with `edition = "2015"` in its `Cargo.toml` would not highlight `async`/`await` unless the code block was set to `edition2018`. Additionally, a project set with `edition = "2018"` in its `Cargo.toml` *would* highlight `async`/`await` unless the code block was set to a version that did not contain those keywords (e.g. `edition2015`).

This PR fixes rust-lang#80004.

r? `@jyn514`

src/librustdoc/html/highlight.rs

@@ -11,7 +11,8 @@ use std::fmt::{Display, Write};
 use std::iter::Peekable;
 
 use rustc_lexer::{LiteralKind, TokenKind};
-use rustc_span::symbol::Ident;
+use rustc_span::edition::Edition;
+use rustc_span::symbol::Symbol;
 use rustc_span::with_default_session_globals;
 
 /// Highlights `src`, returning the HTML output.
@@ -20,6 +21,7 @@ crate fn render_with_highlighting(
  class: Option<&str>,
  playground_button: Option<&str>,
  tooltip: Option<(&str, &str)>,
+ edition: Edition,
 ) -> String {
  debug!("highlighting: ================\n{}\n==============", src);
  let mut out = String::with_capacity(src.len());
@@ -34,7 +36,7 @@ crate fn render_with_highlighting(
  }
 
  write_header(&mut out, class);
- write_code(&mut out, &src);
+ write_code(&mut out, &src, edition);
  write_footer(&mut out, playground_button);
 
  out
@@ -45,10 +47,10 @@ fn write_header(out: &mut String, class: Option<&str>) {
  .unwrap()
 }
 
-fn write_code(out: &mut String, src: &str) {
+fn write_code(out: &mut String, src: &str, edition: Edition) {
  // This replace allows to fix how the code source with DOS backline characters is displayed.
  let src = src.replace("\r\n", "\n");
- Classifier::new(&src).highlight(&mut |highlight| {
+ Classifier::new(&src, edition).highlight(&mut |highlight| {
  match highlight {
  Highlight::Token { text, class } => string(out, Escape(text), class),
  Highlight::EnterSpan { class } => enter_span(out, class),
@@ -139,12 +141,19 @@ struct Classifier<'a> {
  in_attribute: bool,
  in_macro: bool,
  in_macro_nonterminal: bool,
+ edition: Edition,
 }
 
 impl<'a> Classifier<'a> {
- fn new(src: &str) -> Classifier<'_> {
+ fn new(src: &str, edition: Edition) -> Classifier<'_> {
  let tokens = TokenIter { src }.peekable();
- Classifier { tokens, in_attribute: false, in_macro: false, in_macro_nonterminal: false }
+ Classifier {
+ tokens,
+ in_attribute: false,
+ in_macro: false,
+ in_macro_nonterminal: false,
+ edition,
+ }
  }
 
  /// Exhausts the `Classifier` writing the output into `sink`.
@@ -296,7 +305,7 @@ impl<'a> Classifier<'a> {
  "Option" | "Result" => Class::PreludeTy,
  "Some" | "None" | "Ok" | "Err" => Class::PreludeVal,
  // Keywords are also included in the identifier set.
- _ if Ident::from_str(text).is_reserved() => Class::KeyWord,
+ _ if Symbol::intern(text).is_reserved(|| self.edition) => Class::KeyWord,
  _ if self.in_macro_nonterminal => {
  self.in_macro_nonterminal = false;
  Class::MacroNonTerminal

src/librustdoc/html/highlight/tests.rs

@@ -1,5 +1,6 @@
 use super::write_code;
 use expect_test::expect_file;
+use rustc_span::edition::Edition;
 
 const STYLE: &str = r#"
 <style>
@@ -18,7 +19,7 @@ fn test_html_highlighting() {
  let src = include_str!("fixtures/sample.rs");
  let html = {
  let mut out = String::new();
- write_code(&mut out, src);
+ write_code(&mut out, src, Edition::Edition2018);
  format!("{}<pre><code>{}</code></pre>\n", STYLE, out)
  };
  expect_file!["fixtures/sample.html"].assert_eq(&html);
@@ -30,6 +31,6 @@ fn test_dos_backline() {
  println!(\"foo\");\r\n\
 }\r\n";
  let mut html = String::new();
- write_code(&mut html, src);
+ write_code(&mut html, src, Edition::Edition2018);
  expect_file!["fixtures/dos_line.html"].assert_eq(&html);
 }

src/librustdoc/html/markdown.rs

@@ -314,6 +314,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
  )),
  playground_button.as_deref(),
  Some((s1.as_str(), s2)),
+ edition,
  ));
  Some(Event::Html(s.into()))
  } else {
@@ -335,6 +336,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
  )),
  playground_button.as_deref(),
  None,
+ edition,
  ));
  Some(Event::Html(s.into()))
  }

src/librustdoc/html/render/mod.rs

@@ -4747,6 +4747,7 @@ fn item_macro(w: &mut Buffer, cx: &Context<'_>, it: &clean::Item, t: &clean::Mac
  Some("macro"),
  None,
  None,
+ it.source.span().edition(),
  ))
  });
  document(w, cx, it, None)

src/librustdoc/html/sources.rs

@@ -8,6 +8,7 @@ use crate::html::layout;
 use crate::html::render::{SharedContext, BASIC_KEYWORDS};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_session::Session;
+use rustc_span::edition::Edition;
 use rustc_span::source_map::FileName;
 use std::ffi::OsStr;
 use std::fs;
@@ -132,7 +133,7 @@ impl SourceCollector<'_, '_> {
  &self.scx.layout,
  &page,
  "",
- |buf: &mut _| print_src(buf, contents),
+ |buf: &mut _| print_src(buf, contents, self.scx.edition),
  &self.scx.style_files,
  );
  self.scx.fs.write(&cur, v.as_bytes())?;
@@ -170,7 +171,7 @@ where
 
 /// Wrapper struct to render the source code of a file. This will do things like
 /// adding line numbers to the left-hand side.
-fn print_src(buf: &mut Buffer, s: String) {
+fn print_src(buf: &mut Buffer, s: String, edition: Edition) {
  let lines = s.lines().count();
  let mut cols = 0;
  let mut tmp = lines;
@@ -183,5 +184,5 @@ fn print_src(buf: &mut Buffer, s: String) {
  write!(buf, "<span id=\"{0}\">{0:1$}</span>\n", i, cols);
  }
  write!(buf, "</pre>");
- write!(buf, "{}", highlight::render_with_highlighting(s, None, None, None));
+ write!(buf, "{}", highlight::render_with_highlighting(s, None, None, None, edition));
 }