Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 0e29546

Browse files
committedMay 30, 2024··
[WIP] RIIR HtmlDocCk
1 parent 0a59f11 commit 0e29546

File tree

18 files changed

+915
-314
lines changed

18 files changed

+915
-314
lines changed
 

‎Cargo.lock

+248-302
Large diffs are not rendered by default.

‎Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ members = [
3131
"src/tools/miri/cargo-miri",
3232
"src/tools/rustdoc-themes",
3333
"src/tools/unicode-table-generator",
34+
"src/tools/htmldocck",
3435
"src/tools/jsondocck",
3536
"src/tools/jsondoclint",
3637
"src/tools/llvm-bitcode-linker",

‎src/bootstrap/src/core/build_steps/clippy.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,8 @@ lint_any!(
311311
CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata";
312312
Compiletest, "src/tools/compiletest", "compiletest";
313313
CoverageDump, "src/tools/coverage-dump", "coverage-dump";
314-
Jsondocck, "src/tools/jsondocck", "jsondocck";
314+
HtmldocCk, "src/tools/htmldocck", "htmldocck";
315+
JsondocCk, "src/tools/jsondocck", "jsondocck";
315316
Jsondoclint, "src/tools/jsondoclint", "jsondoclint";
316317
LintDocs, "src/tools/lint-docs", "lint-docs";
317318
LlvmBitcodeLinker, "src/tools/llvm-bitcode-linker", "llvm-bitcode-linker";

‎src/bootstrap/src/core/build_steps/test.rs

+10-5
Original file line numberDiff line numberDiff line change
@@ -1766,13 +1766,18 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
17661766
cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
17671767
}
17681768

1769+
if mode == "rustdoc" {
1770+
// Use the beta compiler for htmldocck.
1771+
let compiler = compiler.with_stage(0);
1772+
cmd.arg("--htmldocck-path").arg(builder.ensure(tool::HtmlDocCk { compiler, target }));
1773+
}
1774+
17691775
if mode == "rustdoc-json" {
1770-
// Use the beta compiler for jsondocck
1771-
let json_compiler = compiler.with_stage(0);
1772-
cmd.arg("--jsondocck-path")
1773-
.arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }));
1776+
// Use the beta compiler for jsondocck.
1777+
let compiler = compiler.with_stage(0);
1778+
cmd.arg("--jsondocck-path").arg(builder.ensure(tool::JsonDocCk { compiler, target }));
17741779
cmd.arg("--jsondoclint-path")
1775-
.arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
1780+
.arg(builder.ensure(tool::JsonDocLint { compiler, target }));
17761781
}
17771782

17781783
if mode == "coverage-map" {

‎src/bootstrap/src/core/build_steps/tool.rs

+1
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ bootstrap_tool!(
303303
RustInstaller, "src/tools/rust-installer", "rust-installer";
304304
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
305305
LintDocs, "src/tools/lint-docs", "lint-docs";
306+
HtmlDocCk, "src/tools/htmldocck", "htmldocck";
306307
JsonDocCk, "src/tools/jsondocck", "jsondocck";
307308
JsonDocLint, "src/tools/jsondoclint", "jsondoclint";
308309
HtmlChecker, "src/tools/html-checker", "html-checker";

‎src/bootstrap/src/core/builder.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -764,7 +764,8 @@ impl<'a> Builder<'a> {
764764
clippy::CollectLicenseMetadata,
765765
clippy::Compiletest,
766766
clippy::CoverageDump,
767-
clippy::Jsondocck,
767+
clippy::HtmldocCk,
768+
clippy::JsondocCk,
768769
clippy::Jsondoclint,
769770
clippy::LintDocs,
770771
clippy::LlvmBitcodeLinker,

‎src/tools/compiletest/src/common.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -193,9 +193,12 @@ pub struct Config {
193193
/// The coverage-dump executable.
194194
pub coverage_dump_path: Option<PathBuf>,
195195

196-
/// The Python executable to use for LLDB and htmldocck.
196+
/// The Python executable to use for LLDB.
197197
pub python: String,
198198

199+
/// The htmldocck executable.
200+
pub htmldocck_path: Option<String>,
201+
199202
/// The jsondocck executable.
200203
pub jsondocck_path: Option<String>,
201204

‎src/tools/compiletest/src/header/tests.rs

+1
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ impl ConfigBuilder {
131131
"--compile-lib-path=",
132132
"--run-lib-path=",
133133
"--python=",
134+
// FIXME(fmease): Do we need to set htmldocck-path to "", too?
134135
"--jsondocck-path=",
135136
"--src-base=",
136137
"--build-base=",

‎src/tools/compiletest/src/lib.rs

+4
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@ pub fn parse_config(args: Vec<String>) -> Config {
4848
.optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
4949
.optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
5050
.optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
51+
// FIXME(fmease): fix docs here
5152
.reqopt("", "python", "path to python to use for doc tests", "PATH")
53+
.optopt("", "htmldocck-path", "path to htmldocck to use for doc tests", "PATH")
5254
.optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
5355
.optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
5456
.optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM")
@@ -235,6 +237,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
235237
rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
236238
coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
237239
python: matches.opt_str("python").unwrap(),
240+
htmldocck_path: matches.opt_str("htmldocck-path"),
238241
jsondocck_path: matches.opt_str("jsondocck-path"),
239242
jsondoclint_path: matches.opt_str("jsondoclint-path"),
240243
valgrind_path: matches.opt_str("valgrind-path"),
@@ -617,6 +620,7 @@ fn common_inputs_stamp(config: &Config) -> Stamp {
617620

618621
if let Some(ref rustdoc_path) = config.rustdoc_path {
619622
stamp.add_path(&rustdoc_path);
623+
// FIXME(fmease): Remove this one once the rewrite is completed.
620624
stamp.add_path(&rust_src_dir.join("src/etc/htmldocck.py"));
621625
}
622626

‎src/tools/compiletest/src/runtest.rs

+10-3
Original file line numberDiff line numberDiff line change
@@ -3030,9 +3030,16 @@ impl<'test> TestCx<'test> {
30303030
if self.props.check_test_line_numbers_match {
30313031
self.check_rustdoc_test_option(proc_res);
30323032
} else {
3033-
let root = self.config.find_rust_src_root().unwrap();
3034-
let mut cmd = Command::new(&self.config.python);
3035-
cmd.arg(root.join("src/etc/htmldocck.py")).arg(&out_dir).arg(&self.testpaths.file);
3033+
// FIXME(fmease): Temporary commented out code:
3034+
// FIXME(fmease): I don't like this unwrap!
3035+
let mut cmd = Command::new(self.config.htmldocck_path.as_ref().unwrap());
3036+
cmd.arg("--doc-dir").arg(&out_dir).arg("--template").arg(&self.testpaths.file);
3037+
3038+
// let root = self.config.find_rust_src_root().unwrap();
3039+
// let mut cmd = Command::new(&self.config.python);
3040+
// cmd.arg(root.join("src/etc/htmldocck.py"));
3041+
// cmd.arg(&out_dir).arg(&self.testpaths.file);
3042+
30363043
if self.config.bless {
30373044
cmd.arg("--bless");
30383045
}

‎src/tools/htmldocck/Cargo.toml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[package]
2+
name = "htmldocck"
3+
version = "0.1.0"
4+
description = "A test framework for rustdoc's HTML backend"
5+
edition = "2021"
6+
7+
[dependencies]
8+
getopts = "0.2"
9+
regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace
10+
shlex = "1.3.0"
11+
unicode-width = "0.1.4"

‎src/tools/htmldocck/src/cache.rs

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
use std::{
2+
collections::{hash_map::Entry, HashMap},
3+
path::Path,
4+
};
5+
6+
use crate::error::DiagCtxt;
7+
8+
pub(crate) struct Cache<'a> {
9+
root: &'a Path,
10+
// FIXME: `&'a str`s
11+
files: HashMap<String, String>,
12+
// FIXME: `&'a str`, comment what this is for -- `-`
13+
last_path: Option<String>,
14+
}
15+
16+
impl<'a> Cache<'a> {
17+
pub(crate) fn new(root: &'a Path) -> Self {
18+
Self { root, files: HashMap::new(), last_path: None }
19+
}
20+
21+
// FIXME: check file vs. dir (`@has <PATH>` vs. `@has-dir <PATH>`)
22+
/// Check if the path points to an existing entity.
23+
pub(crate) fn has(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<bool, ()> {
24+
// FIXME: should we use `try_exists` over `exists` instead? matters the most for `@!has <PATH>`.
25+
let path = self.resolve(path, dcx)?;
26+
27+
Ok(self.files.contains_key(&path) || Path::new(&path).exists())
28+
}
29+
30+
/// Load the contents of the given path.
31+
pub(crate) fn load(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<&str, ()> {
32+
let path = self.resolve(path, dcx)?;
33+
34+
Ok(match self.files.entry(path) {
35+
Entry::Occupied(entry) => entry.into_mut(),
36+
Entry::Vacant(entry) => {
37+
// FIXME: better message, location
38+
let data =
39+
std::fs::read_to_string(self.root.join(entry.key())).map_err(|error| {
40+
dcx.emit(&format!("failed to read file: {error}"), None, None)
41+
})?;
42+
entry.insert(data)
43+
}
44+
})
45+
}
46+
47+
// FIXME: &str -> &str if possible
48+
fn resolve(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<String, ()> {
49+
if path == "-" {
50+
// FIXME: no cloning
51+
return self
52+
.last_path
53+
.clone()
54+
// FIXME better diag, location
55+
.ok_or_else(|| {
56+
dcx.emit(
57+
"attempt to use `-` ('previous path') in the very first command",
58+
None,
59+
None,
60+
)
61+
});
62+
}
63+
64+
// While we could normalize the `path` at this point by
65+
// using `std::path::absolute`, it's likely not worth it.
66+
self.last_path = Some(path.clone());
67+
Ok(path)
68+
}
69+
}

‎src/tools/htmldocck/src/check.rs

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
use crate::cache::Cache;
2+
use crate::error::{DiagCtxt, Source};
3+
use crate::{Command, CommandKind};
4+
5+
impl Command<'_> {
6+
// FIXME: implement all checks!
7+
// FIXME: move regex parsing etc. into the parser maybe
8+
pub(crate) fn run(self, cache: &mut Cache<'_>, dcx: &mut DiagCtxt) -> Result<(), ()> {
9+
let result = self.kind.run(cache, self.source.clone(), dcx)?;
10+
11+
if result == self.negated {
12+
// FIXME: better diag
13+
dcx.emit("check failed", self.source, None);
14+
return Err(());
15+
}
16+
17+
Ok(())
18+
}
19+
}
20+
21+
impl CommandKind {
22+
fn run(
23+
self,
24+
cache: &mut Cache<'_>,
25+
source: Source<'_>,
26+
dcx: &mut DiagCtxt,
27+
) -> Result<bool, ()> {
28+
Ok(match self {
29+
Self::HasFile { path } => cache.has(path, dcx)?, // FIXME: check if it's actually a file
30+
Self::HasDir { path } => cache.has(path, dcx)?, // FIXME: check if it's actually a directory
31+
Self::Has { path, xpath, text } => {
32+
let _data = cache.load(path, dcx)?;
33+
_ = xpath;
34+
_ = text;
35+
true // FIXME
36+
}
37+
Self::HasRaw { path, text } => {
38+
let data = cache.load(path, dcx)?;
39+
40+
if text.is_empty() {
41+
// fast path
42+
return Ok(true);
43+
}
44+
45+
let text = channel_url::instantiate(&text, dcx)?;
46+
let text = text.replace(|c: char| c.is_ascii_whitespace(), " ");
47+
let data = data.replace(|c: char| c.is_ascii_whitespace(), " ");
48+
49+
data.contains(&text)
50+
}
51+
Self::Matches { path, xpath, pattern } => {
52+
let _data = cache.load(path, dcx)?;
53+
_ = xpath;
54+
55+
let Ok(_pattern) =
56+
regex::RegexBuilder::new(&pattern).unicode(true).build().map_err(|error| {
57+
// FIXME: better error message and location
58+
// FIXME: Use `regex_syntax` directly. Its error type exposes the
59+
// underlying span which we can then translate/offset.
60+
_ = error;
61+
dcx.emit(&format!("malformed regex"), Some(source), None)
62+
})
63+
else {
64+
return Err(());
65+
};
66+
67+
true // FIXME
68+
}
69+
Self::MatchesRaw { path, pattern } => {
70+
let data = cache.load(path, dcx)?;
71+
let pattern = channel_url::instantiate(&pattern, dcx)?;
72+
73+
if pattern.is_empty() {
74+
// fast path
75+
return Ok(true);
76+
}
77+
78+
let Ok(pattern) =
79+
regex::RegexBuilder::new(&pattern).unicode(true).build().map_err(|error| {
80+
// FIXME: better error message and location
81+
// FIXME: Use `regex_syntax` directly. Its error type exposes the
82+
// underlying span which we can then translate/offset.
83+
_ = error;
84+
dcx.emit(&format!("malformed regex"), Some(source), None)
85+
})
86+
else {
87+
return Err(());
88+
};
89+
90+
pattern.is_match(data)
91+
}
92+
Self::Count { path, xpath, text, count } => {
93+
let _data = cache.load(path, dcx)?;
94+
_ = xpath;
95+
_ = text;
96+
_ = count;
97+
true // FIXME
98+
}
99+
Self::Files { path, files } => {
100+
let _data = cache.load(path, dcx)?;
101+
_ = files;
102+
true // FIXME
103+
}
104+
Self::Snapshot { name, path, xpath } => {
105+
let _data = cache.load(path, dcx)?;
106+
_ = name;
107+
_ = path;
108+
_ = xpath;
109+
true // FIXME
110+
}
111+
})
112+
}
113+
}
114+
115+
mod channel_url {
116+
use std::{borrow::Cow, sync::OnceLock};
117+
118+
use crate::error::DiagCtxt;
119+
120+
const PLACEHOLDER: &str = "{{channel}}";
121+
122+
pub(super) fn instantiate<'a>(input: &'a str, dcx: &mut DiagCtxt) -> Result<Cow<'a, str>, ()> {
123+
let Some(channel_url) = channel_url(dcx)? else { return Ok(input.into()) };
124+
Ok(input.replace(PLACEHOLDER, channel_url).into())
125+
}
126+
127+
#[allow(dead_code)] // FIXME
128+
pub(super) fn anonymize<'a>(input: &'a str, dcx: &'_ mut DiagCtxt) -> Result<Cow<'a, str>, ()> {
129+
let Some(channel_url) = channel_url(dcx)? else { return Ok(input.into()) };
130+
Ok(input.replace(channel_url, PLACEHOLDER).into())
131+
}
132+
133+
fn channel_url(dcx: &mut DiagCtxt) -> Result<Option<&'static str>, ()> {
134+
static CHANNEL_URL: OnceLock<Option<String>> = OnceLock::new();
135+
136+
// FIXME: Use `get_or_try_init` here (instead of `get`→`set`→`get`) if/once stabilized (on beta).
137+
138+
if let Some(channel_url) = CHANNEL_URL.get() {
139+
return Ok(channel_url.as_deref());
140+
}
141+
142+
const KEY: &str = "DOC_RUST_LANG_ORG_CHANNEL";
143+
144+
let channel_url = match std::env::var(KEY) {
145+
Ok(url) => Some(url),
146+
// FIXME: should we make the channel mandatory instead?
147+
Err(std::env::VarError::NotPresent) => None,
148+
Err(std::env::VarError::NotUnicode(var)) => {
149+
// FIXME: better diag
150+
// FIXME: Use `OsStr::display` (instead of `to_string_lossy`) if/once stabilized (on beta).
151+
dcx.emit(
152+
&format!("env var `{KEY}` is not valid UTF-8: `{}`", var.to_string_lossy()),
153+
None,
154+
None,
155+
);
156+
return Err(());
157+
}
158+
};
159+
160+
// unwrap: The static item is locally scoped and no other thread tries to initialize it.
161+
CHANNEL_URL.set(channel_url).unwrap();
162+
// unwrap: Initialized above.
163+
Ok(CHANNEL_URL.get().unwrap().as_deref())
164+
}
165+
}

0 commit comments

Comments
 (0)
Please sign in to comment.