Add translation support by mdbook-i18n-helpers to bootstrap

dalance · dalance · commit 366dd213730e · 2024-05-21T12:48:57.000+09:00
diff --git a/Cargo.lock b/Cargo.lock
@@ -348,7 +348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
 dependencies = [
  "memchr",
- "regex-automata 0.3.9",
+ "regex-automata 0.3.7",
  "serde",
 ]
 
@@ -2159,6 +2159,21 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "line-wrap"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e"
+
+[[package]]
+name = "linereader"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d921fea6860357575519aca014c6e22470585accdd543b370c404a8a72d0dd1d"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "linkchecker"
 version = "0.1.0"
@@ -2167,6 +2182,12 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
 [[package]]
 name = "lint-docs"
 version = "0.1.0"
@@ -2317,6 +2338,24 @@ dependencies = [
  "topological-sort",
 ]
 
+[[package]]
+name = "mdbook-i18n-helpers"
+version = "0.3.2"
+source = "git+https://github.com/dalance/mdbook-i18n-helpers?branch=export_preprocessor#c2c5251481ed779f44b1abe4238916209f995cc8"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "mdbook",
+ "polib",
+ "pulldown-cmark 0.9.6",
+ "pulldown-cmark-to-cmark",
+ "regex",
+ "semver",
+ "serde_json",
+ "syntect",
+ "textwrap",
+]
+
 [[package]]
 name = "measureme"
 version = "11.0.1"
@@ -2576,6 +2615,28 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "onig"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "opener"
 version = "0.6.1"
@@ -2894,6 +2955,29 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
+[[package]]
+name = "plist"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9d34169e64b3c7a80c8621a48adaf44e0cf62c78a9b25dd9dd35f1881a17cf9"
+dependencies = [
+ "base64",
+ "indexmap",
+ "line-wrap",
+ "quick-xml",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "polib"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b393b155cf9be86249cba1b56cc81be0e6212c66d94ac0d76d37a1761f3bb1b"
+dependencies = [
+ "linereader",
+]
+
 [[package]]
 name = "polonius-engine"
 version = "0.13.0"
@@ -3009,12 +3093,30 @@ version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3"
 
+[[package]]
+name = "pulldown-cmark-to-cmark"
+version = "11.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dd464f32d7631035e849fcd969a603e9bb17ceaebe8467352a7728147f34e42"
+dependencies = [
+ "pulldown-cmark 0.9.6",
+]
+
 [[package]]
 name = "punycode"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe"
 
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quine-mc_cluskey"
 version = "0.2.4"
@@ -3150,12 +3252,13 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.8.4"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
 dependencies = [
  "aho-corasick",
  "memchr",
+ "regex-automata 0.3.7",
  "regex-syntax 0.7.5",
 ]
 
@@ -3179,9 +3282,14 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.9"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
+checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.7.5",
+]
 
 [[package]]
 name = "regex-lite"
@@ -3256,6 +3364,7 @@ dependencies = [
  "clap",
  "env_logger",
  "mdbook",
+ "mdbook-i18n-helpers",
 ]
 
 [[package]]
@@ -5256,6 +5365,28 @@ dependencies = [
  "syn 2.0.64",
 ]
 
+[[package]]
+name = "syntect"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"
+dependencies = [
+ "bincode",
+ "bitflags 1.3.2",
+ "flate2",
+ "fnv",
+ "once_cell",
+ "onig",
+ "plist",
+ "regex-syntax 0.8.3",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "walkdir",
+ "yaml-rust",
+]
+
 [[package]]
 name = "sysinfo"
 version = "0.30.12"
@@ -5375,6 +5506,12 @@ dependencies = [
  "std",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
+
 [[package]]
 name = "thin-vec"
 version = "0.2.13"
@@ -6305,6 +6442,15 @@ dependencies = [
  "lzma-sys",
 ]
 
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "yansi-term"
 version = "0.1.2"
diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
@@ -29,7 +29,7 @@ macro_rules! submodule_helper {
 }
 
 macro_rules! book {
-    ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
+    ($($name:ident, $path:expr, $book_name:expr, $lang:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
         $(
             #[derive(Debug, Clone, Hash, PartialEq, Eq)]
         pub struct $name {
@@ -61,6 +61,7 @@ macro_rules! book {
                     name: $book_name.to_owned(),
                     src: builder.src.join($path),
                     parent: Some(self),
+                    languages: $lang.into(),
                 })
             }
         }
@@ -74,15 +75,15 @@ macro_rules! book {
 // FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules
 // and checking against it?).
 book!(
-    CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo";
-    ClippyBook, "src/tools/clippy/book", "clippy";
-    EditionGuide, "src/doc/edition-guide", "edition-guide", submodule;
-    EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule;
-    Nomicon, "src/doc/nomicon", "nomicon", submodule;
-    Reference, "src/doc/reference", "reference", submodule;
-    RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule;
-    RustdocBook, "src/doc/rustdoc", "rustdoc";
-    StyleGuide, "src/doc/style-guide", "style-guide";
+    CargoBook, "src/tools/cargo/src/doc", "cargo", &[], submodule = "src/tools/cargo";
+    ClippyBook, "src/tools/clippy/book", "clippy", &[];
+    EditionGuide, "src/doc/edition-guide", "edition-guide", &[], submodule;
+    EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[], submodule;
+    Nomicon, "src/doc/nomicon", "nomicon", &[], submodule;
+    Reference, "src/doc/reference", "reference", &[], submodule;
+    RustByExample, "src/doc/rust-by-example", "rust-by-example", &["ja"], submodule;
+    RustdocBook, "src/doc/rustdoc", "rustdoc", &[];
+    StyleGuide, "src/doc/style-guide", "style-guide", &[];
 );
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
@@ -110,6 +111,7 @@ impl Step for UnstableBook {
             name: "unstable-book".to_owned(),
             src: builder.md_doc_out(self.target).join("unstable-book"),
             parent: Some(self),
+            languages: vec![],
         })
     }
 }
@@ -120,6 +122,7 @@ struct RustbookSrc<P: Step> {
     name: String,
     src: PathBuf,
     parent: Option<P>,
+    languages: Vec<&'static str>,
 }
 
 impl<P: Step> Step for RustbookSrc<P> {
@@ -151,7 +154,19 @@ impl<P: Step> Step for RustbookSrc<P> {
             builder.info(&format!("Rustbook ({target}) - {name}"));
             let _ = fs::remove_dir_all(&out);
 
-            builder.run(rustbook_cmd.arg("build").arg(src).arg("-d").arg(out));
+            builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out));
+
+            for lang in &self.languages {
+                let out = out.join(lang);
+
+                builder.info(&format!("Rustbook ({target}) - {name} - {lang}"));
+                let _ = fs::remove_dir_all(&out);
+
+                let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+                builder.run(
+                    rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out).arg("-l").arg(lang),
+                );
+            }
         }
 
         if self.parent.is_some() {
@@ -214,6 +229,7 @@ impl Step for TheBook {
             name: "book".to_owned(),
             src: absolute_path.clone(),
             parent: Some(self),
+            languages: vec![],
         });
 
         // building older edition redirects
@@ -225,6 +241,7 @@ impl Step for TheBook {
                 // There should only be one book that is marked as the parent for each target, so
                 // treat the other editions as not having a parent.
                 parent: Option::<Self>::None,
+                languages: vec![],
             });
         }
 
@@ -1200,6 +1217,7 @@ impl Step for RustcBook {
             name: "rustc".to_owned(),
             src: out_base,
             parent: Some(self),
+            languages: vec![],
         });
     }
 }
diff --git a/src/tools/rustbook/Cargo.toml b/src/tools/rustbook/Cargo.toml
@@ -7,6 +7,10 @@ edition = "2021"
 [dependencies]
 clap = "4.0.32"
 env_logger = "0.11"
+# Some modifications to mdbook-i18n-helpers are required for this PR.
+# So after checking this draft PR is acceptable, I'll send a PR to the original mdbook-i18n-helpers.
+# After the PR is merged and a new version is released, I'll change this entry.
+mdbook-i18n-helpers = {git = "https://github.com/dalance/mdbook-i18n-helpers", branch = "export_preprocessor"}
 
 [dependencies.mdbook]
 version = "0.4.37"
diff --git a/src/tools/rustbook/src/main.rs b/src/tools/rustbook/src/main.rs
