ci(ast_tools): run "AST Changes" CI task when dependency crates change (#15603)

#15565 made `oxc_ast_tools` depend on local workspace versions of various crates. Add those crates to the watch list for "AST Changes" CI task, so if changes are made in them, `ast_tools` codegen will run to ensure generated code has been updated to reflect those changes.

In particular, this will prevent generated code getting out of sync when changes to minifier affect the generated code.

Author: overlookmotel

.github/generated/ast_changes_watch_list.yml

@@ -12,66 +12,27 @@ src:
   - 'apps/oxlint/src-js/generated/visitor.d.ts'
   - 'apps/oxlint/src-js/generated/walk.js'
   - 'apps/oxlint/src/generated/raw_transfer_constants.rs'
-  - 'crates/oxc_allocator/src/generated/assert_layouts.rs'
-  - 'crates/oxc_allocator/src/generated/fixed_size_constants.rs'
-  - 'crates/oxc_allocator/src/pool/fixed_size.rs'
-  - 'crates/oxc_ast/src/ast/comment.rs'
-  - 'crates/oxc_ast/src/ast/js.rs'
-  - 'crates/oxc_ast/src/ast/jsx.rs'
-  - 'crates/oxc_ast/src/ast/literal.rs'
-  - 'crates/oxc_ast/src/ast/ts.rs'
-  - 'crates/oxc_ast/src/generated/assert_layouts.rs'
-  - 'crates/oxc_ast/src/generated/ast_builder.rs'
-  - 'crates/oxc_ast/src/generated/ast_kind.rs'
-  - 'crates/oxc_ast/src/generated/derive_clone_in.rs'
-  - 'crates/oxc_ast/src/generated/derive_content_eq.rs'
-  - 'crates/oxc_ast/src/generated/derive_dummy.rs'
-  - 'crates/oxc_ast/src/generated/derive_estree.rs'
-  - 'crates/oxc_ast/src/generated/derive_get_address.rs'
-  - 'crates/oxc_ast/src/generated/derive_get_span.rs'
-  - 'crates/oxc_ast/src/generated/derive_get_span_mut.rs'
-  - 'crates/oxc_ast/src/generated/derive_take_in.rs'
-  - 'crates/oxc_ast/src/generated/get_id.rs'
-  - 'crates/oxc_ast/src/serialize/basic.rs'
-  - 'crates/oxc_ast/src/serialize/js.rs'
-  - 'crates/oxc_ast/src/serialize/jsx.rs'
-  - 'crates/oxc_ast/src/serialize/literal.rs'
-  - 'crates/oxc_ast/src/serialize/mod.rs'
-  - 'crates/oxc_ast/src/serialize/ts.rs'
+  - 'crates/oxc_allocator/src/**'
+  - 'crates/oxc_ast/src/**'
   - 'crates/oxc_ast_macros/src/generated/derived_traits.rs'
   - 'crates/oxc_ast_macros/src/generated/structs.rs'
   - 'crates/oxc_ast_macros/src/lib.rs'
-  - 'crates/oxc_ast_visit/src/generated/utf8_to_utf16_converter.rs'
-  - 'crates/oxc_ast_visit/src/generated/visit.rs'
-  - 'crates/oxc_ast_visit/src/generated/visit_mut.rs'
+  - 'crates/oxc_ast_visit/src/**'
+  - 'crates/oxc_codegen/src/**'
+  - 'crates/oxc_data_structures/src/**'
   - 'crates/oxc_formatter/src/ast_nodes/generated/ast_nodes.rs'
   - 'crates/oxc_formatter/src/ast_nodes/generated/format.rs'
   - 'crates/oxc_linter/src/generated/assert_layouts.rs'
   - 'crates/oxc_linter/src/lib.rs'
+  - 'crates/oxc_minifier/src/**'
+  - 'crates/oxc_parser/src/**'
   - 'crates/oxc_regular_expression/src/ast.rs'
   - 'crates/oxc_regular_expression/src/generated/assert_layouts.rs'
   - 'crates/oxc_regular_expression/src/generated/derive_clone_in.rs'
   - 'crates/oxc_regular_expression/src/generated/derive_content_eq.rs'
   - 'crates/oxc_regular_expression/src/generated/derive_get_address.rs'
-  - 'crates/oxc_span/src/generated/assert_layouts.rs'
-  - 'crates/oxc_span/src/generated/derive_dummy.rs'
-  - 'crates/oxc_span/src/generated/derive_estree.rs'
-  - 'crates/oxc_span/src/source_type.rs'
-  - 'crates/oxc_span/src/span.rs'
-  - 'crates/oxc_syntax/src/comment_node.rs'
-  - 'crates/oxc_syntax/src/generated/assert_layouts.rs'
-  - 'crates/oxc_syntax/src/generated/derive_clone_in.rs'
-  - 'crates/oxc_syntax/src/generated/derive_content_eq.rs'
-  - 'crates/oxc_syntax/src/generated/derive_dummy.rs'
-  - 'crates/oxc_syntax/src/generated/derive_estree.rs'
-  - 'crates/oxc_syntax/src/lib.rs'
-  - 'crates/oxc_syntax/src/module_record.rs'
-  - 'crates/oxc_syntax/src/number.rs'
-  - 'crates/oxc_syntax/src/operator.rs'
-  - 'crates/oxc_syntax/src/reference.rs'
-  - 'crates/oxc_syntax/src/scope.rs'
-  - 'crates/oxc_syntax/src/serialize.rs'
-  - 'crates/oxc_syntax/src/symbol.rs'
+  - 'crates/oxc_span/src/**'
+  - 'crates/oxc_syntax/src/**'
   - 'crates/oxc_traverse/src/generated/scopes_collector.rs'
   - 'napi/parser/generated/constants.js'
   - 'napi/parser/generated/deserialize/js.js'

Cargo.lock

@@ -225,6 +225,7 @@ similar-asserts = "1.7.0" # Test diff assertions
 smallvec = { version = "1.15.1", features = ["union"] } # Stack-allocated vectors
 tempfile = "3.23.0" # Temporary files
 tokio = { version = "1.48.0", default-features = false } # Async runtime
+toml = { version = "0.9.8" }
 tower-lsp-server = "0.22.1" # LSP server framework
 tracing-subscriber = "0.3.20" # Tracing implementation
 ureq = { version = "3.1.4", default-features = false } # HTTP client

Cargo.toml

@@ -48,6 +48,7 @@ rustc-hash = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 syn = { workspace = true, features = ["clone-impls", "derive", "extra-traits", "full", "parsing", "printing", "proc-macro"] }
+toml = { workspace = true }
 
 [features]
 default = ["generate-js"]

tasks/ast_tools/Cargo.toml

@@ -367,7 +367,7 @@ fn main() {
     // Add CI filter file to outputs.
     // Skip this step if JS generators are disabled, because not all files are generated.
     #[cfg(feature = "generate-js")]
-    outputs.push(generate_ci_filter(&outputs));
+    outputs.push(generate_ci_filter(&outputs, &codegen));
 
     // Write outputs to disk
     if !options.dry_run {
@@ -383,12 +383,12 @@ fn main() {
 /// unless relevant files have changed.
 ///
 /// List includes source files, generated files, and all files in `oxc_ast_tools` itself.
-fn generate_ci_filter(outputs: &[RawOutput]) -> RawOutput {
+fn generate_ci_filter(outputs: &[RawOutput], codegen: &Codegen) -> RawOutput {
     log!("Generate CI filter... ");
 
     let paths =
         SOURCE_PATHS.iter().copied().chain(outputs.iter().map(|output| output.path.as_str()));
-    let output = Output::yaml_watch_list(AST_CHANGES_WATCH_LIST_PATH, paths);
+    let output = Output::yaml_watch_list(AST_CHANGES_WATCH_LIST_PATH, paths, codegen);
 
     log_success!();
 

tasks/ast_tools/src/main.rs

@@ -1,4 +1,8 @@
-use std::fmt::Write;
+use std::{fmt::Write, fs};
+
+use rustc_hash::FxHashSet;
+
+use crate::Codegen;
 
 use super::{Output, add_header};
 
@@ -10,11 +14,52 @@ pub fn print_yaml(code: &str, generator_path: &str) -> String {
 impl Output {
     /// Generate a watch list YAML file.
     ///
-    /// The path of the watch list itself, `tasks/ast_tools/src/**`, and the CI workflow file are added to the list.
+    /// The following are added to the list:
+    /// * The watch list file itself
+    /// * `ast_tools` crate
+    /// * `oxc_*` dependencies of `ast_tools`
+    /// * CI workflow file
     pub fn yaml_watch_list<'s>(
         watch_list_path: &'s str,
         paths: impl IntoIterator<Item = &'s str>,
+        codegen: &Codegen,
     ) -> Self {
+        let mut paths = paths.into_iter().collect::<Vec<_>>();
+
+        // Get `oxc_*` dependencies of `ast_tools`.
+        // `ast_tools` uses these crates, so generated code may change if these crates are changed.
+        let cargo_toml = parse_toml("tasks/ast_tools/Cargo.toml", codegen);
+        let dependency_crates = cargo_toml
+            .get("dependencies")
+            .and_then(|v| v.as_table())
+            .unwrap()
+            .keys()
+            .map(String::as_str)
+            .filter(|krate| {
+                // Exclude crates which are not in this monorepo e.g. `oxc_index`
+                krate.starts_with("oxc_") && codegen.root_path().join("crates").join(krate).is_dir()
+            })
+            .collect::<FxHashSet<_>>();
+
+        // Remove paths from `paths` which are in `src` dir of a dependency crate
+        // (as `crate/{krate}/src/**` pattern will cover them anyway)
+        paths.retain(|path| {
+            if let Some(path) = path.strip_prefix("crates/")
+                && let Some((krate, path)) = path.split_once('/')
+                && path.starts_with("src/")
+            {
+                return !dependency_crates.contains(krate);
+            }
+            true
+        });
+
+        // Get paths for dependency crates
+        let dependency_crate_paths = dependency_crates
+            .into_iter()
+            .map(|krate| format!("crates/{krate}/src/**"))
+            .collect::<Vec<_>>();
+
+        // Additional paths
         let additional_paths = [
             // This watch list file
             watch_list_path,
@@ -24,9 +69,13 @@ impl Output {
             ".github/workflows/ci.yml",
         ];
 
-        let mut paths = paths.into_iter().chain(additional_paths).collect::<Vec<_>>();
+        // Add additional paths and dependency crate paths to `paths`, and sort
+        paths.extend(
+            additional_paths.into_iter().chain(dependency_crate_paths.iter().map(String::as_str)),
+        );
         paths.sort_unstable();
 
+        // Generate YAML
         let mut code = "src:\n".to_string();
         for path in paths {
             writeln!(code, "  - '{path}'").unwrap();
@@ -35,3 +84,9 @@ impl Output {
         Self::Yaml { path: watch_list_path.to_string(), code }
     }
 }
+
+/// Parse TOML file.
+fn parse_toml(path: &str, codegen: &Codegen) -> toml::Table {
+    let toml_content = fs::read_to_string(codegen.root_path().join(path)).unwrap();
+    toml::from_str(&toml_content).unwrap()
+}