Validate forge command outputs in docs

crates/docs/src/validation.rs

@@ -1,15 +1,51 @@
 use regex::Regex;
 use std::{
+    collections::HashMap,
     env, fs, io,
     path::{Path, PathBuf},
 };
+use walkdir::WalkDir;
 
 const EXTENSION: Option<&str> = Some("md");
 
+#[derive(Clone, Debug)]
+pub struct SnippetType(String);
+
+impl SnippetType {
+    #[must_use]
+    pub fn forge() -> Self {
+        SnippetType("snforge".to_string())
+    }
+
+    #[must_use]
+    pub fn sncast() -> Self {
+        SnippetType("sncast".to_string())
+    }
+
+    #[must_use]
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    #[must_use]
+    pub fn get_re(&self) -> Regex {
+        let escaped_command = regex::escape(self.as_str());
+        let pattern = format!(
+            r"(?ms)^(?:<!--\s*(.*?)\s*-->\n)?```shell\n\$ ({escaped_command} .+?)\n```(?:\s*<details>\n<summary>Output:<\/summary>\n\n```shell\n([\s\S]+?)\n```[\s]*<\/details>)?"
+        );
+
+        Regex::new(&pattern).unwrap()
+    }
+}
+
+#[derive(Debug)]
 pub struct Snippet {
     pub command: String,
+    pub output: Option<String>,
     pub file_path: String,
     pub line_start: usize,
+    pub snippet_type: SnippetType,
+    pub ignored: bool,
 }
 
 impl Snippet {
@@ -28,33 +64,56 @@ impl Snippet {
             .map(|arg| arg.trim().to_string())
             .collect()
     }
+
+    #[must_use]
+    pub fn capture_package_from_output(&self) -> Option<String> {
+        let re =
+            Regex::new(r"Collected \d+ test\(s\) from ([a-zA-Z_][a-zA-Z0-9_]*) package").unwrap();
+
+        re.captures_iter(self.output.as_ref()?)
+            .filter_map(|caps| caps.get(1))
+            .last()
+            .map(|m| m.as_str().to_string())
+    }
 }
 
-pub fn extract_snippets_from_file(file_path: &Path, re: &Regex) -> io::Result<Vec<Snippet>> {
+pub fn extract_snippets_from_file(
+    file_path: &Path,
+    snippet_type: &SnippetType,
+) -> io::Result<Vec<Snippet>> {
     let content = fs::read_to_string(file_path)?;
     let file_path_str = file_path
         .to_str()
         .expect("Failed to get file path")
         .to_string();
 
-    let snippets = re
+    let snippets = snippet_type
+        .get_re()
         .captures_iter(&content)
         .filter_map(|caps| {
-            let command_match = caps.get(1)?;
+            let command_match = caps.get(2)?;
             let match_start = caps.get(0)?.start();
+            let output = caps.get(3).map(|m| m.as_str().to_string());
+            let ignored = caps.get(1).map_or(false, |m| m.as_str().contains("ignore"));
 
             Some(Snippet {
                 command: command_match.as_str().to_string(),
+                output,
                 file_path: file_path_str.clone(),
                 line_start: content[..match_start].lines().count() + 1,
+                snippet_type: snippet_type.clone(),
+                ignored,
             })
         })
         .collect();
 
     Ok(snippets)
 }
 
-pub fn extract_snippets_from_directory(dir_path: &Path, re: &Regex) -> io::Result<Vec<Snippet>> {
+pub fn extract_snippets_from_directory(
+    dir_path: &Path,
+    snippet_type: &SnippetType,
+) -> io::Result<Vec<Snippet>> {
     let mut all_snippets = Vec::new();
 
     let files = walkdir::WalkDir::new(dir_path)
@@ -68,7 +127,7 @@ pub fn extract_snippets_from_directory(dir_path: &Path, re: &Regex) -> io::Resul
         if EXTENSION.map_or(true, |ext| {
             path.extension().and_then(|path_ext| path_ext.to_str()) == Some(ext)
         }) {
-            let snippets = extract_snippets_from_file(path, re)?;
+            let snippets = extract_snippets_from_file(path, snippet_type)?;
             all_snippets.extend(snippets);
         }
     }
@@ -90,26 +149,75 @@ pub fn get_parent_dir(levels_up: usize) -> PathBuf {
     dir
 }
 
-pub fn assert_valid_snippet(
-    condition: bool,
-    snippet: &Snippet,
-    tool_name: &str,
-    err_message: &str,
-) {
+pub fn assert_valid_snippet(condition: bool, snippet: &Snippet, err_message: &str) {
     assert!(
         condition,
-        "Found invalid {} snippet in the docs in file: {} at line {}\n{}",
-        tool_name, snippet.file_path, snippet.line_start, err_message
+        "Found invalid {} snippet in the docs in at {}:{}:1\n{}",
+        snippet.snippet_type.as_str(),
+        snippet.file_path,
+        snippet.line_start,
+        err_message
     );
 }
 
 pub fn print_success_message(snippets_len: usize, tool_name: &str) {
     println!("Successfully validated {snippets_len} {tool_name} docs snippets");
 }
 
-pub fn print_skipped_snippet_message(snippet: &Snippet, tool_name: &str) {
+pub fn print_skipped_snippet_message(snippet: &Snippet) {
     println!(
         "Skipped validation of {} snippet in the docs in file: {} at line {}",
-        tool_name, snippet.file_path, snippet.line_start
+        snippet.snippet_type.as_str(),
+        snippet.file_path,
+        snippet.line_start,
     );
 }
+
+pub fn create_listings_to_packages_mapping() -> HashMap<String, Vec<String>> {
+    let docs_listings_path = "../../docs/listings";
+    let mut mapping = HashMap::new();
+
+    for listing in WalkDir::new(docs_listings_path)
+        .min_depth(1)
+        .max_depth(1)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| e.file_type().is_dir())
+    {
+        let listing_path = listing.path();
+        let crates_dir = listing_path.join("crates");
+
+        if crates_dir.is_dir() {
+            let packages = list_packages_in_directory(&crates_dir);
+            if let Some(listing_name) = listing_path.file_name().and_then(|x| x.to_str()) {
+                if !packages.is_empty() {
+                    mapping.insert(listing_name.to_string(), packages);
+                }
+            }
+        }
+    }
+
+    mapping
+}
+
+fn list_packages_in_directory(dir_path: &Path) -> Vec<String> {
+    let crates_path = dir_path.to_owned();
+
+    if crates_path.exists() && crates_path.is_dir() {
+        WalkDir::new(crates_path)
+            .min_depth(1)
+            .max_depth(1)
+            .into_iter()
+            .filter_map(Result::ok)
+            .filter(|e| e.path().is_dir())
+            .filter_map(|e| {
+                e.path()
+                    .file_name()
+                    .and_then(|name| name.to_str())
+                    .map(String::from)
+            })
+            .collect()
+    } else {
+        Vec::new()
+    }
+}

crates/forge/tests/e2e/common/runner.rs

@@ -5,6 +5,7 @@ use indoc::formatdoc;
 use shared::command::CommandExt;
 use shared::test_utils::node_url::node_rpc_url;
 use snapbox::cmd::{cargo_bin, Command as SnapboxCommand};
+use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str::FromStr;
@@ -34,12 +35,17 @@ pub(crate) fn test_runner(temp_dir: &TempDir) -> SnapboxCommand {
 pub(crate) static BASE_FILE_PATTERNS: &[&str] = &["**/*.cairo", "**/*.toml"];
 
 pub(crate) fn setup_package_with_file_patterns(
-    package_name: &str,
+    package_path: &str,
     file_patterns: &[&str],
 ) -> TempDir {
     let temp = tempdir_with_tool_versions().unwrap();
-    temp.copy_from(format!("tests/data/{package_name}"), file_patterns)
-        .unwrap();
+    let package_path = Utf8PathBuf::from_str(package_path)
+        .unwrap()
+        .canonicalize_utf8()
+        .unwrap()
+        .to_string()
+        .replace('\\', "/");
+    temp.copy_from(package_path.clone(), file_patterns).unwrap();
 
     let snforge_std_path = Utf8PathBuf::from_str("../../snforge_std")
         .unwrap()
@@ -60,6 +66,12 @@ pub(crate) fn setup_package_with_file_patterns(
         value(get_assert_macros_version().unwrap().to_string());
     scarb_toml["target.starknet-contract"]["sierra"] = value(true);
 
+    if package_path.contains("docs/listings") {
+        scarb_toml["dev-dependencies"]["snforge_std"]
+            .as_table_mut()
+            .and_then(|snforge_std| snforge_std.remove("workspace"));
+    }
+
     manifest_path.write_str(&scarb_toml.to_string()).unwrap();
 
     // TODO (#2074): do that on .cairo.template files only
@@ -69,7 +81,29 @@ pub(crate) fn setup_package_with_file_patterns(
 }
 
 pub(crate) fn setup_package(package_name: &str) -> TempDir {
-    setup_package_with_file_patterns(package_name, BASE_FILE_PATTERNS)
+    let package_path = "tests/data/".to_string() + package_name;
+    setup_package_with_file_patterns(&package_path, BASE_FILE_PATTERNS)
+}
+
+fn get_listing_name(
+    package: &str,
+    packages_mapping: &HashMap<String, Vec<String>>,
+) -> Option<String> {
+    packages_mapping
+        .iter()
+        .find(|(_, packages)| packages.contains(&package.to_string()))
+        .map(|(listing_name, _)| listing_name.clone())
+}
+
+pub(crate) fn setup_package_from_docs_listings(
+    package_name: &str,
+    packages_mapping: &HashMap<String, Vec<String>>,
+) -> TempDir {
+    let listing_name = get_listing_name(package_name, packages_mapping)
+        .unwrap_or_else(|| panic!("Couldn't find listing for package {package_name}"));
+    let package_path = format!("../../docs/listings/{listing_name}/crates/{package_name}",);
+
+    setup_package_with_file_patterns(&package_path, BASE_FILE_PATTERNS)
 }
 
 fn replace_node_rpc_url_placeholders(dir_path: &Path) {

crates/forge/tests/e2e/docs_snippets_validation.rs

@@ -1,43 +1,91 @@
+use std::collections::HashMap;
+
+use assert_fs::TempDir;
 use clap::Parser;
 use docs::validation::{
-    assert_valid_snippet, extract_snippets_from_directory, get_parent_dir,
-    print_skipped_snippet_message, print_success_message,
+    assert_valid_snippet, create_listings_to_packages_mapping, extract_snippets_from_directory,
+    get_parent_dir, print_skipped_snippet_message, print_success_message, SnippetType,
 };
 use forge::Cli;
-use regex::Regex;
+use shared::test_utils::output_assert::assert_stdout_contains;
+
+use super::common::runner::{
+    setup_hello_workspace, setup_package, setup_package_from_docs_listings, test_runner,
+};
+
+fn is_package_from_docs_listings(
+    package: &str,
+    listings_to_packages_mapping: &HashMap<String, Vec<String>>,
+) -> bool {
+    for packages in listings_to_packages_mapping.values() {
+        if packages.contains(&package.to_string()) {
+            return true;
+        }
+    }
+    false
+}
+
 #[test]
 fn test_docs_snippets() {
+    let listings_to_packages_mapping = create_listings_to_packages_mapping();
+
     let root_dir = get_parent_dir(2);
     let docs_dir = root_dir.join("docs/src");
 
-    let re = Regex::new(r"(?ms)```shell\n\$ (snforge .+?)\n```").expect("Invalid regex pattern");
-
-    let snippets = extract_snippets_from_directory(&docs_dir, &re)
-        .expect("Failed to extract snforge command snippets");
+    let snippet_type = SnippetType::forge();
 
-    // TODO(#2684)
-    let skipped_args = [
-        // for some reason `try_parse_from` fails on `--version` flag
-        vec!["snforge", "--version"],
-    ];
+    let snippets = extract_snippets_from_directory(&docs_dir, &snippet_type)
+        .expect("Failed to extract command snippets");
 
     for snippet in &snippets {
         let args = snippet.to_command_args();
-        let args: Vec<&str> = args.iter().map(String::as_str).collect();
+        let mut args: Vec<&str> = args.iter().map(String::as_str).collect();
 
-        if skipped_args.contains(&args) {
-            print_skipped_snippet_message(snippet, "snforge");
+        if snippet.ignored {
+            print_skipped_snippet_message(snippet);
             continue;
         }
 
-        let parse_result = Cli::try_parse_from(args);
+        let parse_result = Cli::try_parse_from(args.clone());
         let err_message = if let Err(err) = &parse_result {
             err.to_string()
         } else {
             String::new()
         };
-        assert_valid_snippet(parse_result.is_ok(), snippet, "snforge", &err_message);
+
+        assert_valid_snippet(parse_result.is_ok(), snippet, &err_message);
+
+        // Remove "snforge" from the args
+        args.remove(0);
+
+        // Remove "test" from the args
+        args.retain(|element| element != &"test");
+
+        if let Some(snippet_output) = &snippet.output {
+            let package_name = snippet
+                .capture_package_from_output()
+                .expect("Failed to capture package from command output");
+
+            // TODO(#2698)
+            let temp = resolve_temp_dir(&package_name, &listings_to_packages_mapping);
+            let output = test_runner(&temp).args(args).assert();
+
+            assert_stdout_contains(output, snippet_output);
+        }
     }
 
-    print_success_message(snippets.len(), "snforge");
+    print_success_message(snippets.len(), snippet_type.as_str());
+}
+
+fn resolve_temp_dir(
+    package_name: &str,
+    listings_to_packages_mapping: &HashMap<String, Vec<String>>,
+) -> TempDir {
+    if is_package_from_docs_listings(package_name, listings_to_packages_mapping) {
+        setup_package_from_docs_listings(package_name, listings_to_packages_mapping)
+    } else if ["addition", "fibonacci", "hello_workspaces"].contains(&package_name) {
+        setup_hello_workspace()
+    } else {
+        setup_package(package_name)
+    }
 }