A94 Amber Docs generator (#62)

* feat: add new syntax entity - comment doc

* feat: add comment detection to function declarations

* feat: add docs generating + cli capability

* fix: adjust cli help interface to include docs generation

* fix(test): remove test files

* fix(linting): fix linting errors

* feat(declaration): generate function signature

* feat(docs): update contributing guide

* feat(docs-gen): improve docs generating module

* Update src/compiler.rs

Co-authored-by: Hubert Jabłoński <hubik080@gmail.com>

* feat(cli): remove old CLI module

* refactor(main): moved different CLI parts to separate functions

---------

Co-authored-by: Hubert Jabłoński <hubik080@gmail.com>

.gitignore

@@ -3,6 +3,8 @@
 # Test files
 test*.ab
 test*.sh
+test*.md
+/docs/
 
 # Flamegraph files
 flamegraph.svg
@@ -12,4 +14,4 @@ flamegraph.svg
 
 # Nixos
 .direnv
-result
+/result

CONTRIBUTING.md

@@ -27,7 +27,7 @@ Along the way, you may need help with your code. The best way to ask is in [our
 Amber consists of the following layers:
 
 1. [CLI Interface](#1-cli-interface)
-2. [Compiler](#2-compiler)  
+2. [Compiler](#2-compiler)
    1. [Parser & tokenizer](#21-parser--tokenizer)
    2. [Translator](#22-translator)
    2. [Built-in](#23-built-in-creation)
@@ -97,7 +97,7 @@ fn translate() -> String {
 
 Basically, the `translate()` method should return a `String` for the compiler to construct a compiled file from all of them. If it translates to nothing, you should output an empty string, like `String::new()`
 
-#### 2.3. Built-in creation 
+#### 2.3. Built-in creation
 
 In this guide we will see how to create a basic built-in function that in Amber syntax presents like:
 ```sh
@@ -125,6 +125,8 @@ use crate::translate::module::TranslateModule;
 // - `ParserMetadata` - it carries the necessary information about the current parsing context such as variables and functions that were declared up to this point, warning messages aggregated up to this point, information whether this syntax is declared in a loop, function, main block, unsafe scope etc.
 // `TranslateMetadata` - it carries the necessary information for translation such as wether we are in a silent scope, in an eval context or what indentation should be used.
 use crate::utils::{ParserMetadata, TranslateMetadata};
+// Documentation module tells compiler what markdown content should it generate for this syntax module. This is irrelevent to our simple module so we will just return empty string.
+use crate::docs::module::DocumentationModule;
 
 // This is a declaration of your built-in. Set the name accordingly.
 #[derive(Debug, Clone)]
@@ -169,6 +171,13 @@ impl TranslateModule for Builtin {
         format!("echo {}", value)
     }
 }
+
+// Here we implement what should documentation generation render (in markdown format) when encounters this syntax module. Since this is just a simple built-in that does not need to be documented, we simply return an empty String.
+impl DocumentationModule for Expr {
+    fn document(&self, _meta: &ParserMetadata) -> String {
+        String::new()
+    }
+}
 ```
 
 Now let's import it in the main module for built-ins `src/modules/builtin/mod.rs`
@@ -199,7 +208,7 @@ impl Statement {
         Builtin,
         // ...
     }
-    
+
     // ...
 }
 ```
@@ -213,7 +222,7 @@ impl Statement {
 ### 4. Tests
 Amber uses `cargo test` for tests. `stdlib` and `validity` tests usually work by executing amber code and checking its output.
 
-We have [`validity tests`](src/tests/validity.rs) to check if the compiler outputs a valid bash code, [`stdlib tests`](src/tests/stdlib.rs) and [`CLI tests`](src/tests/cli.rs).  
+We have [`validity tests`](src/tests/validity.rs) to check if the compiler outputs a valid bash code, [`stdlib tests`](src/tests/stdlib.rs) and [`CLI tests`](src/tests/cli.rs).
 
 The majority of `stdlib` tests are Written in pure Amber in the folder [`tests/stdlib`](src/tests/stdlib). For every test there is a `*.output.txt` file that contains the expected output.
 Tests will be executed without recompilation. Amber will load the scripts and verify the output in the designated file to determine if the test passes.
@@ -240,4 +249,3 @@ To run ALL tests, run `cargo test`.
 If you want to run only tests from a specific file, let's say from [`stdlib.rs`](src/tests/stdlib.rs), you add the file name to the command: `cargo test stdlib`
 
 And if there is a specific function, like `test_function()` in `stdlib.rs`, you should add the full path to it: `cargo test stdlib::test_function`
-

src/compiler.rs

@@ -1,14 +1,19 @@
 extern crate chrono;
 use chrono::prelude::*;
+use crate::docs::module::DocumentationModule;
 use itertools::Itertools;
 use crate::modules::block::Block;
 use crate::rules;
 use crate::translate::check_all_blocks;
 use crate::translate::module::TranslateModule;
 use crate::utils::{ParserMetadata, TranslateMetadata};
+use std::fs::File;
+use std::io::Write;
 use colored::Colorize;
 use heraclitus_compiler::prelude::*;
 use std::env;
+use std::fs;
+use std::path::PathBuf;
 use std::process::{Command, ExitStatus};
 use std::time::Instant;
 
@@ -81,9 +86,10 @@ impl AmberCompiler {
         }
     }
 
-    pub fn parse(&self, tokens: Vec<Token>) -> Result<(Block, ParserMetadata), Message> {
+    pub fn parse(&self, tokens: Vec<Token>, is_docs_gen: bool) -> Result<(Block, ParserMetadata), Message> {
         let code = self.cc.code.as_ref().expect(NO_CODE_PROVIDED).clone();
         let mut meta = ParserMetadata::new(tokens, self.path.clone(), Some(code));
+        meta.is_docs_gen = is_docs_gen;
         if let Err(Failure::Loud(err)) = check_all_blocks(&meta) {
             return Err(err);
         }
@@ -110,22 +116,32 @@ impl AmberCompiler {
         }
     }
 
-    pub fn translate(&self, block: Block, meta: ParserMetadata) -> String {
+    pub fn get_sorted_ast_forest(&self, block: Block, meta: &ParserMetadata) -> Vec<(String, Block)> {
         let imports_sorted = meta.import_cache.topological_sort();
         let imports_blocks = meta
             .import_cache
             .files
             .iter()
-            .map(|file| file.metadata.as_ref().map(|meta| meta.block.clone()))
-            .collect::<Vec<Option<Block>>>();
-        let mut meta = TranslateMetadata::new(meta);
+            .map(|file| file.metadata.as_ref().map(|meta| (file.path.clone(), meta.block.clone())))
+            .collect::<Vec<Option<(String, Block)>>>();
         let mut result = vec![];
-        let time = Instant::now();
         for index in imports_sorted.iter() {
-            if let Some(block) = imports_blocks[*index].clone() {
-                result.push(block.translate(&mut meta));
+            if let Some((path, block)) = imports_blocks[*index].clone() {
+                result.push((path, block));
             }
         }
+        result.push((self.path.clone().unwrap_or(String::from("unknown")), block));
+        result
+    }
+
+    pub fn translate(&self, block: Block, meta: ParserMetadata) -> String {
+        let ast_forest = self.get_sorted_ast_forest(block, &meta);
+        let mut meta_translate = TranslateMetadata::new(meta);
+        let time = Instant::now();
+        let mut result = vec![];
+        for (_path, block) in ast_forest {
+            result.push(block.translate(&mut meta_translate));
+        }
         if Self::env_flag_set(AMBER_DEBUG_TIME) {
             let pathname = self.path.clone().unwrap_or(String::from("unknown"));
             println!(
@@ -134,7 +150,6 @@ impl AmberCompiler {
                 time.elapsed().as_millis()
             );
         }
-        result.push(block.translate(&mut meta));
         let res = result.join("\n");
         let header = [
             include_str!("header.sh"),
@@ -144,9 +159,55 @@ impl AmberCompiler {
         format!("{}\n{}", header, res)
     }
 
+    pub fn document(&self, block: Block, meta: ParserMetadata, output: String) {
+        let base_path = PathBuf::from(meta.get_path().expect("Input file must exist in docs generation"));
+        let base_dir = fs::canonicalize(base_path)
+            .map(|val| val.parent().expect("Parent dir must exist in docs generation").to_owned().clone());
+        if let Err(err) = base_dir {
+            Message::new_err_msg("Couldn't get the absolute path to the provided input file")
+                .comment(err.to_string())
+                .show();
+            std::process::exit(1);
+        }
+        let base_dir = base_dir.unwrap();
+        let ast_forest = self.get_sorted_ast_forest(block, &meta);
+        for (path, block) in ast_forest {
+            let dep_path = {
+                let dep_path = fs::canonicalize(PathBuf::from(path.clone()));
+                if dep_path.is_err() {
+                    continue
+                }
+                let dep_path = dep_path.unwrap();
+
+                if !dep_path.starts_with(&base_dir) {
+                    continue
+                }
+
+                dep_path
+            };
+            let document = block.document(&meta);
+            // Save to file
+            let dir_path = {
+                let file_dir = dep_path.strip_prefix(&base_dir).unwrap();
+                let parent = file_dir.parent().unwrap().display();
+                format!("{}/{output}/{}", base_dir.to_string_lossy(), parent)
+            };
+            if let Err(err) = fs::create_dir_all(dir_path.clone()) {
+                Message::new_err_msg(format!("Couldn't create directory `{dir_path}`. Do you have sufficient permissions?"))
+                    .comment(err.to_string())
+                    .show();
+                std::process::exit(1);
+            }
+            let filename = dep_path.file_stem().unwrap().to_string_lossy();
+            let path = format!("{dir_path}/{filename}.md");
+            let mut file = File::create(path).unwrap();
+            file.write_all(document.as_bytes()).unwrap();
+        }
+    }
+
     pub fn compile(&self) -> Result<(Vec<Message>, String), Message> {
         self.tokenize()
-            .and_then(|tokens| self.parse(tokens))
+            .and_then(|tokens| self.parse(tokens, false))
             .map(|(block, meta)| (meta.messages.clone(), self.translate(block, meta)))
     }
 
@@ -160,6 +221,11 @@ impl AmberCompiler {
             .wait()?)
     }
 
+    pub fn generate_docs(&self, output: String) -> Result<(), Message> {
+        self.tokenize().and_then(|tokens| self.parse(tokens, true))
+            .map(|(block, meta)| self.document(block, meta, output))
+    }
+
     #[allow(dead_code)]
     pub fn test_eval(&mut self) -> Result<String, Message> {
         self.compile().map_or_else(Err, |(_, code)| {

src/docs/mod.rs

@@ -0,0 +1 @@
+pub mod module;

src/docs/module.rs

@@ -0,0 +1,5 @@
+use crate::utils::ParserMetadata;
+
+pub trait DocumentationModule {
+    fn document(&self, meta: &ParserMetadata) -> String;
+}

src/main.rs

@@ -2,6 +2,7 @@ mod compiler;
 mod modules;
 mod rules;
 mod translate;
+mod docs;
 mod utils;
 mod stdlib;
 
@@ -27,28 +28,27 @@ struct Cli {
     /// Code to evaluate
     #[arg(short, long)]
     eval: Option<String>,
+
+    /// Generate docs
+    #[arg(long)]
+    docs: bool
 }
 
 fn main() -> Result<(), Box<dyn Error>> {
     let cli = Cli::parse();
+    if cli.docs {
+        handle_docs(cli)?;
+    } else if let Some(code) = cli.eval {
+        handle_eval(code)?;
+    } else {
+        handle_compile(cli)?;
+    }
+    Ok(())
+}
 
-    if let Some(code) = cli.eval {
-        let code = format!("import * from \"std\"\n{code}");
-        match AmberCompiler::new(code, None).compile() {
-            Ok((messages, code)) => {
-                messages.iter().for_each(|m| m.show());
-                (!messages.is_empty()).then(|| render_dash());
-                let exit_status = AmberCompiler::execute(code, &vec![])?;
-                std::process::exit(exit_status.code().unwrap_or(1));
-            }
-            Err(err) => {
-                err.show();
-                std::process::exit(1);
-            }
-        }
-    } else if let Some(input) = cli.input {
+fn handle_compile(cli: Cli) -> Result<(), Box<dyn Error>> {
+    if let Some(input) = cli.input {
         let input = String::from(input.to_string_lossy());
-
         match fs::read_to_string(&input) {
             Ok(code) => {
                 match AmberCompiler::new(code, Some(input)).compile() {
@@ -88,11 +88,54 @@ fn main() -> Result<(), Box<dyn Error>> {
                 std::process::exit(1);
             }
         }
-    } else {
     }
     Ok(())
 }
 
+fn handle_eval(code: String) -> Result<(), Box<dyn Error>> {
+    let code = format!("import * from \"std\"\n{code}");
+    match AmberCompiler::new(code, None).compile() {
+        Ok((messages, code)) => {
+            messages.iter().for_each(|m| m.show());
+            (!messages.is_empty()).then(|| render_dash());
+            let exit_status = AmberCompiler::execute(code, &vec![])?;
+            std::process::exit(exit_status.code().unwrap_or(1));
+        }
+        Err(err) => {
+            err.show();
+            std::process::exit(1);
+        }
+    }
+}
+
+fn handle_docs(cli: Cli) -> Result<(), Box<dyn Error>> {
+    if let Some(input) = cli.input {
+        let input = String::from(input.to_string_lossy());
+        let output = {
+            let out = cli.output.unwrap_or_else(|| PathBuf::from("docs"));
+            String::from(out.to_string_lossy())
+        };
+        match fs::read_to_string(&input) {
+            Ok(code) => {
+                match AmberCompiler::new(code, Some(input)).generate_docs(output) {
+                    Ok(_) => Ok(()),
+                    Err(err) => {
+                        err.show();
+                        std::process::exit(1);
+                    }
+                }
+            },
+            Err(err) => {
+                Message::new_err_msg(err.to_string()).show();
+                std::process::exit(1);
+            }
+        }
+    } else {
+        Message::new_err_msg("You need to provide a path to an entry file to generate the documentation").show();
+        std::process::exit(1);
+    }
+}
+
 #[cfg(target_os = "windows")]
 fn set_file_permission(_file: &fs::File, _output: String) {
     // We don't need to set permission on Windows

src/modules/block.rs

@@ -1,7 +1,8 @@
 use std::collections::VecDeque;
 
 use heraclitus_compiler::prelude::*;
-use crate::{utils::{metadata::ParserMetadata, TranslateMetadata}};
+use crate::docs::module::DocumentationModule;
+use crate::utils::{metadata::ParserMetadata, TranslateMetadata};
 use crate::translate::module::TranslateModule;
 use super::statement::stmt::Statement;
 
@@ -40,7 +41,7 @@ impl SyntaxModule<ParserMetadata> for Block {
                 continue;
             }
             // Handle comments
-            if token.word.starts_with("//") {
+            if token.word.starts_with("//") && !token.word.starts_with("///") {
                 meta.increment_index();
                 continue
             }
@@ -83,3 +84,11 @@ impl TranslateModule for Block {
         result
     }
 }
+
+impl DocumentationModule for Block {
+    fn document(&self, meta: &ParserMetadata) -> String {
+        self.statements.iter()
+            .map(|statement| statement.document(meta))
+            .collect::<Vec<_>>().join("")
+    }
+}

src/modules/builtin/echo.rs

@@ -1,4 +1,5 @@
 use heraclitus_compiler::prelude::*;
+use crate::docs::module::DocumentationModule;
 use crate::modules::expression::expr::Expr;
 use crate::translate::module::TranslateModule;
 use crate::utils::{ParserMetadata, TranslateMetadata};
@@ -30,3 +31,9 @@ impl TranslateModule for Echo {
         format!("echo {}", value)
     }
 }
+
+impl DocumentationModule for Echo {
+    fn document(&self, _meta: &ParserMetadata) -> String {
+        "".to_string()
+    }
+}