Merge pull request #246 from candy-lang/add-ci

Add fuzzing to the CI

.github/workflows/compiler.yaml

@@ -67,3 +67,18 @@ jobs:
         with:
           command: clippy
           args: --manifest-path compiler/Cargo.toml -- -D warnings
+
+  # fuzzing:
+  #   name: Fuzzing
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: actions-rs/toolchain@v1
+  #       with:
+  #         profile: minimal
+  #         toolchain: ${{ env.RUST_VERSION }}
+  #         override: true
+  #     - uses: actions-rs/cargo@v1
+  #       with:
+  #         command: run
+  #         args: --manifest-path compiler/Cargo.toml -- fuzz packages/Benchmark.candy

compiler/src/fuzzer/fuzzer.rs

@@ -79,6 +79,9 @@ impl Fuzzer {
     pub fn status(&self) -> &Status {
         self.status.as_ref().unwrap()
     }
+    pub fn into_status(self) -> Status {
+        self.status.unwrap()
+    }
 
     pub fn run<U: UseProvider, E: ExecutionController>(
         &mut self,

compiler/src/fuzzer/mod.rs

@@ -12,13 +12,14 @@ use crate::{
     module::Module,
     vm::{
         context::{DbUseProvider, RunForever, RunLimitedNumberOfInstructions},
-        Closure, Heap, Pointer, Vm,
+        tracer::full::FullTracer,
+        Closure, Heap, Packet, Pointer, Vm,
     },
 };
 use itertools::Itertools;
 use tracing::{error, info};
 
-pub async fn fuzz(db: &Database, module: Module) {
+pub async fn fuzz(db: &Database, module: Module) -> Vec<FailingFuzzCase> {
     let (fuzzables_heap, fuzzables): (Heap, Vec<(Id, Pointer)>) = {
         let mut tracer = FuzzablesFinder::default();
         let mut vm = Vm::new();
@@ -32,30 +33,59 @@ pub async fn fuzz(db: &Database, module: Module) {
         fuzzables.len()
     );
 
+    let mut failing_cases = vec![];
+
     for (id, closure) in fuzzables {
         info!("Fuzzing {id}.");
         let mut fuzzer = Fuzzer::new(&fuzzables_heap, closure, id.clone());
         fuzzer.run(
             &mut DbUseProvider { db },
             &mut RunLimitedNumberOfInstructions::new(1000),
         );
-        match fuzzer.status() {
+        match fuzzer.into_status() {
             Status::StillFuzzing { .. } => {}
             Status::PanickedForArguments {
                 arguments,
                 reason,
                 tracer,
             } => {
                 error!("The fuzzer discovered an input that crashes {id}:");
-                error!(
-                    "Calling `{id} {}` doesn't work because {reason}.",
-                    arguments.iter().map(|arg| format!("{arg:?}")).join(" "),
-                );
-                error!(
-                    "This is the stack trace:\n{}",
-                    tracer.format_panic_stack_trace_to_root_fiber(db)
-                );
+                let case = FailingFuzzCase {
+                    closure: id,
+                    arguments,
+                    reason,
+                    tracer,
+                };
+                case.dump(db);
+                failing_cases.push(case);
             }
         }
     }
+
+    failing_cases
+}
+
+pub struct FailingFuzzCase {
+    closure: Id,
+    arguments: Vec<Packet>,
+    reason: String,
+    tracer: FullTracer,
+}
+
+impl FailingFuzzCase {
+    pub fn dump(&self, db: &Database) {
+        error!(
+            "Calling `{} {}` doesn't work because {}.",
+            self.closure,
+            self.arguments
+                .iter()
+                .map(|arg| format!("{arg:?}"))
+                .join(" "),
+            self.reason,
+        );
+        error!(
+            "This is the stack trace:\n{}",
+            self.tracer.format_panic_stack_trace_to_root_fiber(db)
+        );
+    }
 }

compiler/src/main.rs

@@ -91,7 +91,7 @@ struct CandyFuzzOptions {
 }
 
 #[tokio::main]
-async fn main() {
+async fn main() -> ProgramResult {
     match CandyOptions::from_args() {
         CandyOptions::Build(options) => build(options),
         CandyOptions::Run(options) => run(options),
@@ -100,16 +100,26 @@ async fn main() {
     }
 }
 
-fn build(options: CandyBuildOptions) {
+type ProgramResult = Result<(), Exit>;
+#[derive(Debug)]
+enum Exit {
+    FileNotFound,
+    FuzzingFoundFailingCases,
+    CodePanicked,
+}
+
+fn build(options: CandyBuildOptions) -> ProgramResult {
     init_logger(true);
     let module = Module::from_package_root_and_file(
         current_dir().unwrap(),
         options.file.clone(),
         ModuleKind::Code,
     );
-    raw_build(module.clone(), options.debug);
+    let result = raw_build(module.clone(), options.debug);
 
-    if options.watch {
+    if !options.watch {
+        result.ok_or(Exit::FileNotFound).map(|_| ())
+    } else {
         let (tx, rx) = channel();
         let mut watcher = watcher(tx, Duration::from_secs(1)).unwrap();
         watcher
@@ -196,7 +206,7 @@ fn raw_build(module: Module, debug: bool) -> Option<Arc<Lir>> {
     Some(lir)
 }
 
-fn run(options: CandyRunOptions) {
+fn run(options: CandyRunOptions) -> ProgramResult {
     init_logger(true);
     let module = Module::from_package_root_and_file(
         current_dir().unwrap(),
@@ -206,8 +216,8 @@ fn run(options: CandyRunOptions) {
     let db = Database::default();
 
     if raw_build(module.clone(), false).is_none() {
-        warn!("Build failed.");
-        return;
+        warn!("File not found.");
+        return Err(Exit::FileNotFound);
     };
     // TODO: Optimize the code before running.
 
@@ -256,7 +266,7 @@ fn run(options: CandyRunOptions) {
                 "This is the stack trace:\n{}",
                 tracer.format_panic_stack_trace_to_root_fiber(&db)
             );
-            return;
+            return Err(Exit::CodePanicked);
         }
     };
 
@@ -265,7 +275,7 @@ fn run(options: CandyRunOptions) {
         Some(main) => main,
         None => {
             error!("The module doesn't contain a main function.");
-            return;
+            return Err(Exit::CodePanicked);
         }
     };
 
@@ -307,6 +317,7 @@ fn run(options: CandyRunOptions) {
                 .for_fiber(FiberId::root())
                 .call_ended(return_value.address, &return_value.heap);
             debug!("The main function returned: {return_value:?}");
+            Ok(())
         }
         ExecutionResult::Panicked {
             reason,
@@ -322,6 +333,7 @@ fn run(options: CandyRunOptions) {
                 "This is the stack trace:\n{}",
                 tracer.format_panic_stack_trace_to_root_fiber(&db)
             );
+            Err(Exit::CodePanicked)
         }
     }
 }
@@ -351,7 +363,7 @@ impl StdoutService {
     }
 }
 
-async fn fuzz(options: CandyFuzzOptions) {
+async fn fuzz(options: CandyFuzzOptions) -> ProgramResult {
     init_logger(true);
     let module = Module::from_package_root_and_file(
         current_dir().unwrap(),
@@ -360,22 +372,37 @@ async fn fuzz(options: CandyFuzzOptions) {
     );
 
     if raw_build(module.clone(), false).is_none() {
-        warn!("Build failed.");
-        return;
+        warn!("File not found.");
+        return Err(Exit::FileNotFound);
     }
 
     debug!("Fuzzing `{module}`.");
     let db = Database::default();
-    fuzzer::fuzz(&db, module).await;
+    let failing_cases = fuzzer::fuzz(&db, module).await;
+
+    if failing_cases.is_empty() {
+        info!("All found fuzzable closures seem fine.");
+        Ok(())
+    } else {
+        error!("");
+        error!("Finished fuzzing.");
+        error!("These are the failing cases:");
+        for case in failing_cases {
+            error!("");
+            case.dump(&db);
+        }
+        Err(Exit::FuzzingFoundFailingCases)
+    }
 }
 
-async fn lsp() {
+async fn lsp() -> ProgramResult {
     init_logger(false);
     info!("Starting language server…");
     let (service, socket) = LspService::new(CandyLanguageServer::from_client);
     Server::new(tokio::io::stdin(), tokio::io::stdout(), socket)
         .serve(service)
         .await;
+    Ok(())
 }
 
 fn init_logger(use_stdout: bool) {

compiler/src/vm/mod.rs

@@ -12,6 +12,7 @@ pub use self::{
     fiber::{ExecutionResult, Fiber},
     heap::{Closure, Heap, Object, Pointer, Struct},
     ids::{ChannelId, FiberId, OperationId},
+    tracer::{full::FullTracer, Tracer},
 };
 use self::{
     channel::{Channel, Completer, Performer},
@@ -21,7 +22,6 @@ use self::{
     },
     heap::SendPort,
     ids::{CountableId, IdGenerator},
-    tracer::Tracer,
 };
 use crate::compiler::hir::Id;
 use itertools::Itertools;

packages/Core/channel.candy

@@ -1,14 +1,19 @@
 bool = use "..bool"
 equals = (use "..equality").equals
 if = (use "..conditionals").if
-isInt = (use "..int").is
+int = use "..int"
 isType = (use "..type").is
 
 isSendPort value := isType value SendPort
 isReceivePort value := isType value ReceivePort
 
 create capacity :=
-  needs (isInt capacity) "`capacity` is not an integer. Channels need a capacity because otherwise, there would be no backpressure among fibers, and memory leaks would go unnoticed."
+  needs (int.is capacity) "`capacity` is not an integer. Channels need a capacity because otherwise, there would be no backpressure among fibers, and memory leaks would go unnoticed."
+  needs (int.isNonNegative capacity)
+  needs (int.fitsInRustU32 capacity)
+  # Technically, it needs to fit in a usize, the natural word length on systems.
+  # Typically, this is 64-bits, so we are conservative here, although there
+  # also exist exotic/old systems with smaller word sizes of 16 bits.
   ✨.channelCreate capacity
 
 send port packet :=

packages/Core/int.candy

@@ -39,6 +39,7 @@ compareTo valueA valueB :=
   needs (is valueB)
   result = valueA | ✨.intCompareTo valueB
   check (equals result Equal | bool.implies (equals valueA valueB))
+  # check ((equals result Equal) | bool.implies (equals valueA valueB))
   result
 isLessThan valueA valueB :=
   needs (is valueA)

packages/benchmark.candy

@@ -1,26 +1,33 @@
-# Benchmark by navigating to the Candy folder and then running this command:
-# `cargo build --release --manifest-path=compiler/Cargo.toml && time compiler/target/release/candy run packages/Benchmark.candy`
+# Run or benchmark by navigating to the Candy folder and then running this command:
+# `cargo build --release --manifest-path=compiler/Cargo.toml -- run packages/benchmark.candy`
+# `cargo build --release --manifest-path=compiler/Cargo.toml && time target/release/candy run packages/benchmark.candy`
 
 core = use "..Core"
 
-fibRec fibRec n =
+fibRec = { fibRec n ->
   core.ifElse (n | core.int.isLessThan 2) { n } {
     fibRec fibRec (n | core.int.subtract 1)
     | core.int.add (fibRec fibRec (n | core.int.subtract 2))
   }
-fib n = fibRec fibRec n
+}
+fib n =
+  needs (core.int.is n)
+  fibRec fibRec n
 twentyOne := fib 8
 
 main := { environment ->
-  print message = { core.channel.send environment.stdout message }
+  print message =
+    needs (core.text.is message)
+    core.channel.send environment.stdout message
+
   print "Hello, world!"
 
   core.parallel { nursery ->
     nursery
     | core.async {
-      print "Hello from fiber!"
-      "Hello, async await!"
-    }
+        print "Hello from fiber!"
+        "Hello, async await!"
+      }
     | core.await
     | print