fix: return correct exit code when one env failed

.github/workflows/ci.yml

@@ -64,9 +64,24 @@ jobs:
         run: |
           make test
           echo "Checking if ${LOCK_FILE} has changed..."
-      - name: Run examples
+      - name: Run basic example
         run: |
           make basic-example
+      - name: Run bad example
+        run: |
+          if make bad-example; then
+            echo "Cannot reach here, this example should fail"
+            exit 1
+          else
+            echo "Expected, this example should fail"
+          fi
+
+          if [ -f "/tmp/sqlness-bad-example.lock" ]; then
+            echo "Lock file should be deleted after run."
+            exit 1
+          else
+            echo "Expected, lock file is deleted."
+          fi
       - name: Check Lock File
         run: |
           diff ${LOCK_FILE} ${LOCK_FILE}.bak

.gitignore

@@ -1 +1,5 @@
 /target
+# Ignore this output file, since the bad example will always fail.
+# For normal case, output file should not be ignored, and it acts as
+# a signal that our testcase need to be fixed.
+examples/bad-case/simple/select.output

Makefile

@@ -26,4 +26,7 @@ clippy:
 	cd $(DIR); cargo clippy --all-targets --all-features --workspace -- -D warnings
 
 basic-example:
-	cd $(DIR); cargo run --example basic -- examples/basic.toml
+	cd $(DIR); cargo run --example basic
+
+bad-example:
+	cd $(DIR); cargo run --example bad

README.md

@@ -23,13 +23,12 @@ examples/
 │       ├── select.result    # Expected result file
 │       └── select.sql       # Input SQL testcase
 ├── basic.rs                 # Entrypoint of this example
-└── basic.toml               # Config of this example
 
 ```
 
 When run it via
 ```bash
-cargo run --example basic -- examples/basic.toml
+cargo run --example basic
 ```
 It will do following things:
 1. Collect all environments(first-level directory) under `basic-case`.

examples/bad-case/simple/select.result

@@ -0,0 +1,7 @@
+SELECT
+    *
+FROM
+    `table`;
+
+ok
+

examples/bad-case/simple/select.sql

@@ -0,0 +1,4 @@
+SELECT
+    *
+FROM
+    `table`;

examples/bad.rs

@@ -0,0 +1,64 @@
+// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0.
+
+//! A demo designed to run failed.
+//!
+//! When there is any diff between ${testcase}.output and ${testcase}.result,
+//! Users must resolve the diff, and keep the result file up to date.
+
+use std::{fmt::Display, fs::File, path::Path};
+
+use async_trait::async_trait;
+use sqlness::{ConfigBuilder, Database, EnvController, Runner};
+
+struct MyController;
+struct MyDB;
+
+#[async_trait]
+impl Database for MyDB {
+    async fn query(&self, _query: String) -> Box<dyn Display> {
+        return Box::new("Unexpected".to_string());
+    }
+}
+
+// Used as a flag to indicate MyDB has started
+const LOCK_FILE: &str = "/tmp/sqlness-bad-example.lock";
+
+impl MyDB {
+    fn new(_env: &str, _config: Option<&Path>) -> Self {
+        File::create(LOCK_FILE).unwrap();
+        MyDB
+    }
+
+    fn stop(self) {
+        std::fs::remove_file(LOCK_FILE).unwrap();
+    }
+}
+
+#[async_trait]
+impl EnvController for MyController {
+    type DB = MyDB;
+
+    async fn start(&self, env: &str, config: Option<&Path>) -> Self::DB {
+        MyDB::new(env, config)
+    }
+
+    async fn stop(&self, _env: &str, database: Self::DB) {
+        database.stop();
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    let env = MyController;
+    let config = ConfigBuilder::default()
+        .case_dir("examples/bad-case".to_string())
+        .build()
+        .unwrap();
+    let runner = Runner::new_with_config(config, env)
+        .await
+        .expect("Create Runner failed");
+
+    println!("Run testcase...");
+
+    runner.run().await.unwrap();
+}

examples/basic.rs

@@ -1,9 +1,9 @@
 // Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0.
 
-use std::{env, fmt::Display, path::Path, process};
+use std::{fmt::Display, path::Path};
 
 use async_trait::async_trait;
-use sqlness::{Database, EnvController, Runner};
+use sqlness::{ConfigBuilder, Database, EnvController, Runner};
 
 struct MyController;
 struct MyDB;
@@ -44,15 +44,12 @@ impl EnvController for MyController {
 
 #[tokio::main]
 async fn main() {
-    let args: Vec<String> = env::args().collect();
-    if args.len() < 2 {
-        println!("Usage: {} config-path", args[0]);
-        process::exit(1);
-    }
-
     let env = MyController;
-    let config_path = &args[1];
-    let runner = Runner::try_new(config_path, env)
+    let config = ConfigBuilder::default()
+        .case_dir("examples/basic-case".to_string())
+        .build()
+        .unwrap();
+    let runner = Runner::new_with_config(config, env)
         .await
         .expect("Create Runner failed");
 

src/runner.rs

@@ -2,7 +2,6 @@
 
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::sync::Arc;
 
 use prettydiff::basic::DiffOp;
 use prettydiff::diff_lines;
@@ -32,11 +31,11 @@ use crate::{config::Config, environment::EnvController};
 /// For more detailed explaination, refer to crate level documentment.
 pub struct Runner<E: EnvController> {
     config: Config,
-    env_controller: Arc<E>,
+    env_controller: E,
 }
 
 impl<E: EnvController> Runner<E> {
-    pub async fn try_new<P: AsRef<Path>>(config_path: P, env: E) -> Result<Self> {
+    pub async fn try_new<P: AsRef<Path>>(config_path: P, env_controller: E) -> Result<Self> {
         let mut config_file =
             File::open(config_path.as_ref())
                 .await
@@ -55,19 +54,20 @@ impl<E: EnvController> Runner<E> {
 
         Ok(Self {
             config,
-            env_controller: Arc::new(env),
+            env_controller,
         })
     }
 
-    pub async fn new_with_config(config: Config, env: E) -> Result<Self> {
+    pub async fn new_with_config(config: Config, env_controller: E) -> Result<Self> {
         Ok(Self {
             config,
-            env_controller: Arc::new(env),
+            env_controller,
         })
     }
 
     pub async fn run(&self) -> Result<()> {
         let environments = self.collect_env().await?;
+        let mut errors = Vec::new();
         for env in environments {
             let env_config = self.read_env_config(&env).await;
             let config_path = env_config.as_path();
@@ -77,10 +77,23 @@ impl<E: EnvController> Runner<E> {
                 None
             };
             let db = self.env_controller.start(&env, config_path).await;
-            if let Err(e) = self.run_env(&env, &db).await {
-                println!("Environment {} run failed with error {:?}", env, e);
-            }
+            let run_result = self.run_env(&env, &db).await;
             self.env_controller.stop(&env, db).await;
+
+            if let Err(e) = run_result {
+                println!("Environment {} run failed, error:{:?}.", env, e);
+
+                if self.config.fail_fast {
+                    return Err(e);
+                }
+
+                errors.push(e);
+            }
+        }
+
+        // only return first error
+        if let Some(e) = errors.pop() {
+            return Err(e);
         }
 
         Ok(())