Deps update, Argon2 hashes, wait for more resource types, context imp…

…rovement, rust formatting and preparing 0.3.1
sebt3 · Nov 12, 2024 · a14741a · a14741a
1 parent bb7b47b
commit a14741a
Showing 41 changed files with 1,416 additions and 476 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/agent/scripts/lib/wait.rhai b/agent/scripts/lib/wait.rhai
@@ -0,0 +1,64 @@
+fn workload(list, duration) {
+    for wl in list.filter(|wl| ["Deployment", "DaemonSet", "StatefulSet"].contains(wl.kind)) {
+        log_info(`Waiting for ${wl.kind} ${wl.namespace} ${wl.name} to be available`);
+        if wl.kind == "Deployment" {
+            let deploy = get_deployment(wl.namespace, w.name);
+            deploy.wait_available(duration);
+        } else if wl.kind == "DaemonSet" {
+            let ds = get_deamonset(wl.namespace, wl.name);
+            ds.wait_available(duration);
+        } else if wl.kind == "StatefulSet" {
+            let sts = get_statefulset(wl.namespace, wl.name);
+            sts.wait_available(duration);
+        }
+    }
+}
+fn workload(list) {
+    workload(list, 2*60);
+}
+
+fn job(list, duration) {
+    for j in applied_objs.filter(|j| j.kind == "Job") {
+        log_info(`Waiting for ${j.kind} ${j.namespace} ${j.name} to finish`);
+        let job = get_job(j.namespace, j.name);
+        job.wait_done(duration);
+    }
+}
+fn job(list) {
+    job(list, 5*60);
+}
+
+fn vital(list, duration) {
+    for v in list {
+        if ["Cluster"].contains(v.kind) {
+            log_info(`Waiting for ${v.kind} ${v.namespace} ${v.name} to be available`);
+            let api = k8s_resource(v.kind, v.namespace);
+            let obj = api.get_obj(v.name);
+            obj.wait_condition("Ready", duration);
+        } else if ["RabbitmqCluster"].contains(v.kind) {
+            log_info(`Waiting for ${v.kind} ${v.namespace} ${v.name} to be available`);
+            let api = k8s_resource(v.kind, v.namespace);
+            let obj = api.get_obj(v.name);
+            obj.wait_condition("ClusterAvailable", duration);
+        } else if ["Redis", "MongoDBCommunity"].contains(v.kind) {
+            log_info(`Waiting for ${v.kind} ${v.namespace} ${v.name} to be available`);
+            let api = k8s_resource(v.kind, v.namespace);
+            let sts = get_statefulset(v.namespace, v.name);
+            sts.wait_available(duration);
+        }
+    }
+}
+fn vital(list) {
+    vital(list, 2*60);
+}
+
+fn all(list, duration) {
+    vital(list, duration);
+    job(list, duration);
+    workload(list, duration);
+}
+fn all(list) {
+    vital(list);
+    job(list);
+    workload(list);
+}
diff --git a/agent/scripts/system/context.rhai b/agent/scripts/system/context.rhai
@@ -1,6 +1,7 @@
 import "build_context" as build;
 fn run(instance, args) {
     let context = build::run(instance, args);
+    context = import_run("context_system", instance, context);
     let extra = import_run("context_extra", instance, context);
     context["extra"] = extra;
     context

diff --git a/agent/scripts/system/install_systems.rhai b/agent/scripts/system/install_systems.rhai
@@ -1,4 +1,5 @@
 import "install_from_dir" as dir;
+import "wait" as wait;
 fn run(instance, context) {
     import_run("install_systems_pre", instance, context);
     try {
@@ -7,11 +8,7 @@ fn run(instance, context) {
             dir::remove(applied_objs, instance.status.systems, instance.metadata.namespace);
         }
         let tmp = instance.set_status_systems(applied_objs);
-        for d in applied_objs.filter(|d| d.kind == "Deployment") {
-            log_info(`Waiting for ${d.kind} ${d.namespace} ${d.name} to be available`);
-            let deploy = get_deployment(d.namespace, d.name);
-            deploy.wait_available(60*5);
-        }
+        wait::all(applied_objs);
     } catch (e) {
         switch type_of(e) {
             "string" => instance.set_status_system_failed(e),

diff --git a/agent/scripts/tenant/context.rhai b/agent/scripts/tenant/context.rhai
@@ -3,12 +3,11 @@ fn run(instance, args) {
     let context = build::run(instance, args);
     let name = instance.get_tenant_name();
     let nss = instance.get_tenant_namespaces();
-    //TODO: complete tenant values (ha, backup, sso, mail)
     context["tenant"] = #{
         name: name,
         namespaces: nss,
     };
-    //TODO: namespace config
+    context = import_run("context_tenant", instance, context);
     let extra = import_run("context_extra", instance, context);
     context["extra"] = extra;
     context

diff --git a/agent/scripts/tenant/install_others.rhai b/agent/scripts/tenant/install_others.rhai
@@ -1,4 +1,5 @@
 import "install_from_dir" as dir;
+import "wait" as wait;
 fn run(instance, context) {
     import_run("install_others_pre", instance, context);
     try {
@@ -7,11 +8,7 @@ fn run(instance, context) {
             dir::remove(applied_objs, instance.status.others, instance.metadata.namespace);
         }
         let tmp = instance.set_status_others(applied_objs);
-        for j in applied_objs.filter(|j| j.kind == "Job") {
-            log_info(`Waiting for ${j.kind} ${j.name} to finish`);
-            let job = get_job(j.namespace, j.name);
-            job.wait_done(60*5);
-        }
+        wait::all(applied_objs);
     } catch (e) {
         switch type_of(e) {
             "string" => instance.set_status_other_failed(e),

diff --git a/agent/scripts/tenant/install_scalables.rhai b/agent/scripts/tenant/install_scalables.rhai
@@ -1,4 +1,5 @@
 import "install_from_dir" as dir;
+import "wait" as wait;
 fn run(instance, context) {
     import_run("install_scalables_pre", instance, context);
     try {
@@ -8,11 +9,7 @@ fn run(instance, context) {
         }
         // TODO: scale a target
         let tmp = instance.set_status_scalables(applied_objs);
-        for d in applied_objs.filter(|d| d.kind == "Deployment") {
-            log_info(`Waiting for ${d.kind} ${d.name} to be available`);
-            let deploy = get_deployment(d.namespace, d.name);
-            deploy.wait_available(60*5);
-        }
+        wait::all(applied_objs);
     } catch (e) {
         switch type_of(e) {
             "string" => instance.set_status_scalable_failed(e),

diff --git a/agent/scripts/tenant/install_vitals.rhai b/agent/scripts/tenant/install_vitals.rhai
@@ -1,4 +1,5 @@
 import "install_from_dir" as dir;
+import "wait" as wait;
 fn run(instance, context) {
     import_run("install_vitals_pre", instance, context);
     try {
@@ -12,6 +13,7 @@ fn run(instance, context) {
             }
         }
         let tmp = instance.set_status_vitals(applied_objs);
+        wait::all(applied_objs);
     } catch (e) {
         switch type_of(e) {
             "string" => instance.set_status_vital_failed(e),

diff --git a/agent/src/boxes/scan.rs b/agent/src/boxes/scan.rs
@@ -1,7 +1,7 @@
+use clap::Args;
 use common::{context::set_box, jukebox::JukeBox, rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
-use clap::Args;
 
 #[derive(Args, Debug, Serialize, Deserialize)]
 pub struct Parameters {

diff --git a/agent/src/package/build.rs b/agent/src/package/build.rs
@@ -1,7 +1,7 @@
+use clap::Args;
 use common::{rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
-use clap::Args;
 
 #[derive(clap::ValueEnum, Clone, Default, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
@@ -103,5 +103,5 @@ pub async fn run(args: &Parameters) -> Result<()> {
         "import(\"build\") as build;\n\
         build::run(args);",
     )?;
-     Ok(())
+    Ok(())
 }
diff --git a/agent/src/package/mod.rs b/agent/src/package/mod.rs
@@ -1,7 +1,7 @@
 mod build;
+mod test;
 mod unpack;
 mod update;
-mod test;
 use clap::{Parser, Subcommand};
 use std::process;
 

diff --git a/agent/src/package/test.rs b/agent/src/package/test.rs
@@ -1,7 +1,13 @@
-use common::{context::{set_system, set_tenant}, instancesystem::SystemInstance, instancetenant::TenantInstance, rhaihandler::Script, Result};
+use clap::Args;
+use common::{
+    context::{set_system, set_tenant},
+    instancesystem::SystemInstance,
+    instancetenant::TenantInstance,
+    rhaihandler::Script,
+    Result,
+};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
-use clap::Args;
 
 #[derive(clap::ValueEnum, Clone, Default, Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "snake_case")]
@@ -83,5 +89,5 @@ pub async fn run(args: &Parameters) -> Result<()> {
         "import(\"test\") as test;\n\
         test::run(instance, args);",
     )?;
-     Ok(())
+    Ok(())
 }
diff --git a/agent/src/package/unpack.rs b/agent/src/package/unpack.rs
@@ -1,5 +1,5 @@
 use clap::Args;
-use common::{ocihandler::Registry, Error, Result, rhaihandler::base64_decode};
+use common::{ocihandler::Registry, rhaihandler::base64_decode, Error, Result};
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 
@@ -83,16 +83,14 @@ pub async fn run(args: &Parameters) -> Result<()> {
                 args.password.clone(),
             )
         } else {
-            let pull_secret_string = std::fs::read_to_string(format!("{}/.dockerconfigjson", args.pull_path)).map_err(|e| Error::Stdio(e))?;
-            let pull_secret: serde_json::Value = serde_json::from_str(&pull_secret_string).map_err(|e| Error::SerializationError(e))?;
+            let pull_secret_string = std::fs::read_to_string(format!("{}/.dockerconfigjson", args.pull_path))
+                .map_err(|e| Error::Stdio(e))?;
+            let pull_secret: serde_json::Value =
+                serde_json::from_str(&pull_secret_string).map_err(|e| Error::SerializationError(e))?;
             let hash = pull_secret["auths"][args.registry.clone()]["auth"].clone();
             let user_pass = base64_decode(hash.as_str().unwrap().to_string())?;
             let auth = user_pass.split(":").collect::<Vec<&str>>();
-            Registry::new(
-                args.registry.clone(),
-                auth[0].to_string(),
-                auth[1].to_string(),
-            )
+            Registry::new(args.registry.clone(), auth[0].to_string(), auth[1].to_string())
         };
         cli.pull_image(&args.destination, args.image.clone(), args.tag.clone())
     }

diff --git a/agent/src/package/update.rs b/agent/src/package/update.rs
@@ -1,7 +1,7 @@
+use clap::Args;
 use common::{rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
-use clap::Args;
 
 #[derive(Args, Debug, Serialize, Deserialize)]
 pub struct Parameters {
@@ -36,5 +36,5 @@ pub async fn run(args: &Parameters) -> Result<()> {
         "import(\"update\") as update;\n\
         update::run(args);",
     )?;
-     Ok(())
+    Ok(())
 }
diff --git a/agent/src/system/delete.rs b/agent/src/system/delete.rs
@@ -1,6 +1,6 @@
+use clap::Args;
 use common::{context::set_system, instancesystem::SystemInstance, rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
-use clap::Args;
 
 #[derive(Args, Debug, Serialize, Deserialize)]
 pub struct Parameters {
@@ -52,6 +52,7 @@ pub async fn run(args: &Parameters) -> Result<()> {
         format!("{}/scripts", args.package_dir),
         format!("{}/system", args.script_dir),
         format!("{}/lib", args.script_dir),
+        format!("{}", args.config_dir),
     ]);
     let context = SystemInstance::get(args.namespace.clone(), args.instance.clone()).await?;
     set_system(context.clone());

diff --git a/agent/src/system/install.rs b/agent/src/system/install.rs
@@ -1,6 +1,6 @@
+use clap::Args;
 use common::{context::set_system, instancesystem::SystemInstance, rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
-use clap::Args;
 
 #[derive(Args, Debug, Serialize, Deserialize)]
 pub struct Parameters {
@@ -55,6 +55,7 @@ pub async fn run(args: &Parameters) -> Result<()> {
         format!("{}/scripts", args.package_dir),
         format!("{}/system", args.script_dir),
         format!("{}/lib", args.script_dir),
+        format!("{}", args.config_dir),
     ]);
     let context = SystemInstance::get(args.namespace.clone(), args.instance.clone()).await?;
     set_system(context.clone());

diff --git a/agent/src/tenant/delete.rs b/agent/src/tenant/delete.rs
@@ -1,6 +1,6 @@
+use clap::Args;
 use common::{context::set_tenant, instancetenant::TenantInstance, rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
-use clap::Args;
 
 #[derive(Args, Debug, Serialize, Deserialize)]
 pub struct Parameters {
@@ -52,6 +52,7 @@ pub async fn run(args: &Parameters) -> Result<()> {
         format!("{}/scripts", args.package_dir),
         format!("{}/tenant", args.script_dir),
         format!("{}/lib", args.script_dir),
+        format!("{}", args.config_dir),
     ]);
     let context = TenantInstance::get(args.namespace.clone(), args.instance.clone()).await?;
     set_tenant(context.clone());

diff --git a/agent/src/tenant/install.rs b/agent/src/tenant/install.rs
@@ -1,6 +1,6 @@
+use clap::Args;
 use common::{context::set_tenant, instancetenant::TenantInstance, rhaihandler::Script, Result};
 use serde::{Deserialize, Serialize};
-use clap::Args;
 
 #[derive(Args, Debug, Serialize, Deserialize)]
 pub struct Parameters {
@@ -55,6 +55,7 @@ pub async fn run(args: &Parameters) -> Result<()> {
         format!("{}/scripts", args.package_dir),
         format!("{}/tenant", args.script_dir),
         format!("{}/lib", args.script_dir),
+        format!("{}", args.config_dir),
     ]);
     let context = TenantInstance::get(args.namespace.clone(), args.instance.clone()).await?;
     set_tenant(context.clone());

diff --git a/agent/src/tenant/mod.rs b/agent/src/tenant/mod.rs
@@ -19,11 +19,11 @@ pub enum Commands {
     /// Delete an instance
     Delete(delete::Parameters),
     // Backup an instance
-//    Backup(backup::Parameters),
+    //    Backup(backup::Parameters),
     // Restore an instance
-//    Restore(restore::Parameters),
+    //    Restore(restore::Parameters),
     // Reconfigure an instance
-//    Reconfigure(reconfigure::Parameters),
+    //    Reconfigure(reconfigure::Parameters),
 }
 
 pub async fn run(cmd: &Parameters) {

diff --git a/box/vynil/package.yaml b/box/vynil/package.yaml
@@ -9,7 +9,7 @@ metadata:
   features:
   - upgrade
   - auto_config
-  app_version: 0.3.0
+  app_version: 0.3.1
 images:
   agent:
     registry: docker.io

diff --git a/box/vynil/systems/controller.yaml.hbs b/box/vynil/systems/controller.yaml.hbs
@@ -47,3 +47,11 @@ spec:
             port: http
           initialDelaySeconds: 5
           periodSeconds: 15
+        volumeMounts:
+        - name: config
+          mountPath: /etc/vynil
+      volumes:
+      - name: config
+        configMap:
+          name: vynil
+          optional: true
diff --git a/common/Cargo.toml b/common/Cargo.toml
@@ -18,7 +18,7 @@ chrono.workspace = true
 schemars.workspace = true
 actix-web.workspace = true
 base64.workspace = true
-thiserror = "1.0.65"
+thiserror = "2.0.3"
 handlebars = { version = "6.2.0", features = ["script_helper", "string_helpers"] }
 handlebars_misc_helpers = { version = "0.17.0", features = ["string", "json", "jsonnet", "regex", "uuid"] }
 rhai = { version = "1.20.0", features = ["sync", "serde"] }
@@ -32,6 +32,7 @@ flate2 = "1.0.33"
 oci-client = "0.12.1"
 openapiv3 = "2.0.0"
 sha256 = "1.5.0"
+argon2 = { version = "0.5.3", features = ["std"] }
 
 [lib]
 name = "common"