From 24f0981a23851d9d093cad5b52fc3798b11bb926 Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Fri, 17 May 2024 11:05:39 +0200 Subject: [PATCH 1/3] Reintroduce progress changes --- model/Cargo.toml | 1 + model/src/activity.rs | 2 +- model/src/activity/exe_script_command.rs | 25 ++++++ model/src/activity/runtime_event.rs | 20 +++++ specs/activity-api.yaml | 97 +++++++++++++++++++++--- 5 files changed, 133 insertions(+), 12 deletions(-) diff --git a/model/Cargo.toml b/model/Cargo.toml index 64ecd380..1d47cc44 100644 --- a/model/Cargo.toml +++ b/model/Cargo.toml @@ -17,6 +17,7 @@ sgx = ['secp256k1', 'openssl', 'hex', 'secp256k1/serde'] bigdecimal = { version = "0.2", features = ["serde"]} chrono = { version = "0.4", features = ["serde"]} derive_more = "0.99" +humantime-serde = "1.1" rand = "0.8" serde = { version = "1.0.146", features = ["derive"] } serde_bytes = "0.11.14" diff --git a/model/src/activity.rs b/model/src/activity.rs index 2c1a2ae0..76558435 100644 --- a/model/src/activity.rs +++ b/model/src/activity.rs @@ -21,7 +21,7 @@ pub use self::exe_script_command_result::{CommandOutput, CommandResult, ExeScrip pub use self::exe_script_command_state::ExeScriptCommandState; pub use self::exe_script_request::ExeScriptRequest; pub use self::provider_event::ProviderEvent; -pub use self::runtime_event::{RuntimeEvent, RuntimeEventKind}; +pub use self::runtime_event::{CommandProgress, RuntimeEvent, RuntimeEventKind}; #[cfg(feature = "sgx")] pub use self::sgx_credentials::SgxCredentials; diff --git a/model/src/activity/exe_script_command.rs b/model/src/activity/exe_script_command.rs index df72e5c2..c7c4c67a 100644 --- a/model/src/activity/exe_script_command.rs +++ b/model/src/activity/exe_script_command.rs @@ -11,6 +11,7 @@ use crate::activity::ExeScriptCommandState; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::time::Duration; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -21,6 +22,18 @@ pub enum ExeScriptCommand { net: Vec, #[serde(default)] hosts: HashMap, // hostname -> IP + + #[serde(default)] + hostname: Option, + + #[serde(default)] + volumes: Vec, + + #[serde(default)] + env: HashMap, + + #[serde(default)] + progress: Option, }, Start { #[serde(default)] @@ -39,6 +52,8 @@ pub enum ExeScriptCommand { to: String, #[serde(flatten)] args: TransferArgs, + #[serde(default)] + progress: Option, }, Terminate {}, } @@ -107,6 +122,16 @@ pub struct TransferArgs { pub fileset: Option, } +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct ProgressArgs { + #[serde(default)] + #[serde(with = "humantime_serde")] + pub update_interval: Option, + /// Number of bytes after which next progress event will be sent. + pub update_step: Option, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum FileSet { diff --git a/model/src/activity/runtime_event.rs b/model/src/activity/runtime_event.rs index a1890047..2efeb8f7 100644 --- a/model/src/activity/runtime_event.rs +++ b/model/src/activity/runtime_event.rs @@ -47,6 +47,10 @@ impl RuntimeEvent { pub fn stderr(batch_id: String, idx: usize, out: CommandOutput) -> Self { Self::new(batch_id, idx, RuntimeEventKind::StdErr(out)) } + + pub fn progress(batch_id: String, idx: usize, progress: CommandProgress) -> Self { + Self::new(batch_id, idx, RuntimeEventKind::Progress(progress)) + } } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -61,4 +65,20 @@ pub enum RuntimeEventKind { }, StdOut(CommandOutput), StdErr(CommandOutput), + Progress(CommandProgress), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub struct CommandProgress { + /// Steps are counted starting from 0. That means that first step from 4-steps tasks + /// will report 0/4. Task is finished when counter reaches 4/4. + pub step: (usize, usize), + /// May contain additional arbitrary information about, what is happening with the task + /// like retrying transfer or that image was deployed from cache. + pub message: Option, + /// Granular progress of currently executed step. The first element describes current + /// progress, the second the size of the whole task, which can be unknown. + pub progress: (u64, Option), + pub unit: Option, } diff --git a/specs/activity-api.yaml b/specs/activity-api.yaml index b5393c02..925d67bb 100644 --- a/specs/activity-api.yaml +++ b/specs/activity-api.yaml @@ -486,7 +486,8 @@ components: "deploy": { "net": [{ "id": "id", "ip": "10.0.0.2", "mask": "255.255.0.0" }], "hosts": {"master": "10.0.0.1"}, - "nodes": {"10.0.0.1": "0xdeadbeef"} + "nodes": {"10.0.0.1": "0xdeadbeef"}, + "progress" : {"updateInterval": "300ms", "updateStep": null} } }, { @@ -500,7 +501,8 @@ components: "to": "container:/input/file_in", "format": "zip", "depth": 2, - "fileset": [{"desc":"all images", "includes": ["*.jpg"], "excludes": ["db*.*"] }] + "fileset": [{"desc":"all images", "includes": ["*.jpg"], "excludes": ["db*.*"] }], + "progress" : {"updateInterval": null, "updateStep": 1048576} } }, { @@ -557,6 +559,8 @@ components: type: object additionalProperties: type: string + progress: + $ref: '#/components/schemas/ProgressArgs' DeployNetwork: type: object @@ -572,6 +576,25 @@ components: mask: type: string + ProgressArgs: + type: object + description: Configuration of progress reporting. + Presence of this field in ExeUnitCommand indicates, that ExeUnit should send + '#/components/schemas/RuntimeEventKindProgress' events. If non of properties is set + ExeUnit will use default values. + Behavior when both properties are defined is ExeUnit specific. + properties: + update-interval: + type: string + description: Interval between progress reporting events expressed as + described in specification https://docs.rs/humantime/latest/humantime/fn.parse_duration.html + update-step: + type: number + format: int64 + minimum: 1 + description: Number of units (for example Bytes in case of transfer) after which next + progress event will be sent. + StartCommand: allOf: - $ref: '#/components/schemas/ExeScriptCommand' @@ -616,6 +639,8 @@ components: type: array items: $ref: '#/components/schemas/FileSet' + progress: + $ref: '#/components/schemas/ProgressArgs' FileSet: properties: @@ -842,15 +867,65 @@ components: $ref: '#/components/schemas/CommandOutput' RuntimeEventKindFinishedBody: - type: object - required: - - returnCode - properties: - returnCode: - type: integer - format: int32 - message: - type: string + allOf: + - $ref: '#/components/schemas/RuntimeEventKind' + - type: object + required: + - returnCode + properties: + returnCode: + type: integer + format: int32 + message: + type: string + + RuntimeEventKindProgress: + allOf: + - $ref: '#/components/schemas/RuntimeEventKind' + - type: object + description: Reports progress of currently executed command. This event will be sent only, + if `progress` field was set in `deploy` or `transfer` command. + required: + - step + properties: + step: + $ref: '#/components/schemas/ProgressStep' + message: + type: string + description: May contain additional arbitrary information, what is happening with the task, + for example "retrying transfer" or "image deployed from cache". + Content of this field is ExeUnit specific. + progress: + $ref: '#/components/schemas/ProgressDetails' + unit: + type: string + description: Units in which `progress` field is expressed. This should be human readable + string for displaying in UI. List of possible values is ExeUnit specific. + + ProgressStep: + description: Can be used if single ExeUnit command is executing multiple steps. Number of steps is + Exeunit specific. + + Steps are counted starting from 0. That means that first step from 4-steps task + will report 0/4. Task is finished when counter reaches 4/4. + type: array + items: + type: integer + format: int64 + minItems: 2 + maxItems: 2 + minimum: 0 + + ProgressDetails: + description: Granular progress of currently executed step. The first element describes current + progress, the second the size of the whole task, which can be unknown. + type: array + items: + type: integer + format: int64 + minItems: 1 + maxItems: 2 + minimum: 0 CommandOutput: type: object From dacad31b5bbd039b8ffc97adb70696655d0872ad Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Fri, 17 May 2024 11:05:39 +0200 Subject: [PATCH 2/3] Reintroduce progress changes --- model/Cargo.toml | 1 + model/src/activity.rs | 2 +- model/src/activity/exe_script_command.rs | 25 ++++++ model/src/activity/runtime_event.rs | 20 +++++ specs/activity-api.yaml | 97 +++++++++++++++++++++--- 5 files changed, 133 insertions(+), 12 deletions(-) diff --git a/model/Cargo.toml b/model/Cargo.toml index 77ac2ff2..5c58e3a1 100644 --- a/model/Cargo.toml +++ b/model/Cargo.toml @@ -17,6 +17,7 @@ sgx = ['secp256k1', 'openssl', 'hex', 'secp256k1/serde'] bigdecimal = { version = "0.2", features = ["serde"]} chrono = { version = "0.4", features = ["serde"]} derive_more = "0.99" +humantime-serde = "1.1" rand = "0.8" serde = { version = "1.0.146", features = ["derive"] } serde_with = { version = "3" } diff --git a/model/src/activity.rs b/model/src/activity.rs index 2c1a2ae0..76558435 100644 --- a/model/src/activity.rs +++ b/model/src/activity.rs @@ -21,7 +21,7 @@ pub use self::exe_script_command_result::{CommandOutput, CommandResult, ExeScrip pub use self::exe_script_command_state::ExeScriptCommandState; pub use self::exe_script_request::ExeScriptRequest; pub use self::provider_event::ProviderEvent; -pub use self::runtime_event::{RuntimeEvent, RuntimeEventKind}; +pub use self::runtime_event::{CommandProgress, RuntimeEvent, RuntimeEventKind}; #[cfg(feature = "sgx")] pub use self::sgx_credentials::SgxCredentials; diff --git a/model/src/activity/exe_script_command.rs b/model/src/activity/exe_script_command.rs index df72e5c2..c7c4c67a 100644 --- a/model/src/activity/exe_script_command.rs +++ b/model/src/activity/exe_script_command.rs @@ -11,6 +11,7 @@ use crate::activity::ExeScriptCommandState; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::time::Duration; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -21,6 +22,18 @@ pub enum ExeScriptCommand { net: Vec, #[serde(default)] hosts: HashMap, // hostname -> IP + + #[serde(default)] + hostname: Option, + + #[serde(default)] + volumes: Vec, + + #[serde(default)] + env: HashMap, + + #[serde(default)] + progress: Option, }, Start { #[serde(default)] @@ -39,6 +52,8 @@ pub enum ExeScriptCommand { to: String, #[serde(flatten)] args: TransferArgs, + #[serde(default)] + progress: Option, }, Terminate {}, } @@ -107,6 +122,16 @@ pub struct TransferArgs { pub fileset: Option, } +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct ProgressArgs { + #[serde(default)] + #[serde(with = "humantime_serde")] + pub update_interval: Option, + /// Number of bytes after which next progress event will be sent. + pub update_step: Option, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum FileSet { diff --git a/model/src/activity/runtime_event.rs b/model/src/activity/runtime_event.rs index a1890047..2efeb8f7 100644 --- a/model/src/activity/runtime_event.rs +++ b/model/src/activity/runtime_event.rs @@ -47,6 +47,10 @@ impl RuntimeEvent { pub fn stderr(batch_id: String, idx: usize, out: CommandOutput) -> Self { Self::new(batch_id, idx, RuntimeEventKind::StdErr(out)) } + + pub fn progress(batch_id: String, idx: usize, progress: CommandProgress) -> Self { + Self::new(batch_id, idx, RuntimeEventKind::Progress(progress)) + } } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -61,4 +65,20 @@ pub enum RuntimeEventKind { }, StdOut(CommandOutput), StdErr(CommandOutput), + Progress(CommandProgress), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub struct CommandProgress { + /// Steps are counted starting from 0. That means that first step from 4-steps tasks + /// will report 0/4. Task is finished when counter reaches 4/4. + pub step: (usize, usize), + /// May contain additional arbitrary information about, what is happening with the task + /// like retrying transfer or that image was deployed from cache. + pub message: Option, + /// Granular progress of currently executed step. The first element describes current + /// progress, the second the size of the whole task, which can be unknown. + pub progress: (u64, Option), + pub unit: Option, } diff --git a/specs/activity-api.yaml b/specs/activity-api.yaml index b5393c02..925d67bb 100644 --- a/specs/activity-api.yaml +++ b/specs/activity-api.yaml @@ -486,7 +486,8 @@ components: "deploy": { "net": [{ "id": "id", "ip": "10.0.0.2", "mask": "255.255.0.0" }], "hosts": {"master": "10.0.0.1"}, - "nodes": {"10.0.0.1": "0xdeadbeef"} + "nodes": {"10.0.0.1": "0xdeadbeef"}, + "progress" : {"updateInterval": "300ms", "updateStep": null} } }, { @@ -500,7 +501,8 @@ components: "to": "container:/input/file_in", "format": "zip", "depth": 2, - "fileset": [{"desc":"all images", "includes": ["*.jpg"], "excludes": ["db*.*"] }] + "fileset": [{"desc":"all images", "includes": ["*.jpg"], "excludes": ["db*.*"] }], + "progress" : {"updateInterval": null, "updateStep": 1048576} } }, { @@ -557,6 +559,8 @@ components: type: object additionalProperties: type: string + progress: + $ref: '#/components/schemas/ProgressArgs' DeployNetwork: type: object @@ -572,6 +576,25 @@ components: mask: type: string + ProgressArgs: + type: object + description: Configuration of progress reporting. + Presence of this field in ExeUnitCommand indicates, that ExeUnit should send + '#/components/schemas/RuntimeEventKindProgress' events. If non of properties is set + ExeUnit will use default values. + Behavior when both properties are defined is ExeUnit specific. + properties: + update-interval: + type: string + description: Interval between progress reporting events expressed as + described in specification https://docs.rs/humantime/latest/humantime/fn.parse_duration.html + update-step: + type: number + format: int64 + minimum: 1 + description: Number of units (for example Bytes in case of transfer) after which next + progress event will be sent. + StartCommand: allOf: - $ref: '#/components/schemas/ExeScriptCommand' @@ -616,6 +639,8 @@ components: type: array items: $ref: '#/components/schemas/FileSet' + progress: + $ref: '#/components/schemas/ProgressArgs' FileSet: properties: @@ -842,15 +867,65 @@ components: $ref: '#/components/schemas/CommandOutput' RuntimeEventKindFinishedBody: - type: object - required: - - returnCode - properties: - returnCode: - type: integer - format: int32 - message: - type: string + allOf: + - $ref: '#/components/schemas/RuntimeEventKind' + - type: object + required: + - returnCode + properties: + returnCode: + type: integer + format: int32 + message: + type: string + + RuntimeEventKindProgress: + allOf: + - $ref: '#/components/schemas/RuntimeEventKind' + - type: object + description: Reports progress of currently executed command. This event will be sent only, + if `progress` field was set in `deploy` or `transfer` command. + required: + - step + properties: + step: + $ref: '#/components/schemas/ProgressStep' + message: + type: string + description: May contain additional arbitrary information, what is happening with the task, + for example "retrying transfer" or "image deployed from cache". + Content of this field is ExeUnit specific. + progress: + $ref: '#/components/schemas/ProgressDetails' + unit: + type: string + description: Units in which `progress` field is expressed. This should be human readable + string for displaying in UI. List of possible values is ExeUnit specific. + + ProgressStep: + description: Can be used if single ExeUnit command is executing multiple steps. Number of steps is + Exeunit specific. + + Steps are counted starting from 0. That means that first step from 4-steps task + will report 0/4. Task is finished when counter reaches 4/4. + type: array + items: + type: integer + format: int64 + minItems: 2 + maxItems: 2 + minimum: 0 + + ProgressDetails: + description: Granular progress of currently executed step. The first element describes current + progress, the second the size of the whole task, which can be unknown. + type: array + items: + type: integer + format: int64 + minItems: 1 + maxItems: 2 + minimum: 0 CommandOutput: type: object From 56d87785fca50edf7536b0942a19aa074749f62f Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Thu, 12 Sep 2024 11:31:53 +0200 Subject: [PATCH 3/3] activity: rust buggy warning workaround --- src/activity/requestor/control.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/activity/requestor/control.rs b/src/activity/requestor/control.rs index e3ddb7c2..415c5be3 100644 --- a/src/activity/requestor/control.rs +++ b/src/activity/requestor/control.rs @@ -69,7 +69,12 @@ impl ActivityRequestorControlApi { /// Destroys given Activity. pub async fn destroy_activity(&self, activity_id: &str) -> Result<()> { let uri = url_format!("activity/{activity_id}"); - self.client.delete(&uri).send().json().await?; + + // Specify serialization target because of a rustc bug falsely + // emitting dependency_on_unit_never_type_fallback. + // If some time has passed try removing ::<()> and see if it still + // emits a warning. + self.client.delete(&uri).send().json::<()>().await?; Ok(()) }