Various updates for --progress-fd

docs/src/SUMMARY.md

@@ -55,6 +55,7 @@
 # Experimental features
 
 - [bootc image](experimental-bootc-image.md)
+- [--progress-fd](experimental-progress-fd.md)
 
 # More information
 

docs/src/bootc-via-api.md

@@ -7,7 +7,7 @@ are stable and will not change.
 ## Using `bootc edit` and `bootc status --json`
 
 While bootc does not depend on Kubernetes, it does currently
-also offere a Kubernetes *style* API, especially oriented
+also offer a Kubernetes *style* API, especially oriented
 towards the [spec and status and other conventions](https://kubernetes.io/docs/reference/using-api/api-concepts/).
 
 In general, most use cases of driving bootc via API are probably

docs/src/experimental-progress-fd.md

@@ -0,0 +1,32 @@
+
+# Interactive progress with `--progress-fd`
+
+This is an experimental feature; tracking issue: <https://github.com/containers/bootc/issues/1016>
+
+While the `bootc status` tooling allows a client to discover the state
+of the system, during interactive changes such as `bootc upgrade`
+or `bootc switch` it is possible to monitor the status of downloads
+or other operations at a fine-grained level with `-progress-fd`.
+
+The format of data output over `--progress-fd` is [JSON Lines](https://jsonlines.org)
+which is a series of JSON objects separated by newlines (the intermediate
+JSON content is guaranteed not to contain a literal newline).
+
+You can find the JSON schema describing this version here:
+[progress-v0.schema.json](progress-v0.schema.json).
+
+Deploying a new image with either switch or upgrade consists
+of three stages: `pulling`, `importing`, and `staging`. The `pulling` step
+downloads the image from the registry, offering per-layer and progress in
+each message. The `importing` step imports the image into storage and consists
+of a single step. Finally, `staging` runs a variety of staging
+tasks. Currently, they are staging the image to disk, pulling bound images,
+and removing old images.
+
+Note that new stages or fields may be added at any time.
+
+Importing and staging are affected by disk speed and the total image size. Pulling
+is affected by network speed and how many layers invalidate between pulls.
+Therefore, a large image with a good caching strategy will have longer
+importing and staging times, and a small bespoke container image will have
+negligible importing and staging times.

docs/src/progress-v0.schema.json

@@ -0,0 +1,239 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Event",
+  "description": "An event emitted as JSON.",
+  "oneOf": [
+    {
+      "type": "object",
+      "required": [
+        "type",
+        "version"
+      ],
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "Start"
+          ]
+        },
+        "version": {
+          "description": "The semantic version of the progress protocol.",
+          "type": "string"
+        }
+      }
+    },
+    {
+      "description": "An incremental update to a container image layer download",
+      "type": "object",
+      "required": [
+        "bytes",
+        "bytes_cached",
+        "bytes_total",
+        "description",
+        "id",
+        "steps",
+        "steps_cached",
+        "steps_total",
+        "subtasks",
+        "task",
+        "type"
+      ],
+      "properties": {
+        "bytes": {
+          "description": "The number of bytes already fetched.",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "bytes_cached": {
+          "description": "The number of bytes fetched by a previous run.",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "bytes_total": {
+          "description": "Total number of bytes. If zero, then this should be considered \"unspecified\".",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "description": {
+          "description": "A human readable description of the task if i18n is not available.",
+          "type": "string"
+        },
+        "id": {
+          "description": "A human and machine readable unique identifier for the task (e.g., the image name). For tasks that only happen once, it can be set to the same value as task.",
+          "type": "string"
+        },
+        "steps": {
+          "description": "The initial position of progress.",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "steps_cached": {
+          "description": "The number of steps fetched by a previous run.",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "steps_total": {
+          "description": "The total number of steps (e.g. container image layers, RPMs)",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "subtasks": {
+          "description": "The currently running subtasks.",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/SubTaskBytes"
+          }
+        },
+        "task": {
+          "description": "A machine readable type (e.g., pulling) for the task (used for i18n and UI customization).",
+          "type": "string"
+        },
+        "type": {
+          "type": "string",
+          "enum": [
+            "ProgressBytes"
+          ]
+        }
+      }
+    },
+    {
+      "description": "An incremental update with discrete steps",
+      "type": "object",
+      "required": [
+        "description",
+        "id",
+        "steps",
+        "steps_cached",
+        "steps_total",
+        "subtasks",
+        "task",
+        "type"
+      ],
+      "properties": {
+        "description": {
+          "description": "A human readable description of the task if i18n is not available.",
+          "type": "string"
+        },
+        "id": {
+          "description": "A human and machine readable unique identifier for the task (e.g., the image name). For tasks that only happen once, it can be set to the same value as task.",
+          "type": "string"
+        },
+        "steps": {
+          "description": "The initial position of progress.",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "steps_cached": {
+          "description": "The number of steps fetched by a previous run.",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "steps_total": {
+          "description": "The total number of steps (e.g. container image layers, RPMs)",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "subtasks": {
+          "description": "The currently running subtasks.",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/SubTaskStep"
+          }
+        },
+        "task": {
+          "description": "A machine readable type (e.g., pulling) for the task (used for i18n and UI customization).",
+          "type": "string"
+        },
+        "type": {
+          "type": "string",
+          "enum": [
+            "ProgressSteps"
+          ]
+        }
+      }
+    }
+  ],
+  "definitions": {
+    "SubTaskBytes": {
+      "description": "An incremental update to e.g. a container image layer download. The first time a given \"subtask\" name is seen, a new progress bar should be created. If bytes == bytes_total, then the subtask is considered complete.",
+      "type": "object",
+      "required": [
+        "bytes",
+        "bytesCached",
+        "bytesTotal",
+        "description",
+        "id",
+        "subtask"
+      ],
+      "properties": {
+        "bytes": {
+          "description": "Updated byte level progress",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "bytesCached": {
+          "description": "The number of bytes fetched by a previous run (e.g., zstd_chunked).",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "bytesTotal": {
+          "description": "Total number of bytes",
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "description": {
+          "description": "A human readable description of the task if i18n is not available. (e.g., \"OSTree Chunk:\", \"Derived Layer:\")",
+          "type": "string"
+        },
+        "id": {
+          "description": "A human and machine readable identifier for the task (e.g., ostree chunk/layer hash).",
+          "type": "string"
+        },
+        "subtask": {
+          "description": "A machine readable type for the task (used for i18n). (e.g., \"ostree_chunk\", \"ostree_derived\")",
+          "type": "string"
+        }
+      }
+    },
+    "SubTaskStep": {
+      "description": "Marks the beginning and end of a dictrete step",
+      "type": "object",
+      "required": [
+        "completed",
+        "description",
+        "id",
+        "subtask"
+      ],
+      "properties": {
+        "completed": {
+          "description": "Starts as false when beginning to execute and turns true when completed.",
+          "type": "boolean"
+        },
+        "description": {
+          "description": "A human readable description of the task if i18n is not available. (e.g., \"OSTree Chunk:\", \"Derived Layer:\")",
+          "type": "string"
+        },
+        "id": {
+          "description": "A human and machine readable identifier for the task (e.g., ostree chunk/layer hash).",
+          "type": "string"
+        },
+        "subtask": {
+          "description": "A machine readable type for the task (used for i18n). (e.g., \"ostree_chunk\", \"ostree_derived\")",
+          "type": "string"
+        }
+      }
+    }
+  }
+}

lib/src/cli.rs

@@ -37,16 +37,16 @@ pub(crate) struct ProgressOptions {
     ///
     /// Interactive progress will be written to this file descriptor as "JSON lines"
     /// format, where each value is separated by a newline.
-    #[clap(long)]
-    pub(crate) json_fd: Option<RawProgressFd>,
+    #[clap(long, hide = true)]
+    pub(crate) progress_fd: Option<RawProgressFd>,
 }
 
 impl TryFrom<ProgressOptions> for ProgressWriter {
     type Error = anyhow::Error;
 
     fn try_from(value: ProgressOptions) -> Result<Self> {
         let r = value
-            .json_fd
+            .progress_fd
             .map(TryInto::try_into)
             .transpose()?
             .unwrap_or_default();
@@ -359,6 +359,12 @@ pub(crate) enum ImageOpts {
     Cmd(ImageCmdOpts),
 }
 
+#[derive(Debug, Clone, clap::ValueEnum, PartialEq, Eq)]
+pub(crate) enum SchemaType {
+    Host,
+    Progress,
+}
+
 /// Hidden, internal only options
 #[derive(Debug, clap::Subcommand, PartialEq, Eq)]
 pub(crate) enum InternalsOpts {
@@ -371,7 +377,10 @@ pub(crate) enum InternalsOpts {
     },
     FixupEtcFstab,
     /// Should only be used by `make update-generated`
-    PrintJsonSchema,
+    PrintJsonSchema {
+        #[clap(long)]
+        of: SchemaType,
+    },
     /// Perform cleanup actions
     Cleanup,
     /// Proxy frontend for the `ostree-ext` CLI.
@@ -1090,8 +1099,11 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
                 .await
             }
             InternalsOpts::FixupEtcFstab => crate::deploy::fixup_etc_fstab(&root),
-            InternalsOpts::PrintJsonSchema => {
-                let schema = schema_for!(crate::spec::Host);
+            InternalsOpts::PrintJsonSchema { of } => {
+                let schema = match of {
+                    SchemaType::Host => schema_for!(crate::spec::Host),
+                    SchemaType::Progress => schema_for!(crate::progress_jsonl::Event),
+                };
                 let mut stdout = std::io::stdout().lock();
                 serde_json::to_writer_pretty(&mut stdout, &schema)?;
                 Ok(())