From fe6bf2c0f637363e0134d396f15f3d1afb69a444 Mon Sep 17 00:00:00 2001 From: "henry.hj" Date: Mon, 29 Nov 2021 20:00:59 +0800 Subject: [PATCH 1/2] nydusify: update examples for chunk-dict Signed-off-by: henry.hj --- .../examples/manifest/cache_manifest.json | 17 ++++++++++++++--- .../nydusify/examples/manifest/manifest.json | 13 +++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/contrib/nydusify/examples/manifest/cache_manifest.json b/contrib/nydusify/examples/manifest/cache_manifest.json index 03786443ab4..58c1ad65ba4 100644 --- a/contrib/nydusify/examples/manifest/cache_manifest.json +++ b/contrib/nydusify/examples/manifest/cache_manifest.json @@ -7,6 +7,14 @@ "size": 76 }, "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.nydus.blob.v1", + "digest": "sha256:09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059", + "size": 83528010, + "annotations": { + "containerd.io/snapshot/nydus-blob": "true" + } + }, { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:fdfe86772cfb157dd364a7caf7a64fdc6f10abd047701c0a3fcd629b8ebc8766", @@ -14,7 +22,8 @@ "annotations": { "containerd.io/snapshot/nydus-bootstrap": "true", "containerd.io/snapshot/nydus-source-chainid": "sha256:bacd3af13903e13a43fe87b6944acd1ff21024132aad6e74b4452d984fb1a99a", - "containerd.io/uncompressed": "sha256:032ef23acc516fb5ffda4900db1616f85b39cffb626bc0def51915e14a6a7d8d" + "containerd.io/uncompressed": "sha256:032ef23acc516fb5ffda4900db1616f85b39cffb626bc0def51915e14a6a7d8d", + "containerd.io/snapshot/nydus-reference-blob-ids": "[\"09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059\"]" } }, { @@ -33,7 +42,8 @@ "annotations": { "containerd.io/snapshot/nydus-bootstrap": "true", "containerd.io/snapshot/nydus-source-chainid": "sha256:3779241fda7b1caf03964626c3503e930f2f19a5ffaba6f4b4ad21fd38df3b6b", - "containerd.io/uncompressed": "sha256:06014764637029de0a5d37c5a2e52249d46f45a5edecca0ad81d98347f076d7a" + "containerd.io/uncompressed": "sha256:06014764637029de0a5d37c5a2e52249d46f45a5edecca0ad81d98347f076d7a", + "containerd.io/snapshot/nydus-reference-blob-ids": "[\"09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059\"]" } }, { @@ -52,7 +62,8 @@ "annotations": { "containerd.io/snapshot/nydus-bootstrap": "true", "containerd.io/snapshot/nydus-source-chainid": "sha256:9386795d450ce06c6819c8bc5eff8daa71d47ccb9f9fb8d49fe1ccfb5fb3edbe", - "containerd.io/uncompressed": "sha256:e3a229c2fa7d489240052abe2f9ad235e26f1aa10d70060fc8e78d478b624503" + "containerd.io/uncompressed": "sha256:e3a229c2fa7d489240052abe2f9ad235e26f1aa10d70060fc8e78d478b624503", + "containerd.io/snapshot/nydus-reference-blob-ids": "[\"09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059\"]" } }, { diff --git a/contrib/nydusify/examples/manifest/manifest.json b/contrib/nydusify/examples/manifest/manifest.json index 4e643252fc9..ec70a612a9b 100644 --- a/contrib/nydusify/examples/manifest/manifest.json +++ b/contrib/nydusify/examples/manifest/manifest.json @@ -6,6 +6,14 @@ "size": 523 }, "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.nydus.blob.v1", + "digest": "sha256:09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059", + "size": 83528010, + "annotations": { + "containerd.io/snapshot/nydus-blob": "true" + } + }, { "mediaType": "application/vnd.oci.image.layer.nydus.blob.v1", "digest": "sha256:b413839e4ee5248697ef30fe9a84b659fa744d69bbc9b7754113adc2b2b6bc90", @@ -35,8 +43,9 @@ "digest": "sha256:aec98c9e3dce739877b8f5fe1cddd339de1db2b36c20995d76f6265056dbdb08", "size": 273320, "annotations": { - "containerd.io/snapshot/nydus-blob-ids": "[\"b413839e4ee5248697ef30fe9a84b659fa744d69bbc9b7754113adc2b2b6bc90\",\"b6a85be8248b0d3c2f0565ef71d549f404f8edcee1ab666c9871a8e6d9384860\",\"00d151e7d392e68e2c756a6fc42640006ddc0a98d37dba3f90a7b73f63188bbd\"]", - "containerd.io/snapshot/nydus-bootstrap": "true" + "containerd.io/snapshot/nydus-blob-ids": "[\"09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059\",\"b413839e4ee5248697ef30fe9a84b659fa744d69bbc9b7754113adc2b2b6bc90\",\"b6a85be8248b0d3c2f0565ef71d549f404f8edcee1ab666c9871a8e6d9384860\",\"00d151e7d392e68e2c756a6fc42640006ddc0a98d37dba3f90a7b73f63188bbd\"]", + "containerd.io/snapshot/nydus-bootstrap": "true", + "containerd.io/snapshot/nydus-reference-blob-ids": "[\"09845cce1d983b158d4865fc37c23bbfb892d4775c786e8114d3cf868975c059\"]" } } ] From ee61d0f1102776dda9159652ca45430adadcc834 Mon Sep 17 00:00:00 2001 From: "henry.hj" Date: Mon, 14 Feb 2022 17:24:59 +0800 Subject: [PATCH 2/2] docs: update docs of nydus-image & nydusify 1. nydus-image: chunk-dict, compact subcommand 2. nydusify: pack subcommand Signed-off-by: henry.hj --- docs/nydus-image.md | 73 ++++++++++++++++++++++++++++++++++++++++----- docs/nydusify.md | 35 +++++++++++++++++++++- 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/docs/nydus-image.md b/docs/nydus-image.md index 93a3e30bef6..a22c697afea 100644 --- a/docs/nydus-image.md +++ b/docs/nydus-image.md @@ -13,8 +13,8 @@ Get `nydus-image` binary from [release](https://github.com/dragonflyoss/image-se ```shell nydus-image create \ - --bootstrap /path/to/bootstrap - --blob /path/to/blob + --bootstrap /path/to/bootstrap \ + --blob /path/to/blob \ /path/to/source/dir ``` @@ -35,17 +35,76 @@ Generally, this is regular file which blob content will be dumped into. It can a ```shell # Build from lower layer nydus-image create \ - --bootstrap /path/to/parent-bootstrap - --blob /path/to/blob + --bootstrap /path/to/parent-bootstrap \ + --blob /path/to/blob \ /path/to/lower/dir # Build from upper layer based on lower layer nydus-image create \ - --parent-bootstrap /path/to/parent-bootstrap - --bootstrap /path/to/bootstrap - --blob /path/to/blob + --parent-bootstrap /path/to/parent-bootstrap \ + --bootstrap /path/to/bootstrap \ + --blob /path/to/blob \ /path/to/upper/dir ``` +## Build Nydus Image With Chunk-Dict +`nydus-image` tool supports to build Nydus image with chunk-dict for chunk deduplication: +1. reference chunks which are same as chunks in chunk-dict to blobs in chunk-dict +2. new dumped blob would be smaller than without using chunk-dict +3. save space of remote storage because of chunk-deduplication between images (e.g. oss, registry) +```shell +# Build with bootstrap type chunk-dict +nydus-image create \ + --bootstrap /path/to/parent-bootstrap \ + --chunk-dict bootstrap=/path/to/dict.boot \ + --blob /path/to/blob \ + /path/to/lower/dir +``` + +## Compact Nydus Image +`nydus-image` tool supports to compact Nydus image for +1. reduce number of blobs +2. optimize size of blobs (remove unused chunks in blob, merge small blobs) +```shell +# backend config for getting chunk data from remote storage +# e.g. OSS backend config +cat /path/to/backend-config.json +{ + "endpoint": "region.aliyuncs.com", + "access_key_id": "", + "access_key_secret": "", + "bucket_name": "" +} + +# min_used_ratio: +# rebuild blobs whose used_ratio < min_used_ratio +# used_ratio = (compress_size of all chunks which are referenced by bootstrap) / blob_compress_size +# available value: 0-99, 0 means disable +# compact_blob_size: +# we only compact blob whose compress_size < compact_blob_size +# max_compact_size: +# final merged blob compress_size <= max_compact_size +# layers_to_compact: +# if number of blobs >= layers_to_compact, try compact nydus image +# 0 means always try compact +cat /path/to/compact.json +{ + "min_used_ratio": 10, + "compact_blob_size": 10485760, + "max_compact_size": 104857600, + "layers_to_compact": 32, + "blobs_dir": "/path/to/blobs" +} + +# Compact Nydus image with chunk-dict +nydus-image create \ + --bootstrap /path/to/bootstrap \ + --chunk-dict bootstrap=/path/to/dict \ + --config /path/to/compact.json \ + --backend-config-file /path/to/backend-config.json \ + --backend-type oss \ + /path/to/lower/dir +``` + ## Build Nydus Image From Stargz Index ### Convert image layer to stargz format diff --git a/docs/nydusify.md b/docs/nydusify.md index 5f6452adfa9..f12326cea66 100644 --- a/docs/nydusify.md +++ b/docs/nydusify.md @@ -1,6 +1,8 @@ # Nydusify -The Nydusify CLI tool converts an OCI container image from source registry into a Nydus image using `nydus-image` CLI layer by layer, then pushes Nydus image to target registry. +The Nydusify CLI tool supports: +1. Convert an OCI container image from source registry into a Nydus image using `nydus-image` CLI layer by layer, then pushes Nydus image to target registry. +2. Convert local file system dictionary into Nydus image using `nydus-image`, then push Nydus-image to target remote storage(e.g. oss) optionally. ### Get binaries from release page @@ -8,11 +10,19 @@ Get `nydus-image`, `nydusd` and `nydusify` binaries from [release](https://githu ## Basic Usage +Convert oci image: ``` nydusify convert \ --source myregistry/repo:tag \ --target myregistry/repo:tag-nydus ``` +Pack local file system dictionary: +``` +nydusify pack \ + --bootstrap target.bootstrap \ + --target-dir /path/to/target \ + --output-dir /path/to/output +``` ## Upload blob to storage backend @@ -38,6 +48,29 @@ nydusify convert \ --backend-config-file /path/to/backend-config.json ``` +## Push Nydus Image to OSS with subcommand pack +``` shell +# meta_prefix: +# push bootstrap into oss://$bucket_name/$meta_prefix/$bootstrap_name +# blob_prefix: +# push blobs into oss://$bucket_name/$blob_prefix/$blob_id +cat /path/to/backend-config.json +{ + "bucket_name": "", + "endpoint": "region.aliyuncs.com", + "access_key_id": "", + "access_key_secret": "", + "meta_prefix": "meta", + "blob_prefix": "blob" +} + +nydusify pack --bootstrap target.bootstrap \ + --backend-push \ + --backend-config-file /path/to/backend-config.json \ + --target-dir /path/to/target \ + --output-dir /path/to/output +``` + ## Check Nydus image Nydusify provides a checker to validate Nydus image, the checklist includes image manifest, Nydus bootstrap, file metadata, and data consistency in rootfs with the original OCI image. Meanwhile, the checker dumps OCI & Nydus image information to `output` (default) directory.