From 175b63fb0c6143ddaefe65ca6e676d2dc088dade Mon Sep 17 00:00:00 2001 From: xiechengsheng Date: Thu, 12 Jul 2018 15:36:17 +0800 Subject: [PATCH] feature: add pouch save functionality Signed-off-by: xiechengsheng --- apis/server/image_bridge.go | 21 +++++++++ apis/server/router.go | 1 + apis/swagger.yml | 23 ++++++++++ cli/main.go | 1 + cli/save.go | 82 ++++++++++++++++++++++++++++++++++ client/image_save.go | 19 ++++++++ client/image_save_test.go | 56 +++++++++++++++++++++++ client/interface.go | 1 + ctrd/image.go | 37 +++++++++++++++ ctrd/interface.go | 2 + daemon/mgr/image.go | 3 ++ daemon/mgr/image_load.go | 2 +- daemon/mgr/image_save.go | 23 ++++++++++ test/cli_load_test.go | 22 --------- test/cli_save_and_load_test.go | 63 ++++++++++++++++++++++++++ 15 files changed, 333 insertions(+), 23 deletions(-) create mode 100644 cli/save.go create mode 100644 client/image_save.go create mode 100644 client/image_save_test.go create mode 100644 daemon/mgr/image_save.go delete mode 100644 test/cli_load_test.go create mode 100644 test/cli_save_and_load_test.go diff --git a/apis/server/image_bridge.go b/apis/server/image_bridge.go index 79196899f8..3452aa55fb 100644 --- a/apis/server/image_bridge.go +++ b/apis/server/image_bridge.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "net/http" "strings" "time" @@ -151,3 +152,23 @@ func (s *Server) loadImage(ctx context.Context, rw http.ResponseWriter, req *htt rw.WriteHeader(http.StatusOK) return nil } + +// saveImage saves an image by http tar stream. +func (s *Server) saveImage(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + imageName := req.FormValue("name") + + rw.Header().Set("Content-Type", "application/x-tar") + + r, err := s.ImageMgr.SaveImage(ctx, imageName) + if err != nil { + return err + } + defer r.Close() + + output := newWriteFlusher(rw) + if _, err := io.Copy(output, r); err != nil { + return err + } + + return nil +} diff --git a/apis/server/router.go b/apis/server/router.go index b5c7902694..4213bb6042 100644 --- a/apis/server/router.go +++ b/apis/server/router.go @@ -63,6 +63,7 @@ func initRoute(s *Server) http.Handler { s.addRoute(r, http.MethodGet, "/images/{name:.*}/json", s.getImage) s.addRoute(r, http.MethodPost, "/images/{name:.*}/tag", s.postImageTag) s.addRoute(r, http.MethodPost, "/images/load", withCancelHandler(s.loadImage)) + s.addRoute(r, http.MethodGet, "/images/save", withCancelHandler(s.saveImage)) // volume s.addRoute(r, http.MethodGet, "/volumes", s.listVolume) diff --git a/apis/swagger.yml b/apis/swagger.yml index 00fa5b9ec8..ac349b1ec4 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -172,6 +172,29 @@ paths: description: "set the image name for the tar stream, default unknown/unknown" type: "string" + /images/save: + get: + summary: "Save image" + description: | + Save an image by oci.v1 format tar stream. + produces: + - application/x-tar + responses: + 200: + description: "no error" + schema: + type: "string" + format: "binary" + 404: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + parameters: + - name: "name" + in: "query" + description: "Image name which is to be saved" + type: "string" + /images/{imageid}/json: get: summary: "Inspect a image" diff --git a/cli/main.go b/cli/main.go index 7f1c41e0e6..2f6df1f263 100644 --- a/cli/main.go +++ b/cli/main.go @@ -31,6 +31,7 @@ func main() { cli.AddCommand(base, &NetworkCommand{}) cli.AddCommand(base, &TagCommand{}) cli.AddCommand(base, &LoadCommand{}) + cli.AddCommand(base, &SaveCommand{}) cli.AddCommand(base, &InspectCommand{}) cli.AddCommand(base, &RenameCommand{}) diff --git a/cli/save.go b/cli/save.go new file mode 100644 index 0000000000..8cd2d12715 --- /dev/null +++ b/cli/save.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "io" + "os" + + "github.com/spf13/cobra" +) + +// saveDescription is used to describe save command in detail and auto generate command doc. +var saveDescription = "save an image to a tar archive." + +// SaveCommand use to implement 'save' command. +type SaveCommand struct { + baseCommand + output string +} + +// Init initialize save command. +func (save *SaveCommand) Init(c *Cli) { + save.cli = c + save.cmd = &cobra.Command{ + Use: "save [OPTIONS] IMAGE", + Short: "Save an image to a tar archive or STDOUT", + Long: saveDescription, + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return save.runSave(args) + }, + Example: saveExample(), + } + save.addFlags() +} + +// addFlags adds flags for specific command. +func (save *SaveCommand) addFlags() { + flagSet := save.cmd.Flags() + flagSet.StringVarP(&save.output, "output", "o", "", "Save to a tar archive file, instead of STDOUT") +} + +// runSave is the entry of save command. +func (save *SaveCommand) runSave(args []string) error { + ctx := context.Background() + apiClient := save.cli.Client() + + r, err := apiClient.ImageSave(ctx, args[0]) + if err != nil { + return err + } + defer r.Close() + + out := os.Stdout + if save.output != "" { + out, err = os.Create(save.output) + if err != nil { + return nil + } + defer out.Close() + } + + if _, err := io.Copy(out, r); err != nil { + return err + } + if save.output != "" { + if err := out.Close(); err != nil { + return err + } + } + return nil +} + +// saveExample shows examples in save command, and is used in auto-generated cli docs. +func saveExample() string { + return `$ pouch save -o busybox.tar busybox:latest +$ pouch load -i busybox.tar foo +$ pouch images +IMAGE ID IMAGE NAME SIZE +8c811b4aec35 registry.hub.docker.com/library/busybox:latest 710.81 KB +8c811b4aec35 foo:latest 710.81 KB +` +} diff --git a/client/image_save.go b/client/image_save.go new file mode 100644 index 0000000000..54bc9330b3 --- /dev/null +++ b/client/image_save.go @@ -0,0 +1,19 @@ +package client + +import ( + "context" + "io" + "net/url" +) + +// ImageSave requests daemon to save an image to a tar archive. +func (client *APIClient) ImageSave(ctx context.Context, imageName string) (io.ReadCloser, error) { + q := url.Values{} + q.Set("name", imageName) + + resp, err := client.get(ctx, "/images/save", q, nil) + if err != nil { + return nil, err + } + return resp.Body, nil +} diff --git a/client/image_save_test.go b/client/image_save_test.go new file mode 100644 index 0000000000..da71f61885 --- /dev/null +++ b/client/image_save_test.go @@ -0,0 +1,56 @@ +package client + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "net/http" + "strings" + "testing" +) + +func TestImageSaveServerError(t *testing.T) { + expectedError := "Server error" + + client := &APIClient{ + HTTPCli: newMockClient(errorMockResponse(http.StatusInternalServerError, expectedError)), + } + + _, err := client.ImageSave(context.Background(), "test_image_save_500") + if err == nil || !strings.Contains(err.Error(), expectedError) { + t.Fatalf("expected (%v), got (%v)", expectedError, err) + } +} + +func TestImageSaveOK(t *testing.T) { + expectedImageName := "test_image_save_ok" + expectedURL := "/images/save" + + httpClient := newMockClient(func(req *http.Request) (*http.Response, error) { + if !strings.HasPrefix(req.URL.Path, expectedURL) { + return nil, fmt.Errorf("expected URL '%s', got '%s'", expectedURL, req.URL) + } + + if req.Method != "GET" { + return nil, fmt.Errorf("expected GET method, got %s", req.Method) + } + + if got := req.FormValue("name"); got != expectedImageName { + return nil, fmt.Errorf("expected (%s), got %s", expectedImageName, got) + } + + return &http.Response{ + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(bytes.NewReader([]byte(""))), + }, nil + }) + + client := &APIClient{ + HTTPCli: httpClient, + } + + if _, err := client.ImageSave(context.Background(), expectedImageName); err != nil { + t.Fatal(err) + } +} diff --git a/client/interface.go b/client/interface.go index 15f50738b7..96cea78924 100644 --- a/client/interface.go +++ b/client/interface.go @@ -50,6 +50,7 @@ type ImageAPIClient interface { ImageRemove(ctx context.Context, name string, force bool) error ImageTag(ctx context.Context, image string, tag string) error ImageLoad(ctx context.Context, name string, r io.Reader) error + ImageSave(ctx context.Context, imageName string) (io.ReadCloser, error) } // VolumeAPIClient defines methods of Volume client. diff --git a/ctrd/image.go b/ctrd/image.go index 46649245c2..863726927d 100644 --- a/ctrd/image.go +++ b/ctrd/image.go @@ -10,6 +10,7 @@ import ( "github.com/alibaba/pouch/apis/types" "github.com/alibaba/pouch/pkg/jsonstream" + "github.com/alibaba/pouch/pkg/reference" "github.com/containerd/containerd" "github.com/containerd/containerd/errdefs" @@ -65,6 +66,42 @@ func (c *Client) RemoveImage(ctx context.Context, ref string) error { return nil } +// SaveImage saves image to tarstream +func (c *Client) SaveImage(ctx context.Context, exporter ctrdmetaimages.Exporter, ref string) (io.ReadCloser, error) { + wrapperCli, err := c.Get(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get a containerd grpc client: %v", err) + } + + image, err := c.GetImage(ctx, ref) + if err != nil { + return nil, err + } + + desc := image.Target() + // add annotations in image description + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + if s, exist := desc.Annotations[ocispec.AnnotationRefName]; !exist || s == "" { + namedRef, err := reference.Parse(ref) + if err != nil { + return nil, err + } + + if reference.IsNameTagged(namedRef) { + desc.Annotations[ocispec.AnnotationRefName] = namedRef.(reference.Tagged).Tag() + } + } + + exportedStream, err := wrapperCli.client.Export(ctx, exporter, desc) + if err != nil { + return nil, err + } + + return exportedStream, nil +} + // ImportImage creates a set of images by tarstream. // // NOTE: One tar may have several manifests. diff --git a/ctrd/interface.go b/ctrd/interface.go index 7dadb5837e..e0c92dba21 100644 --- a/ctrd/interface.go +++ b/ctrd/interface.go @@ -75,6 +75,8 @@ type ImageAPIClient interface { RemoveImage(ctx context.Context, ref string) error // ImportImage creates a set of images by tarstream. ImportImage(ctx context.Context, importer ctrdmetaimages.Importer, reader io.Reader) ([]containerd.Image, error) + // SaveImage saves image to tarstream + SaveImage(ctx context.Context, exporter ctrdmetaimages.Exporter, ref string) (io.ReadCloser, error) } // SnapshotAPIClient provides access to containerd snapshot features diff --git a/daemon/mgr/image.go b/daemon/mgr/image.go index 0ec40315f4..ebbcee5341 100644 --- a/daemon/mgr/image.go +++ b/daemon/mgr/image.go @@ -49,6 +49,9 @@ type ImageMgr interface { // LoadImage creates a set of images by tarstream. LoadImage(ctx context.Context, imageName string, tarstream io.ReadCloser) error + + // SaveImage saves image to tarstream. + SaveImage(ctx context.Context, idOrRef string) (io.ReadCloser, error) } // ImageManager is an implementation of interface ImageMgr. diff --git a/daemon/mgr/image_load.go b/daemon/mgr/image_load.go index fc5d4ef017..e43933faba 100644 --- a/daemon/mgr/image_load.go +++ b/daemon/mgr/image_load.go @@ -41,7 +41,7 @@ func (mgr *ImageManager) LoadImage(ctx context.Context, imageName string, tarstr } // FIXME(fuwei): if the store fails to update reference cache, the daemon - // may fails to load after restart. + // may fail to load after restart. merrs := new(multierror.Multierrors) for _, img := range imgs { if err := mgr.storeImageReference(ctx, img); err != nil { diff --git a/daemon/mgr/image_save.go b/daemon/mgr/image_save.go new file mode 100644 index 0000000000..caf0d2b0e4 --- /dev/null +++ b/daemon/mgr/image_save.go @@ -0,0 +1,23 @@ +package mgr + +import ( + "context" + "io" + + ociimage "github.com/containerd/containerd/images/oci" +) + +// SaveImage saves image to the oci.v1 format tarstream. +func (mgr *ImageManager) SaveImage(ctx context.Context, idOrRef string) (io.ReadCloser, error) { + _, _, ref, err := mgr.CheckReference(ctx, idOrRef) + if err != nil { + return nil, err + } + + exportedStream, err := mgr.client.SaveImage(ctx, &ociimage.V1Exporter{}, ref.String()) + if err != nil { + return nil, err + } + + return exportedStream, nil +} diff --git a/test/cli_load_test.go b/test/cli_load_test.go deleted file mode 100644 index 4c2706bc81..0000000000 --- a/test/cli_load_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "github.com/alibaba/pouch/test/environment" - - "github.com/go-check/check" -) - -// PouchLoadSuite is the test suite for load CLI. -type PouchLoadSuite struct{} - -func init() { - check.Suite(&PouchLoadSuite{}) -} - -// SetUpSuite does common setup in the beginning of each test suite. -func (suite *PouchLoadSuite) SetUpSuite(c *check.C) { - SkipIfFalse(c, environment.IsLinux) -} - -// TODO(fuwei): We cannot upload the oci.v1 format tar into repo because it will -// increase our repo size. The test will be done with "pouch save" functionality. diff --git a/test/cli_save_and_load_test.go b/test/cli_save_and_load_test.go new file mode 100644 index 0000000000..32e2e3ef6c --- /dev/null +++ b/test/cli_save_and_load_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/alibaba/pouch/apis/types" + "github.com/alibaba/pouch/test/command" + "github.com/alibaba/pouch/test/environment" + + "github.com/go-check/check" + "github.com/gotestyourself/gotestyourself/icmd" +) + +// PouchSaveLoadSuite is the test suite for save and load CLI. +type PouchSaveLoadSuite struct{} + +func init() { + check.Suite(&PouchSaveLoadSuite{}) +} + +// SetUpSuite does common setup in the beginning of each test suite. +func (suite *PouchSaveLoadSuite) SetUpSuite(c *check.C) { + SkipIfFalse(c, environment.IsLinux) + environment.PruneAllContainers(apiClient) +} + +// TestSaveLoadWorks tests "pouch save" and "pouch load" work. +func (suite *PouchSaveLoadSuite) TestSaveLoadWorks(c *check.C) { + res := command.PouchRun("pull", busyboxImage) + res.Assert(c, icmd.Success) + + res = command.PouchRun("image", "inspect", busyboxImage) + res.Assert(c, icmd.Success) + + before := []types.ImageInfo{} + if err := json.Unmarshal([]byte(res.Stdout()), &before); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + + dir := "/tmp/" + res = command.PouchRun("save", "-o", filepath.Join(dir, "busybox.tar"), busyboxImage) + res.Assert(c, icmd.Success) + + loadImageName := "load-busybox" + res = command.PouchRun("load", "-i", dir+"busybox.tar", loadImageName) + res.Assert(c, icmd.Success) + + defer os.Remove(dir + "busybox.tar") + + res = command.PouchRun("image", "inspect", loadImageName+":"+environment.BusyboxTag) + res.Assert(c, icmd.Success) + + after := []types.ImageInfo{} + if err := json.Unmarshal([]byte(res.Stdout()), &after); err != nil { + c.Errorf("failed to decode inspect output: %v", err) + } + + c.Assert(before[0].ID, check.Equals, after[0].ID) + c.Assert(before[0].CreatedAt, check.Equals, after[0].CreatedAt) + c.Assert(before[0].Size, check.Equals, after[0].Size) +}