From b1e1d8742e6cf3ea2317667014dd172e6101e273 Mon Sep 17 00:00:00 2001 From: zhangyue Date: Thu, 30 May 2019 10:53:38 +0800 Subject: [PATCH] feature: add pouch cp daemon side Signed-off-by: zhangyue --- apis/server/copy_bridge.go | 97 +++++++++++++++++ apis/server/router.go | 5 + apis/swagger.yml | 137 +++++++++++++++++++++++- client/request.go | 10 ++ daemon/mgr/container.go | 10 ++ daemon/mgr/container_copy.go | 181 ++++++++++++++++++++++++++++++++ daemon/mgr/container_storage.go | 16 +++ daemon/mgr/container_types.go | 19 ++++ 8 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 apis/server/copy_bridge.go create mode 100644 daemon/mgr/container_copy.go diff --git a/apis/server/copy_bridge.go b/apis/server/copy_bridge.go new file mode 100644 index 0000000000..39c80f9f36 --- /dev/null +++ b/apis/server/copy_bridge.go @@ -0,0 +1,97 @@ +package server + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/alibaba/pouch/pkg/httputils" + + "github.com/gorilla/mux" +) + +func (s *Server) putContainersArchive(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + name := mux.Vars(req)["name"] + path := req.FormValue("path") + + if name == "" { + return httputils.NewHTTPError(errors.New("name can't be empty"), http.StatusBadRequest) + } + if path == "" { + return httputils.NewHTTPError(errors.New("path can't be empty"), http.StatusBadRequest) + } + noOverwriteDirNonDir := httputils.BoolValue(req, "noOverwriteDirNonDir") + copyUIDGID := httputils.BoolValue(req, "copyUIDGID") + + return s.ContainerMgr.ExtractToDir(ctx, name, path, copyUIDGID, noOverwriteDirNonDir, req.Body) +} + +func (s *Server) headContainersArchive(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + name := mux.Vars(req)["name"] + path := req.FormValue("path") + + if name == "" { + return httputils.NewHTTPError(errors.New("name can't be empty"), http.StatusBadRequest) + } + if path == "" { + return httputils.NewHTTPError(errors.New("path can't be empty"), http.StatusBadRequest) + } + + stat, err := s.ContainerMgr.StatPath(ctx, name, path) + if err != nil { + return err + } + + statJSON, err := json.Marshal(stat) + if err != nil { + return err + } + + rw.Header().Set( + "X-Docker-Container-Path-Stat", + base64.StdEncoding.EncodeToString(statJSON), + ) + + rw.WriteHeader(http.StatusOK) + return nil +} + +func (s *Server) getContainersArchive(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + name := mux.Vars(req)["name"] + path := req.FormValue("path") + + if name == "" { + return httputils.NewHTTPError(errors.New("name can't be empty"), http.StatusBadRequest) + } + if path == "" { + return httputils.NewHTTPError(errors.New("path can't be empty"), http.StatusBadRequest) + } + + tarArchive, stat, err := s.ContainerMgr.ArchivePath(ctx, name, path) + if err != nil { + return err + } + defer tarArchive.Close() + + statJSON, err := json.Marshal(stat) + if err != nil { + return err + } + rw.Header().Set( + "X-Docker-Container-Path-Stat", + base64.StdEncoding.EncodeToString(statJSON), + ) + + rw.Header().Set("Content-Type", "application/x-tar") + _, err = io.Copy(rw, tarArchive) + + if err != nil { + return err + } + + rw.WriteHeader(http.StatusOK) + return nil +} diff --git a/apis/server/router.go b/apis/server/router.go index f71ba6e231..cfb75aaa71 100644 --- a/apis/server/router.go +++ b/apis/server/router.go @@ -100,6 +100,11 @@ func initRoute(s *Server) *mux.Router { {Method: http.MethodPost, Path: "/attach/{token}", HandlerFunc: s.criAttach}, {Method: http.MethodGet, Path: "/portforward/{token}", HandlerFunc: s.criPortForward}, {Method: http.MethodPost, Path: "/portforward/{token}", HandlerFunc: s.criPortForward}, + + // copy + {Method: http.MethodPut, Path: "/containers/{name:.*}/archive", HandlerFunc: s.putContainersArchive}, + {Method: http.MethodHead, Path: "/containers/{name:.*}/archive", HandlerFunc: s.headContainersArchive}, + {Method: http.MethodGet, Path: "/containers/{name:.*}/archive", HandlerFunc: s.getContainersArchive}, } if s.APIPlugin != nil { diff --git a/apis/swagger.yml b/apis/swagger.yml index 96fb792635..9416e4085f 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -1510,6 +1510,134 @@ paths: schema: $ref: "#/definitions/ContainerCommitOptions" + /containers/{id}/archive: + head: + summary: "Get information about files in a container" + description: "A response header `X-Docker-Container-Path-Stat` is return containing a base64 - encoded JSON object with some filesystem header information about the path." + operationId: "ContainerArchiveInfo" + responses: + 200: + description: "no error" + headers: + X-Docker-Container-Path-Stat: + type: "string" + description: "A base64 - encoded JSON object with some filesystem header information about the path" + 400: + description: "Bad parameter" + schema: + $ref: "#/responses/400ErrorResponse" + 404: + description: "Container or path does not exist" + schema: + $ref: "#/responses/404ErrorResponse" + examples: + application/json: + message: "No such container: c2ada9df5af8" + 500: + description: "Server error" + schema: + $ref: "#/responses/500ErrorResponse" + parameters: + - name: "id" + in: "path" + required: true + description: "ID or name of the container" + type: "string" + - name: "path" + in: "query" + required: true + description: "Resource in the container’s filesystem to archive." + type: "string" + tags: ["Copy"] + get: + summary: "Get an archive of a filesystem resource in a container" + description: "Get a tar archive of a resource in the filesystem of container id." + operationId: "ContainerArchive" + produces: ["application/x-tar"] + responses: + 200: + description: "no error" + 400: + description: "Bad parameter" + schema: + $ref: "#/responses/400ErrorResponse" + 404: + description: "Container or path does not exist" + schema: + $ref: "#/responses/404ErrorResponse" + examples: + application/json: + message: "No such container: c2ada9df5af8" + 500: + description: "server error" + schema: + $ref: "#/responses/500ErrorResponse" + parameters: + - name: "id" + in: "path" + required: true + description: "ID or name of the container" + type: "string" + - name: "path" + in: "query" + required: true + description: "Resource in the container’s filesystem to archive." + type: "string" + tags: ["Copy"] + put: + summary: "Extract an archive of files or folders to a directory in a container" + description: "Upload a tar archive to be extracted to a path in the filesystem of container id." + operationId: "PutContainerArchive" + consumes: ["application/x-tar", "application/octet-stream"] + responses: + 200: + description: "The content was extracted successfully" + 400: + description: "Bad parameter" + schema: + $ref: "#/responses/400ErrorResponse" + 403: + description: "Permission denied, the volume or container rootfs is marked as read-only." + schema: + $ref: "#/responses/403ErrorResponse" + 404: + description: "No such container or path does not exist inside the container" + schema: + $ref: "#/responses/404ErrorResponse" + examples: + application/json: + message: "No such container: c2ada9df5af8" + 500: + description: "Server error" + schema: + $ref: "#/responses/500ErrorResponse" + parameters: + - name: "id" + in: "path" + required: true + description: "ID or name of the container" + type: "string" + - name: "path" + in: "query" + required: true + description: "Path to a directory in the container to extract the archive’s contents into. " + type: "string" + - name: "noOverwriteDirNonDir" + in: "query" + description: "If “1”, “true”, or “True” then it will be an error if unpacking the given content would cause an existing directory to be replaced with a non-directory and vice versa." + type: "string" + - name: "copyUIDGID" + in: "query" + description: "If “1”, “true”, or “True” then it will copy UID/GID maps to dest file." + type: "string" + - name: "inputStream" + in: "body" + required: true + description: "The input stream must be a tar archive compressed with one of the following algorithms: identity (no compression), gzip, bzip2, xz." + schema: + type: "string" + tags: ["Copy"] + definitions: Error: type: "object" @@ -4378,7 +4506,6 @@ definitions: type: "string" description: "ID uniquely identifies an image committed by a container" - parameters: id: name: id @@ -4394,10 +4521,18 @@ parameters: type: string responses: + 400ErrorResponse: + description: An unexpected 400 error occurred. + schema: + $ref: "#/definitions/Error" 401ErrorResponse: description: An unexpected 401 error occurred. schema: $ref: "#/definitions/Error" + 403ErrorResponse: + description: An unexpected 403 error occurred. + schema: + $ref: "#/definitions/Error" 404ErrorResponse: description: An unexpected 404 error occurred. schema: diff --git a/client/request.go b/client/request.go index c016432e9e..b526d6c7b7 100644 --- a/client/request.go +++ b/client/request.go @@ -34,6 +34,7 @@ func (e RespError) Code() int { type Response struct { StatusCode int Status string + Header http.Header Body io.ReadCloser } @@ -50,6 +51,14 @@ func (client *APIClient) post(ctx context.Context, path string, query url.Values return client.sendRequest(ctx, "POST", path, query, body, headers) } +func (client *APIClient) head(ctx context.Context, path string, query url.Values, headers map[string][]string) (*Response, error) { + return client.sendRequest(ctx, "HEAD", path, query, nil, headers) +} + +func (client *APIClient) putRaw(ctx context.Context, path string, query url.Values, data io.Reader, headers map[string][]string) (*Response, error) { + return client.sendRequest(ctx, "PUT", path, query, data, headers) +} + func (client *APIClient) postRawData(ctx context.Context, path string, query url.Values, data io.Reader, headers map[string][]string) (*Response, error) { return client.sendRequest(ctx, "POST", path, query, data, headers) } @@ -134,6 +143,7 @@ func (client *APIClient) sendRequest(ctx context.Context, method, path string, q StatusCode: resp.StatusCode, Status: resp.Status, Body: resp.Body, + Header: resp.Header, }, nil } diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go index 393c095a45..8cd411822f 100644 --- a/daemon/mgr/container.go +++ b/daemon/mgr/container.go @@ -45,6 +45,7 @@ import ( // 1. regular container management; // 2. container exec management; // 3. container network management. +// 4. container copy management type ContainerMgr interface { // 1. the following functions are related to regular container management @@ -163,6 +164,15 @@ type ContainerMgr interface { // Commit commits an image from a container. Commit(ctx context.Context, name string, options *types.ContainerCommitOptions) (*types.ContainerCommitResp, error) + + // StatPath stats the dir info at the specified path in the container. + StatPath(ctx context.Context, name, path string) (stat *ContainerPathStat, err error) + + // ArchivePath return an archive and dir info at the specified path in the container. + ArchivePath(ctx context.Context, name, path string) (content io.ReadCloser, stat *ContainerPathStat, err error) + + // ExtractToDir extracts the given archive at the specified path in the container. + ExtractToDir(ctx context.Context, name, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) error } // ContainerManager is the default implement of interface ContainerMgr. diff --git a/daemon/mgr/container_copy.go b/daemon/mgr/container_copy.go new file mode 100644 index 0000000000..b06a727a99 --- /dev/null +++ b/daemon/mgr/container_copy.go @@ -0,0 +1,181 @@ +package mgr + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/chrootarchive" +) + +// StatPath stats the dir info at the specified path in the container. +func (mgr *ContainerManager) StatPath(ctx context.Context, name, path string) (stat *ContainerPathStat, err error) { + c, err := mgr.container(name) + if err != nil { + return nil, err + } + c.Lock() + defer c.Unlock() + + err = mgr.Mount(ctx, c) + if err != nil { + return nil, err + } + defer mgr.Unmount(ctx, c) + + err = mgr.attachVolumes(ctx, c) + if err != nil { + return nil, err + } + defer mgr.detachVolumes(ctx, c, false) + + // Consider the given path as an absolute path in the container. + absPath := path + if !filepath.IsAbs(absPath) { + absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join("/", path), path, os.PathSeparator) + } + + resolvedDirPath := c.GetResourcePath(c.BaseFS, filepath.Dir(absPath)) + + resolvedPath := resolvedDirPath + filepath.Base(absPath) + + lstat, err := os.Lstat(resolvedPath) + if err != nil { + return nil, err + } + + return &ContainerPathStat{ + Name: lstat.Name(), + Path: absPath, + Size: lstat.Size(), + Mode: lstat.Mode(), + Mtime: lstat.ModTime(), + }, nil +} + +// ArchivePath return an archive and dir info at the specified path in the container. +func (mgr *ContainerManager) ArchivePath(ctx context.Context, name, path string) (content io.ReadCloser, stat *ContainerPathStat, err error) { + c, err := mgr.container(name) + if err != nil { + return nil, nil, err + } + c.Lock() + defer c.Unlock() + + err = mgr.Mount(ctx, c) + if err != nil { + return nil, nil, err + } + defer mgr.Unmount(ctx, c) + + err = mgr.attachVolumes(ctx, c) + if err != nil { + return nil, nil, err + } + defer mgr.detachVolumes(ctx, c, false) + + // Consider the given path as an absolute path in the container. + absPath := path + if !filepath.IsAbs(absPath) { + absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join("/", path), path, os.PathSeparator) + } + + // get the real path on the host + resolvedDirPath := c.GetResourcePath(c.BaseFS, filepath.Dir(absPath)) + + resolvedPath := resolvedDirPath + filepath.Base(absPath) + + lstat, err := os.Lstat(resolvedPath) + if err != nil { + return nil, nil, err + } + + stat = &ContainerPathStat{ + Name: lstat.Name(), + Path: absPath, + Size: lstat.Size(), + Mode: lstat.Mode(), + Mtime: lstat.ModTime(), + } + // TODO: support follow link in container rootfs + copyInfo, err := archive.CopyInfoSourcePath(resolvedPath, false) + if err != nil { + return nil, nil, err + } + data, err := archive.TarResource(copyInfo) + if err != nil { + return nil, nil, err + } + + return data, stat, nil +} + +// ExtractToDir extracts the given archive at the specified path in the container. +func (mgr *ContainerManager) ExtractToDir(ctx context.Context, name, path string, copyUIDGID, noOverwriteDirNonDir bool, content io.Reader) error { + c, err := mgr.container(name) + if err != nil { + return err + } + c.Lock() + defer c.Unlock() + + err = mgr.Mount(ctx, c) + if err != nil { + return err + } + defer mgr.Unmount(ctx, c) + + err = mgr.attachVolumes(ctx, c) + if err != nil { + return err + } + defer mgr.detachVolumes(ctx, c, false) + + // Consider the given path as an absolute path in the container. + absPath := path + if !filepath.IsAbs(absPath) { + absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join("/", path), path, os.PathSeparator) + } + + // get the real path on the host + resolvedDirPath := c.GetResourcePath(c.BaseFS, filepath.Dir(absPath)) + + resolvedPath := resolvedDirPath + filepath.Base(absPath) + + lstat, err := os.Lstat(resolvedPath) + if err != nil { + return err + } + + if !lstat.IsDir() { + return errors.New("can't extract to not dir position") + } + + // first check if the dir in volume + inVolume := false + for _, mp := range c.Mounts { + if !strings.HasPrefix(path, mp.Destination) { + continue + } + inVolume = true + if mp.RW { + break + } + return errors.New("can't extract to dir because volume read only") + } + + if !inVolume && c.HostConfig.ReadonlyRootfs { + return errors.New("can't extract to dir because rootfs read only") + } + + // TODO: support copy uid/gid maps + opts := &archive.TarOptions{ + NoOverwriteDirNonDir: noOverwriteDirNonDir, + } + + return chrootarchive.Untar(content, resolvedPath, opts) +} diff --git a/daemon/mgr/container_storage.go b/daemon/mgr/container_storage.go index ba8acefbd4..2ff6e41a2f 100644 --- a/daemon/mgr/container_storage.go +++ b/daemon/mgr/container_storage.go @@ -668,6 +668,22 @@ func (mgr *ContainerManager) detachVolumes(ctx context.Context, c *Container, re return nil } +func (mgr *ContainerManager) attachVolumes(ctx context.Context, c *Container) error { + for _, mount := range c.Mounts { + name := mount.Name + if name == "" { + continue + } + + _, err := mgr.VolumeMgr.Attach(ctx, name, map[string]string{volumetypes.OptionRef: c.ID}) + if err != nil { + logrus.Warnf("failed to attach volume(%s), err(%v)", name, err) + return err + } + } + return nil +} + // setMountFS is used to set mountfs directory. func (mgr *ContainerManager) setMountFS(ctx context.Context, c *Container) { c.MountFS = path.Join(mgr.Store.Path(c.ID), "rootfs") diff --git a/daemon/mgr/container_types.go b/daemon/mgr/container_types.go index 114d38c416..7523e6be23 100644 --- a/daemon/mgr/container_types.go +++ b/daemon/mgr/container_types.go @@ -166,6 +166,25 @@ type ContainerStatsConfig struct { OutStream io.Writer } +// ContainerPathStat is used to encode the header from +// GET "/containers/{name:.*}/archive" +type ContainerPathStat struct { + // the file or directory's name. + Name string `json:"name"` + + // the path of the file or directory + Path string `json:"path"` + + // the size of the file or directory + Size int64 `json:"size"` + + // FileMode represents a file's mode and permission bits. + Mode os.FileMode `json:"mode"` + + // modify time of the file or directory + Mtime time.Time `json:"mtime"` +} + // Container represents the container's meta data. type Container struct { sync.Mutex