diff --git a/Dockerfile b/Dockerfile index d70cf70fd4a..efdde7b15cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,9 @@ # syntax = hashicorp.jfrog.io/docker/docker/dockerfile:experimental +#-------------------------------------------------------------------- +# builder builds the Waypoint binaries +#-------------------------------------------------------------------- + FROM hashicorp.jfrog.io/docker/golang:alpine AS builder RUN apk add --no-cache git gcc libc-dev openssh @@ -26,18 +30,88 @@ RUN go get github.com/kevinburke/go-bindata/... RUN --mount=type=cache,target=/root/.cache/go-build make bin RUN --mount=type=cache,target=/root/.cache/go-build make bin/entrypoint +#-------------------------------------------------------------------- +# imgbase builds the "img" tool and all of its dependencies +#-------------------------------------------------------------------- + +# We build a fork of img for now so we can get the `img inspect` CLI +# Watch this PR: https://github.com/genuinetools/img/pull/324 +FROM hashicorp.jfrog.io/docker/golang:alpine AS imgbuilder + +RUN apk add --no-cache \ + bash \ + build-base \ + gcc \ + git \ + libseccomp-dev \ + linux-headers \ + make + +RUN git clone https://github.com/mitchellh/img.git /img +WORKDIR /img +RUN git checkout inspect +RUN go get github.com/go-bindata/go-bindata/go-bindata +RUN make static && mv img /usr/bin/img + +# Copied from img repo, see notes for specific reasons: +# https://github.com/genuinetools/img/blob/d858ac71f93cc5084edd2ba2d425b90234cf2ead/Dockerfile +FROM hashicorp.jfrog.io/docker/alpine AS imgbase +RUN apk add --no-cache autoconf automake build-base byacc gettext gettext-dev \ + gcc git libcap-dev libtool libxslt img runc +RUN git clone https://github.com/shadow-maint/shadow.git /shadow +WORKDIR /shadow +RUN git checkout 59c2dabb264ef7b3137f5edb52c0b31d5af0cf76 +RUN ./autogen.sh --disable-nls --disable-man --without-audit \ + --without-selinux --without-acl --without-attr --without-tcb \ + --without-nscd \ + && make \ + && cp src/newuidmap src/newgidmap /usr/bin + +#-------------------------------------------------------------------- +# final image +#-------------------------------------------------------------------- + +# Notes on img and what is required to make it work, since there's a lot +# of small details below that are absolutely required for everything to +# come together: +# +# - img, runc, newuidmap, newgidmap need to be installed +# - libseccomp-dev must be installed for runc +# - newuidmap/newgidmap need to have suid set (u+s) +# - /etc/subuid and /etc/subgid need to have an entry for the user +# - USER, HOME, and XDG_RUNTIME_DIR all need to be set +# + FROM hashicorp.jfrog.io/docker/alpine +COPY --from=imgbuilder /usr/bin/img /usr/bin/img +COPY --from=imgbase /usr/bin/runc /usr/bin/runc +COPY --from=imgbase /usr/bin/newuidmap /usr/bin/newuidmap +COPY --from=imgbase /usr/bin/newgidmap /usr/bin/newgidmap + +# libseccomp-dev is required for runc +# git is for gitrefpretty() and other calls for Waypoint +RUN apk add --no-cache libseccomp-dev git + COPY --from=builder /tmp/wp-src/waypoint /usr/bin/waypoint COPY --from=builder /tmp/wp-src/waypoint-entrypoint /usr/bin/waypoint-entrypoint VOLUME ["/data"] RUN addgroup waypoint && \ - adduser -S -G waypoint waypoint && \ + adduser -S -u 1000 -G waypoint waypoint && \ mkdir /data/ && \ chown -R waypoint:waypoint /data +# configure newuidmap/newgidmap to work with our waypoint user +RUN chmod u+s /usr/bin/newuidmap /usr/bin/newgidmap \ + && mkdir -p /run/user/1000 \ + && chown -R waypoint /run/user/1000 /home/waypoint \ + && echo waypoint:100000:65536 | tee /etc/subuid | tee /etc/subgid + USER waypoint +ENV USER waypoint +ENV HOME /home/waypoint +ENV XDG_RUNTIME_DIR=/run/user/1000 ENTRYPOINT ["/usr/bin/waypoint"] diff --git a/builtin/docker/builder.go b/builtin/docker/builder.go index 21b0d3d5f6a..f3fbd8c013f 100644 --- a/builtin/docker/builder.go +++ b/builtin/docker/builder.go @@ -15,6 +15,8 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/jsonmessage" + "github.com/golang/protobuf/ptypes/empty" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/waypoint-plugin-sdk/component" "github.com/hashicorp/waypoint-plugin-sdk/docs" "github.com/hashicorp/waypoint-plugin-sdk/terminal" @@ -107,26 +109,27 @@ func (b *Builder) Build( ctx context.Context, ui terminal.UI, src *component.Source, + log hclog.Logger, ) (*Image, error) { - stdout, _, err := ui.OutputWriters() - if err != nil { - return nil, err - } - sg := ui.StepGroup() + defer sg.Wait() step := sg.Add("Initializing Docker client...") - defer step.Abort() + defer func() { + if step != nil { + step.Abort() + } + }() result := &Image{ - Image: fmt.Sprintf("waypoint.local/%s", src.App), - Tag: "latest", + Image: fmt.Sprintf("waypoint.local/%s", src.App), + Tag: "latest", + Location: &Image_Docker{Docker: &empty.Empty{}}, } cli, err := wpdockerclient.NewClientWithOpts(client.FromEnv) if err != nil { return nil, status.Errorf(codes.FailedPrecondition, "unable to create Docker client: %s", err) } - cli.NegotiateAPIVersion(ctx) dockerfile := b.config.Dockerfile @@ -160,13 +163,102 @@ func (b *Builder) Build( return nil, status.Errorf(codes.FailedPrecondition, "unable to create Docker context: %s", err) } + // We now test if Docker is actually functional. We do this here because we + // need all of the above to complete the actual build. + log.Debug("testing if we should use a Docker fallback") + useImg := false + if fallback, err := wpdockerclient.Fallback(ctx, log, cli); err != nil { + log.Warn("error during check if we should use Docker fallback", "err", err) + return nil, status.Errorf(codes.Internal, + "error validating Docker connection: %s", err) + } else if fallback && hasImg() { + // If we're falling back and have "img" available, use that. If we + // don't have "img" available, we continue to try to use Docker. We'll + // fail but that error message should help the user. + step.Update("Docker isn't available. Falling back to daemonless image build...") + step.Done() + step = nil + if err := b.buildWithImg( + ctx, ui, sg, relDockerfile, contextDir, result.Name(), + ); err != nil { + return nil, err + } + + // Our image is in the img registry now. We set this so that + // future users of this result type know where to look. + result.Location = &Image_Img{Img: &empty.Empty{}} + + // We set this to true so we use the img-based injector later + useImg = true + } else { + // No fallback, build with Docker + step.Done() + step = nil + if err := b.buildWithDocker( + ctx, ui, sg, cli, contextDir, relDockerfile, result.Name(), + ); err != nil { + return nil, err + } + } + + if !b.config.DisableCEB { + step = sg.Add("Injecting Waypoint Entrypoint...") + + asset, err := assets.Asset("ceb/ceb") + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to restore custom entry point binary: %s", err) + } + + assetInfo, err := assets.AssetInfo("ceb/ceb") + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to restore custom entry point binary: %s", err) + } + + callback := func(cur []string) (*epinject.NewEntrypoint, error) { + ep := &epinject.NewEntrypoint{ + Entrypoint: append([]string{"/waypoint-entrypoint"}, cur...), + InjectFiles: map[string]epinject.InjectFile{ + "/waypoint-entrypoint": { + Reader: bytes.NewReader(asset), + Info: assetInfo, + }, + }, + } + + return ep, nil + } + + if !useImg { + _, err = epinject.AlterEntrypoint(ctx, result.Name(), callback) + } else { + _, err = epinject.AlterEntrypointImg(ctx, result.Name(), callback) + } + if err != nil { + return nil, status.Errorf(codes.Internal, "unable to set modify Docker entrypoint: %s", err) + } + + step.Done() + } + + return result, nil +} + +func (b *Builder) buildWithDocker( + ctx context.Context, + ui terminal.UI, + sg terminal.StepGroup, + cli *client.Client, + contextDir string, + relDockerfile string, + tag string, +) error { excludes, err := build.ReadDockerignore(contextDir) if err != nil { - return nil, status.Errorf(codes.Internal, "unable to read .dockerignore: %s", err) + return status.Errorf(codes.Internal, "unable to read .dockerignore: %s", err) } if err := build.ValidateContextDirectory(contextDir, excludes); err != nil { - return nil, status.Errorf(codes.Internal, "error checking context: %s", err) + return status.Errorf(codes.Internal, "error checking context: %s", err) } // And canonicalize dockerfile name to a platform-independent one @@ -179,7 +271,7 @@ func (b *Builder) Build( }) if err != nil { - return nil, status.Errorf(codes.Internal, "unable to compress context: %s", err) + return status.Errorf(codes.Internal, "unable to compress context: %s", err) } ver := types.BuilderV1 @@ -187,17 +279,22 @@ func (b *Builder) Build( ver = types.BuilderBuildKit } - step.Done() - step = sg.Add("Building image...") + step := sg.Add("Building image...") + defer step.Abort() + + stdout, _, err := ui.OutputWriters() + if err != nil { + return err + } resp, err := cli.ImageBuild(ctx, buildCtx, types.ImageBuildOptions{ Version: ver, Dockerfile: relDockerfile, - Tags: []string{result.Name()}, + Tags: []string{tag}, Remove: true, }) if err != nil { - return nil, status.Errorf(codes.Internal, "error building image: %s", err) + return status.Errorf(codes.Internal, "error building image: %s", err) } defer resp.Body.Close() @@ -208,44 +305,9 @@ func (b *Builder) Build( err = jsonmessage.DisplayJSONMessagesStream(resp.Body, step.TermOutput(), termFd, true, nil) if err != nil { - return nil, status.Errorf(codes.Internal, "unable to stream build logs to the terminal: %s", err) + return status.Errorf(codes.Internal, "unable to stream build logs to the terminal: %s", err) } step.Done() - - if !b.config.DisableCEB { - step = sg.Add("Injecting Waypoint Entrypoint...") - - asset, err := assets.Asset("ceb/ceb") - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to restore custom entry point binary: %s", err) - } - - assetInfo, err := assets.AssetInfo("ceb/ceb") - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to restore custom entry point binary: %s", err) - } - - _, err = epinject.AlterEntrypoint(ctx, result.Name(), func(cur []string) (*epinject.NewEntrypoint, error) { - ep := &epinject.NewEntrypoint{ - Entrypoint: append([]string{"/waypoint-entrypoint"}, cur...), - InjectFiles: map[string]epinject.InjectFile{ - "/waypoint-entrypoint": { - Reader: bytes.NewReader(asset), - Info: assetInfo, - }, - }, - } - - return ep, nil - }) - - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to set modify Docker entrypoint: %s", err) - } - - step.Done() - } - - return result, nil + return nil } diff --git a/builtin/docker/client/fallback.go b/builtin/docker/client/fallback.go new file mode 100644 index 00000000000..00850e2f484 --- /dev/null +++ b/builtin/docker/client/fallback.go @@ -0,0 +1,77 @@ +package client + +import ( + "context" + "os" + "runtime" + + "github.com/docker/docker/client" + "github.com/hashicorp/go-hclog" +) + +// Fallback returns true if we should attempt to fallback to a daemonless +// mechanism. If the return value is (false, nil) then we should not fallback +// and Docker appears ready to use. If the return value is (false, non-nil) +// then we should not fallback but Docker does NOT appear healthy. +// +// Note that a return value of true means that we should attempt a fallback, +// but this method doesn't validate that any fallback mechanism is actually +// available. +func Fallback( + ctx context.Context, + log hclog.Logger, + c *client.Client, +) (bool, error) { + const DockerSocketPath = "/var/run/docker.sock" + + // We always nest ourselves because our logs are annoying (but TRACE) + log = log.Named("docker_fallback_check") + + // Grab the server version, we do this to attempt a connection. If + // this succeeds, we always use Docker. + log.Trace("testing Docker client connection by calling ServerVersion") + _, err := c.ServerVersion(ctx) + if err == nil { + log.Trace("ServerVersion succeeded, will use Docker daemon") + return false, nil + } + if !client.IsErrConnectionFailed(err) { + // If we got an error other than connection failure, we notify the user + // because we shouldn't fallback if anything else happened. + log.Trace("ServerVersion fallback check failed with non-connection error", + "err", err) + return false, err + } + + // If the Docker host is set, the user wants to use Docker, so we never + // fallback. + log.Trace("testing DOCKER_HOST value") + if os.Getenv("DOCKER_HOST") != "" { + log.Trace("will not fallback because DOCKER_HOST is set") + return false, err + } + + // We never fallback on Windows currently because we have no mechanism + // to build without a Docker daemon on Windows. + log.Trace("testing GOOS") + if runtime.GOOS == "windows" { + log.Trace("will not fallback because GOOS is Windows") + return false, err + } + + // If the Docker socket does NOT exist, then fall back. + log.Trace("testing for Docker socket existence", "path", DockerSocketPath) + _, staterr := os.Stat(DockerSocketPath) + if staterr == nil { + // Docker socket does exist, so let's assume the user wants to use Docker. + log.Trace("Docker socket exists, will not use fallback") + return false, err + } + if !os.IsNotExist(staterr) { + log.Trace("error during check for Docker socket", "err", err) + return false, err + } + + log.Trace("will fallback, no Docker socket found") + return true, nil +} diff --git a/builtin/docker/img.go b/builtin/docker/img.go new file mode 100644 index 00000000000..8ec29373762 --- /dev/null +++ b/builtin/docker/img.go @@ -0,0 +1,56 @@ +package docker + +import ( + "context" + "os/exec" + + "github.com/hashicorp/waypoint-plugin-sdk/terminal" +) + +// hasImg returns true if "img" is available on the PATH. +// +// This doesn't do any fancy checking that "img" is the "img" we expect. +// We can make the checking here more advanced later. +func hasImg() bool { + _, err := exec.LookPath("img") + return err == nil +} + +func (b *Builder) buildWithImg( + ctx context.Context, + ui terminal.UI, + sg terminal.StepGroup, + dockerfilePath string, + contextDir string, + tag string, +) error { + step := sg.Add("Building Docker image with img...") + defer func() { + if step != nil { + step.Abort() + } + }() + + // NOTE(mitchellh): we can probably use the img Go pkg directly one day. + cmd := exec.CommandContext(ctx, + "img", + "build", + "-f", dockerfilePath, + "-t", tag, + ".", + ) + + // Working directory to directory with build context + cmd.Dir = contextDir + + // Command output should go to the step + cmd.Stdout = step.TermOutput() + cmd.Stderr = cmd.Stdout + + if err := cmd.Run(); err != nil { + return err + } + + step.Done() + return nil +} diff --git a/builtin/docker/plugin.pb.go b/builtin/docker/plugin.pb.go index 048ce5d6afc..a179fabad49 100644 --- a/builtin/docker/plugin.pb.go +++ b/builtin/docker/plugin.pb.go @@ -1,189 +1,386 @@ // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.23.0 +// protoc v3.11.4 // source: waypoint/builtin/docker/plugin.proto package docker import ( - fmt "fmt" proto "github.com/golang/protobuf/proto" - math "math" + empty "github.com/golang/protobuf/ptypes/empty" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 // Image is the artifact type for the registry. type Image struct { - Image string `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"` - Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *Image) Reset() { *m = Image{} } -func (m *Image) String() string { return proto.CompactTextString(m) } -func (*Image) ProtoMessage() {} -func (*Image) Descriptor() ([]byte, []int) { - return fileDescriptor_2440d536b1a2bd07, []int{0} + Image string `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"` + Tag string `protobuf:"bytes,2,opt,name=tag,proto3" json:"tag,omitempty"` + // location is where this image is currently. This can be used to + // determine if the image is pulled or not based on this proto rather + // than environment inspection. + // + // Types that are assignable to Location: + // *Image_Registry + // *Image_Docker + // *Image_Img + Location isImage_Location `protobuf_oneof:"location"` } -func (m *Image) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Image.Unmarshal(m, b) -} -func (m *Image) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Image.Marshal(b, m, deterministic) +func (x *Image) Reset() { + *x = Image{} + if protoimpl.UnsafeEnabled { + mi := &file_waypoint_builtin_docker_plugin_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Image) XXX_Merge(src proto.Message) { - xxx_messageInfo_Image.Merge(m, src) + +func (x *Image) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Image) XXX_Size() int { - return xxx_messageInfo_Image.Size(m) + +func (*Image) ProtoMessage() {} + +func (x *Image) ProtoReflect() protoreflect.Message { + mi := &file_waypoint_builtin_docker_plugin_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *Image) XXX_DiscardUnknown() { - xxx_messageInfo_Image.DiscardUnknown(m) + +// Deprecated: Use Image.ProtoReflect.Descriptor instead. +func (*Image) Descriptor() ([]byte, []int) { + return file_waypoint_builtin_docker_plugin_proto_rawDescGZIP(), []int{0} } -var xxx_messageInfo_Image proto.InternalMessageInfo +func (x *Image) GetImage() string { + if x != nil { + return x.Image + } + return "" +} -func (m *Image) GetImage() string { - if m != nil { - return m.Image +func (x *Image) GetTag() string { + if x != nil { + return x.Tag } return "" } -func (m *Image) GetTag() string { +func (m *Image) GetLocation() isImage_Location { if m != nil { - return m.Tag + return m.Location } - return "" + return nil } -type Deployment struct { - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Container string `protobuf:"bytes,3,opt,name=container,proto3" json:"container,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +func (x *Image) GetRegistry() *empty.Empty { + if x, ok := x.GetLocation().(*Image_Registry); ok { + return x.Registry + } + return nil } -func (m *Deployment) Reset() { *m = Deployment{} } -func (m *Deployment) String() string { return proto.CompactTextString(m) } -func (*Deployment) ProtoMessage() {} -func (*Deployment) Descriptor() ([]byte, []int) { - return fileDescriptor_2440d536b1a2bd07, []int{1} +func (x *Image) GetDocker() *empty.Empty { + if x, ok := x.GetLocation().(*Image_Docker); ok { + return x.Docker + } + return nil } -func (m *Deployment) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Deployment.Unmarshal(m, b) +func (x *Image) GetImg() *empty.Empty { + if x, ok := x.GetLocation().(*Image_Img); ok { + return x.Img + } + return nil } -func (m *Deployment) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Deployment.Marshal(b, m, deterministic) + +type isImage_Location interface { + isImage_Location() } -func (m *Deployment) XXX_Merge(src proto.Message) { - xxx_messageInfo_Deployment.Merge(m, src) + +type Image_Registry struct { + // registry is set if the image is in a remote registry. + Registry *empty.Empty `protobuf:"bytes,3,opt,name=registry,proto3,oneof"` } -func (m *Deployment) XXX_Size() int { - return xxx_messageInfo_Deployment.Size(m) + +type Image_Docker struct { + // docker is set if the image is in a local Docker daemon registry. + Docker *empty.Empty `protobuf:"bytes,4,opt,name=docker,proto3,oneof"` } -func (m *Deployment) XXX_DiscardUnknown() { - xxx_messageInfo_Deployment.DiscardUnknown(m) + +type Image_Img struct { + // img is set if the image is in a local img content store directory. + // img: https://github.com/genuinetools/img + Img *empty.Empty `protobuf:"bytes,5,opt,name=img,proto3,oneof"` } -var xxx_messageInfo_Deployment proto.InternalMessageInfo +func (*Image_Registry) isImage_Location() {} -func (m *Deployment) GetId() string { - if m != nil { - return m.Id +func (*Image_Docker) isImage_Location() {} + +func (*Image_Img) isImage_Location() {} + +type Deployment struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Container string `protobuf:"bytes,3,opt,name=container,proto3" json:"container,omitempty"` +} + +func (x *Deployment) Reset() { + *x = Deployment{} + if protoimpl.UnsafeEnabled { + mi := &file_waypoint_builtin_docker_plugin_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Deployment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Deployment) ProtoMessage() {} + +func (x *Deployment) ProtoReflect() protoreflect.Message { + mi := &file_waypoint_builtin_docker_plugin_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Deployment.ProtoReflect.Descriptor instead. +func (*Deployment) Descriptor() ([]byte, []int) { + return file_waypoint_builtin_docker_plugin_proto_rawDescGZIP(), []int{1} +} + +func (x *Deployment) GetId() string { + if x != nil { + return x.Id } return "" } -func (m *Deployment) GetName() string { - if m != nil { - return m.Name +func (x *Deployment) GetName() string { + if x != nil { + return x.Name } return "" } -func (m *Deployment) GetContainer() string { - if m != nil { - return m.Container +func (x *Deployment) GetContainer() string { + if x != nil { + return x.Container } return "" } type Release struct { - Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *Release) Reset() { *m = Release{} } -func (m *Release) String() string { return proto.CompactTextString(m) } -func (*Release) ProtoMessage() {} -func (*Release) Descriptor() ([]byte, []int) { - return fileDescriptor_2440d536b1a2bd07, []int{2} + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` } -func (m *Release) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Release.Unmarshal(m, b) -} -func (m *Release) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Release.Marshal(b, m, deterministic) -} -func (m *Release) XXX_Merge(src proto.Message) { - xxx_messageInfo_Release.Merge(m, src) +func (x *Release) Reset() { + *x = Release{} + if protoimpl.UnsafeEnabled { + mi := &file_waypoint_builtin_docker_plugin_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Release) XXX_Size() int { - return xxx_messageInfo_Release.Size(m) + +func (x *Release) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Release) XXX_DiscardUnknown() { - xxx_messageInfo_Release.DiscardUnknown(m) + +func (*Release) ProtoMessage() {} + +func (x *Release) ProtoReflect() protoreflect.Message { + mi := &file_waypoint_builtin_docker_plugin_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Release proto.InternalMessageInfo +// Deprecated: Use Release.ProtoReflect.Descriptor instead. +func (*Release) Descriptor() ([]byte, []int) { + return file_waypoint_builtin_docker_plugin_proto_rawDescGZIP(), []int{2} +} -func (m *Release) GetUrl() string { - if m != nil { - return m.Url +func (x *Release) GetUrl() string { + if x != nil { + return x.Url } return "" } -func init() { - proto.RegisterType((*Image)(nil), "docker.Image") - proto.RegisterType((*Deployment)(nil), "docker.Deployment") - proto.RegisterType((*Release)(nil), "docker.Release") +var File_waypoint_builtin_docker_plugin_proto protoreflect.FileDescriptor + +var file_waypoint_builtin_docker_plugin_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x77, 0x61, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x74, + 0x69, 0x6e, 0x2f, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x1a, 0x1b, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xcf, 0x01, 0x0a, 0x05, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x74, + 0x61, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x74, 0x61, 0x67, 0x12, 0x34, 0x0a, + 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x72, 0x79, 0x12, 0x30, 0x0a, 0x06, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x48, 0x00, 0x52, 0x06, 0x64, + 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x12, 0x2a, 0x0a, 0x03, 0x69, 0x6d, 0x67, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x48, 0x00, 0x52, 0x03, 0x69, 0x6d, + 0x67, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x4e, 0x0a, + 0x0a, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x22, 0x1b, 0x0a, + 0x07, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x42, 0x19, 0x5a, 0x17, 0x77, 0x61, + 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x74, 0x69, 0x6e, 0x2f, 0x64, + 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_waypoint_builtin_docker_plugin_proto_rawDescOnce sync.Once + file_waypoint_builtin_docker_plugin_proto_rawDescData = file_waypoint_builtin_docker_plugin_proto_rawDesc +) + +func file_waypoint_builtin_docker_plugin_proto_rawDescGZIP() []byte { + file_waypoint_builtin_docker_plugin_proto_rawDescOnce.Do(func() { + file_waypoint_builtin_docker_plugin_proto_rawDescData = protoimpl.X.CompressGZIP(file_waypoint_builtin_docker_plugin_proto_rawDescData) + }) + return file_waypoint_builtin_docker_plugin_proto_rawDescData } -func init() { - proto.RegisterFile("waypoint/builtin/docker/plugin.proto", fileDescriptor_2440d536b1a2bd07) +var file_waypoint_builtin_docker_plugin_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_waypoint_builtin_docker_plugin_proto_goTypes = []interface{}{ + (*Image)(nil), // 0: docker.Image + (*Deployment)(nil), // 1: docker.Deployment + (*Release)(nil), // 2: docker.Release + (*empty.Empty)(nil), // 3: google.protobuf.Empty +} +var file_waypoint_builtin_docker_plugin_proto_depIdxs = []int32{ + 3, // 0: docker.Image.registry:type_name -> google.protobuf.Empty + 3, // 1: docker.Image.docker:type_name -> google.protobuf.Empty + 3, // 2: docker.Image.img:type_name -> google.protobuf.Empty + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } -var fileDescriptor_2440d536b1a2bd07 = []byte{ - // 189 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x29, 0x4f, 0xac, 0x2c, - 0xc8, 0xcf, 0xcc, 0x2b, 0xd1, 0x4f, 0x2a, 0xcd, 0xcc, 0x29, 0xc9, 0xcc, 0xd3, 0x4f, 0xc9, 0x4f, - 0xce, 0x4e, 0x2d, 0xd2, 0x2f, 0xc8, 0x29, 0x4d, 0xcf, 0xcc, 0xd3, 0x2b, 0x28, 0xca, 0x2f, 0xc9, - 0x17, 0x62, 0x83, 0x08, 0x2a, 0xe9, 0x73, 0xb1, 0x7a, 0xe6, 0x26, 0xa6, 0xa7, 0x0a, 0x89, 0x70, - 0xb1, 0x66, 0x82, 0x18, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x10, 0x8e, 0x90, 0x00, 0x17, - 0x73, 0x49, 0x62, 0xba, 0x04, 0x13, 0x58, 0x0c, 0xc4, 0x54, 0xf2, 0xe3, 0xe2, 0x72, 0x49, 0x2d, - 0xc8, 0xc9, 0xaf, 0xcc, 0x4d, 0xcd, 0x2b, 0x11, 0xe2, 0xe3, 0x62, 0xca, 0x4c, 0x81, 0x6a, 0x61, - 0xca, 0x4c, 0x11, 0x12, 0xe2, 0x62, 0xc9, 0x4b, 0xcc, 0x4d, 0x85, 0x6a, 0x00, 0xb3, 0x85, 0x64, - 0xb8, 0x38, 0x93, 0xf3, 0xf3, 0x4a, 0x12, 0x33, 0xf3, 0x52, 0x8b, 0x24, 0x98, 0xc1, 0x12, 0x08, - 0x01, 0x25, 0x69, 0x2e, 0xf6, 0xa0, 0xd4, 0x9c, 0xd4, 0xc4, 0x62, 0xb0, 0x65, 0xa5, 0x45, 0x39, - 0x50, 0xd3, 0x40, 0x4c, 0x27, 0xc9, 0x28, 0x71, 0x1c, 0xbe, 0x49, 0x62, 0x03, 0xfb, 0xc3, 0x18, - 0x10, 0x00, 0x00, 0xff, 0xff, 0xe4, 0x69, 0xcd, 0x4f, 0xef, 0x00, 0x00, 0x00, +func init() { file_waypoint_builtin_docker_plugin_proto_init() } +func file_waypoint_builtin_docker_plugin_proto_init() { + if File_waypoint_builtin_docker_plugin_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_waypoint_builtin_docker_plugin_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Image); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_waypoint_builtin_docker_plugin_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Deployment); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_waypoint_builtin_docker_plugin_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Release); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_waypoint_builtin_docker_plugin_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*Image_Registry)(nil), + (*Image_Docker)(nil), + (*Image_Img)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_waypoint_builtin_docker_plugin_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_waypoint_builtin_docker_plugin_proto_goTypes, + DependencyIndexes: file_waypoint_builtin_docker_plugin_proto_depIdxs, + MessageInfos: file_waypoint_builtin_docker_plugin_proto_msgTypes, + }.Build() + File_waypoint_builtin_docker_plugin_proto = out.File + file_waypoint_builtin_docker_plugin_proto_rawDesc = nil + file_waypoint_builtin_docker_plugin_proto_goTypes = nil + file_waypoint_builtin_docker_plugin_proto_depIdxs = nil } diff --git a/builtin/docker/plugin.proto b/builtin/docker/plugin.proto index 92d0ce3b5e2..82ab280d078 100644 --- a/builtin/docker/plugin.proto +++ b/builtin/docker/plugin.proto @@ -4,10 +4,30 @@ package docker; option go_package = "waypoint/builtin/docker"; +import "google/protobuf/empty.proto"; + // Image is the artifact type for the registry. message Image { string image = 1; string tag = 2; + + // location is where this image is currently. This can be used to + // determine if the image is pulled or not based on this proto rather + // than environment inspection. + // + // If this is not set, it will be assumed that the image is in a local + // Docker daemon registry for backwards compatiblity reasons. + oneof location { + // registry is set if the image is in a remote registry. + google.protobuf.Empty registry = 3; + + // docker is set if the image is in a local Docker daemon registry. + google.protobuf.Empty docker = 4; + + // img is set if the image is in a local img content store directory. + // img: https://github.com/genuinetools/img + google.protobuf.Empty img = 5; + } } message Deployment { diff --git a/builtin/docker/registry.go b/builtin/docker/registry.go index 0d5ea0ad74e..2f8e09eba66 100644 --- a/builtin/docker/registry.go +++ b/builtin/docker/registry.go @@ -1,22 +1,12 @@ package docker import ( - "bytes" "context" - "encoding/base64" - "encoding/json" - "os" - - "github.com/docker/cli/cli/config" - "github.com/docker/distribution/reference" - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/jsonmessage" - "github.com/docker/docker/registry" + + "github.com/golang/protobuf/ptypes/empty" "github.com/hashicorp/go-hclog" "github.com/hashicorp/waypoint-plugin-sdk/docs" "github.com/hashicorp/waypoint-plugin-sdk/terminal" - wpdockerclient "github.com/hashicorp/waypoint/builtin/docker/client" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -43,102 +33,50 @@ func (r *Registry) Push( ui terminal.UI, log hclog.Logger, ) (*Image, error) { - stdout, _, err := ui.OutputWriters() - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "unable to create output for logs:%s", err) - } - - sg := ui.StepGroup() - step := sg.Add("Initializing Docker client...") - defer func() { step.Abort() }() - - cli, err := wpdockerclient.NewClientWithOpts(client.FromEnv) - if err != nil { - return nil, status.Errorf(codes.FailedPrecondition, "unable to create Docker client:%s", err) - } - cli.NegotiateAPIVersion(ctx) - - step.Update("Tagging Docker image: %s => %s:%s", img.Name(), r.config.Image, r.config.Tag) - - target := &Image{Image: r.config.Image, Tag: r.config.Tag} - err = cli.ImageTag(ctx, img.Name(), target.Name()) - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to tag image:%s", err) + target := &Image{ + Image: r.config.Image, + Tag: r.config.Tag, } - - step.Done() - - if r.config.Local { - return target, nil + if !r.config.Local { + target.Location = &Image_Registry{Registry: &empty.Empty{}} } - ref, err := reference.ParseNormalizedNamed(target.Name()) - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to parse image name: %s", err) - } - - encodedAuth := r.config.EncodedAuth - if encodedAuth == "" { - // Resolve the Repository name from fqn to RepositoryInfo - repoInfo, err := registry.ParseRepositoryInfo(ref) - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to parse repository info from image name: %s", err) - } - - var server string - - if repoInfo.Index.Official { - info, err := cli.Info(ctx) - if err != nil || info.IndexServerAddress == "" { - server = registry.IndexServer - } else { - server = info.IndexServerAddress - } - } else { - server = repoInfo.Index.Name - } - - var errBuf bytes.Buffer - cf := config.LoadDefaultConfigFile(&errBuf) - if errBuf.Len() > 0 { - // NOTE(mitchellh): I don't know why we ignore this, but we always have. - log.Warn("error loading Docker config file", "err", err) + // Depending on whethere the image is, we diverge at this point. + switch img.Location.(type) { + case *Image_Registry: + // We can't push an image that isn't pulled locally in some form. + return nil, status.Errorf(codes.FailedPrecondition, + "Input image is not pulled locally and therefore can't be pushed. "+ + "Please pull the image or use a builder that pulls the image first.") + + case *Image_Img: + // If the image is already in img, we have to use `img push`. + if err := r.pushWithImg( + ctx, + log, + ui, + img, + target, + ); err != nil { + return nil, err } - authConfig, _ := cf.GetAuthConfig(server) - buf, err := json.Marshal(authConfig) - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to generate authentication info for registry: %s", err) + case *Image_Docker, nil: + // We support "nil" here for backwards compatibility. Images built + // prior to supporting the Location field will set nil. + if err := r.pushWithDocker( + ctx, + log, + ui, + img, + target, + ); err != nil { + return nil, err } - encodedAuth = base64.URLEncoding.EncodeToString(buf) - } - - step = sg.Add("Pushing Docker image...") - - options := types.ImagePushOptions{ - RegistryAuth: encodedAuth, - } - - responseBody, err := cli.ImagePush(ctx, reference.FamiliarString(ref), options) - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to push image to registry: %s", err) } - defer responseBody.Close() - - var termFd uintptr - if f, ok := stdout.(*os.File); ok { - termFd = f.Fd() - } - - err = jsonmessage.DisplayJSONMessagesStream(responseBody, step.TermOutput(), termFd, true, nil) - if err != nil { - return nil, status.Errorf(codes.Internal, "unable to stream Docker logs to terminal: %s", err) - } - - step.Done() - - step = sg.Add("Docker image pushed: %s:%s", r.config.Image, r.config.Tag) + sg := ui.StepGroup() + step := sg.Add("Docker image pushed: %s:%s", r.config.Image, r.config.Tag) step.Done() return target, nil @@ -208,6 +146,10 @@ build { "encoded_auth", "the authentication information to log into the docker repository", docs.Summary( + "The format of this is base64-encoded JSON. The structure is the ", + "[`AuthConfig`](https://pkg.go.dev/github.com/docker/cli/cli/config/types#AuthConfig)", + "structure used by Docker.", + "", "WARNING: be very careful to not leak the authentication information", "by hardcoding it here. Use a helper function like `file()` to read", "the information from a file not stored in VCS", diff --git a/builtin/docker/registry_docker.go b/builtin/docker/registry_docker.go new file mode 100644 index 00000000000..3a8b19e630c --- /dev/null +++ b/builtin/docker/registry_docker.go @@ -0,0 +1,125 @@ +package docker + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "os" + + "github.com/docker/cli/cli/config" + "github.com/docker/distribution/reference" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/jsonmessage" + "github.com/docker/docker/registry" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/waypoint-plugin-sdk/terminal" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + wpdockerclient "github.com/hashicorp/waypoint/builtin/docker/client" +) + +func (r *Registry) pushWithDocker( + ctx context.Context, + log hclog.Logger, + ui terminal.UI, + source *Image, + target *Image, +) error { + stdout, _, err := ui.OutputWriters() + if err != nil { + return status.Errorf(codes.FailedPrecondition, "unable to create output for logs:%s", err) + } + + sg := ui.StepGroup() + defer sg.Wait() + step := sg.Add("Initializing Docker client...") + defer func() { step.Abort() }() + + cli, err := wpdockerclient.NewClientWithOpts(client.FromEnv) + if err != nil { + return status.Errorf(codes.FailedPrecondition, "unable to create Docker client:%s", err) + } + cli.NegotiateAPIVersion(ctx) + + step.Update("Tagging Docker image: %s => %s:%s", source.Name(), r.config.Image, r.config.Tag) + err = cli.ImageTag(ctx, source.Name(), target.Name()) + if err != nil { + return status.Errorf(codes.Internal, "unable to tag image:%s", err) + } + + step.Done() + + if r.config.Local { + return nil + } + + ref, err := reference.ParseNormalizedNamed(target.Name()) + if err != nil { + return status.Errorf(codes.Internal, "unable to parse image name: %s", err) + } + + encodedAuth := r.config.EncodedAuth + if encodedAuth == "" { + // Resolve the Repository name from fqn to RepositoryInfo + repoInfo, err := registry.ParseRepositoryInfo(ref) + if err != nil { + return status.Errorf(codes.Internal, "unable to parse repository info from image name: %s", err) + } + + var server string + + if repoInfo.Index.Official { + info, err := cli.Info(ctx) + if err != nil || info.IndexServerAddress == "" { + server = registry.IndexServer + } else { + server = info.IndexServerAddress + } + } else { + server = repoInfo.Index.Name + } + + var errBuf bytes.Buffer + cf := config.LoadDefaultConfigFile(&errBuf) + if errBuf.Len() > 0 { + // NOTE(mitchellh): I don't know why we ignore this, but we always have. + log.Warn("error loading Docker config file", "err", err) + } + + authConfig, _ := cf.GetAuthConfig(server) + buf, err := json.Marshal(authConfig) + if err != nil { + return status.Errorf(codes.Internal, "unable to generate authentication info for registry: %s", err) + } + encodedAuth = base64.URLEncoding.EncodeToString(buf) + } + + step = sg.Add("Pushing Docker image...") + + options := types.ImagePushOptions{ + RegistryAuth: encodedAuth, + } + + responseBody, err := cli.ImagePush(ctx, reference.FamiliarString(ref), options) + if err != nil { + return status.Errorf(codes.Internal, "unable to push image to registry: %s", err) + } + + defer responseBody.Close() + + var termFd uintptr + if f, ok := stdout.(*os.File); ok { + termFd = f.Fd() + } + + err = jsonmessage.DisplayJSONMessagesStream(responseBody, step.TermOutput(), termFd, true, nil) + if err != nil { + return status.Errorf(codes.Internal, "unable to stream Docker logs to terminal: %s", err) + } + + step.Done() + return nil +} diff --git a/builtin/docker/registry_img.go b/builtin/docker/registry_img.go new file mode 100644 index 00000000000..b7785ef8c70 --- /dev/null +++ b/builtin/docker/registry_img.go @@ -0,0 +1,174 @@ +package docker + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/types" + "github.com/docker/distribution/reference" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/waypoint-plugin-sdk/terminal" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (r *Registry) pushWithImg( + ctx context.Context, + log hclog.Logger, + ui terminal.UI, + source *Image, + target *Image, +) error { + sg := ui.StepGroup() + defer sg.Wait() + var step terminal.Step + defer func() { + if step != nil { + step.Abort() + } + }() + + step = sg.Add("Preparing Docker configuration...") + env := os.Environ() + if path, err := r.createDockerConfig(log, target); err != nil { + return err + } else if path != "" { + defer os.RemoveAll(path) + env = append(env, "DOCKER_CONFIG="+path) + } + + step.Done() + step = sg.Add("Tagging the image from %s => %s", source.Name(), target.Name()) + + // Tag + cmd := exec.CommandContext(ctx, + "img", + "tag", + source.Name(), + target.Name(), + ) + cmd.Env = env + cmd.Stdout = step.TermOutput() // Command output should go to the step + cmd.Stderr = cmd.Stdout + if err := cmd.Run(); err != nil { + return err + } + + step.Done() + + // If we're a local image, then we don't want to push, just tag. + if r.config.Local { + return nil + } + + step = sg.Add("Pushing image...") + + // Push the image + var buf bytes.Buffer + cmd = exec.CommandContext(ctx, + "img", + "push", + target.Name(), + ) + cmd.Env = env + cmd.Stdout = io.MultiWriter(&buf, step.TermOutput()) // Command output should go to the step + cmd.Stderr = cmd.Stdout + buf.Reset() + if err := cmd.Run(); err != nil { + return status.Errorf(codes.Internal, + "Failure while pushing image: %s\n\n%s", err, buf.String()) + } + + step.Done() + return nil +} + +// createDockerConfig creates a new Docker configuration with the +// configured auth in it. It saves this Docker config to a temporary path +// and returns the path to that Docker file. +// +// We have to do this because `img` doesn't support setting auth for +// a single operation. Therefore, we must set auth in the Docker config, +// but we don't want to pollute any concurrent runs or the main file. So +// we create a copy. +// +// This can return ("", nil) if there is no custom Docker config necessary. +// +// Callers should defer file deletion for this temporary file. +func (r *Registry) createDockerConfig( + log hclog.Logger, + target *Image, +) (string, error) { + if r.config.EncodedAuth == "" { + return "", nil + } + + // Create a reader that base64 decodes our encoded auth and then + // JSON decodes that. + var authCfg types.AuthConfig + var rdr io.Reader = strings.NewReader(r.config.EncodedAuth) + rdr = base64.NewDecoder(base64.URLEncoding, rdr) + dec := json.NewDecoder(rdr) + if err := dec.Decode(&authCfg); err != nil { + return "", status.Errorf(codes.FailedPrecondition, + "Failed to decode encoded_auth: %s", err) + } + + // Determine the host that we're setting auth for. We have to parse the + // image for this cause it may not contain a host. Luckily Docker has + // libs to normalize this all for us. + log.Trace("determining host for auth configuration", "image", target.Name()) + ref, err := reference.ParseNormalizedNamed(target.Name()) + if err != nil { + return "", status.Errorf(codes.Internal, "unable to parse image name: %s", err) + } + host := reference.Domain(ref) + log.Trace("auth host", "host", host) + + // Parse our old Docker config and add the auth. + log.Trace("loading Docker configuration") + file, err := config.Load(config.Dir()) + if err != nil { + return "", err + } + + if file.AuthConfigs == nil { + file.AuthConfigs = map[string]types.AuthConfig{} + } + file.AuthConfigs[host] = authCfg + + // Create a temporary directory for our config + td, err := ioutil.TempDir("", "wp-docker-config") + if err != nil { + return "", status.Errorf(codes.Internal, + "Failed to create temporary directory for Docker config: %s", err) + } + + // Create a temporary file and write our Docker config to it + f, err := os.Create(filepath.Join(td, "config.json")) + if err != nil { + return "", status.Errorf(codes.Internal, + "Failed to create temporary file for Docker config: %s", err) + } + defer f.Close() + if err := file.SaveToWriter(f); err != nil { + return "", status.Errorf(codes.Internal, + "Failed to create temporary file for Docker config: %s", err) + } + + log.Info("temporary Docker config created for auth", + "auth_host", host, + "path", td, + ) + + return td, nil +} diff --git a/go.mod b/go.mod index e84722c357a..769fee38b74 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( github.com/oklog/ulid v1.3.1 github.com/oklog/ulid/v2 v2.0.2 github.com/olekukonko/tablewriter v0.0.4 + github.com/opencontainers/image-spec v1.0.1 github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.3 github.com/r3labs/diff v1.1.0 diff --git a/internal/pkg/epinject/epinject.go b/internal/pkg/epinject/epinject.go index dcfe0efbb45..aaff3a916a4 100644 --- a/internal/pkg/epinject/epinject.go +++ b/internal/pkg/epinject/epinject.go @@ -45,7 +45,11 @@ type InjectFile struct { Info os.FileInfo } -func AlterEntrypoint(ctx context.Context, image string, f func(cur []string) (*NewEntrypoint, error)) (string, error) { +func AlterEntrypoint( + ctx context.Context, + image string, + f func(cur []string) (*NewEntrypoint, error), +) (string, error) { dc, err := dockerClient(ctx) if err != nil { return "", err diff --git a/internal/pkg/epinject/img.go b/internal/pkg/epinject/img.go new file mode 100644 index 00000000000..72bfd2e8cf8 --- /dev/null +++ b/internal/pkg/epinject/img.go @@ -0,0 +1,223 @@ +package epinject + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + + "github.com/hashicorp/go-hclog" + "github.com/oklog/ulid" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// AlterEntrypointImg has the same signature as AlterEntrypoint but uses +// "img" under the hood to perform the entrypoint modification. +func AlterEntrypointImg( + ctx context.Context, + image string, + cb func(cur []string) (*NewEntrypoint, error), +) (string, error) { + L := hclog.FromContext(ctx).With("image", image) + L.Debug("altering entrypoint of docker image using img") + + // Create a temporary directory. We do this in case img creates state + // (it does not in the current directory but in case it ever does) + td, err := ioutil.TempDir("", "wp-epinject") + if err != nil { + return "", err + } + defer os.RemoveAll(td) + + // Determine the existing entrypoint + L.Debug("executing img inspect to determine existing entrypoint") + var buf bytes.Buffer + cmd := exec.CommandContext(ctx, + "img", + "inspect", + image, + ) + cmd.Dir = td + cmd.Stdout = &buf + cmd.Stderr = cmd.Stdout + if err := cmd.Run(); err != nil { + L.Warn("img inspect failed", "err", err, "output", buf.String()) + return "", err + } + + // Parse the image + var imageSpec ocispec.Image + if err := json.Unmarshal(buf.Bytes(), &imageSpec); err != nil { + return "", err + } + + L.Debug("extracted existing entrypoint", "entrypoint", imageSpec.Config.Entrypoint) + + // Determine the new entrypoint configuration based on the existing + newEp, err := cb(imageSpec.Config.Entrypoint) + if err != nil { + return "", err + } + if newEp.Entrypoint == nil { + newEp.Entrypoint = imageSpec.Config.Entrypoint + } + if newEp.NewImage == "" { + newEp.NewImage = image + } + + // Create a random name for our new container image + u, err := ulid.New(ulid.Now(), rand.Reader) + if err != nil { + return "", err + } + name := strings.ToLower("epinject-" + u.String()) + + // Build our template data. The entrypoint generates the actual + // entrypoint string to use in the Dockerfile. + var tplData tplData + tplData.Base = image + if len(newEp.Entrypoint) > 0 { + v, err := json.Marshal(newEp.Entrypoint) + if err != nil { + return "", err + } + + tplData.Entrypoint = string(v) + } + + // For every file, we copy it into our temporary directory so it can be copied. + L.Debug("copying files for injection", "n", len(newEp.InjectFiles)) + idx := 0 + for containerPath, finfo := range newEp.InjectFiles { + // Local path for our file copy + localPath := filepath.Join(td, fmt.Sprintf("%d", idx)) + idx++ + L.Trace("staging file for copy", "from", localPath, "to", containerPath) + + f, err := os.Create(localPath) + if err != nil { + return "", err + } + if _, err := io.Copy(f, finfo.Reader); err != nil { + f.Close() + return "", err + } + if err := f.Sync(); err != nil { + f.Close() + return "", err + } + + // We have to chmod the file because Buildkit inherits the file + // permissions of the source file. This doesn't work on Windows but + // we don't support img on Windows so that's okay. + if err := f.Chmod(finfo.Info.Mode()); err != nil { + f.Close() + return "", err + } + + f.Close() + + tplData.Copy = append(tplData.Copy, tplDataFile{ + From: "./" + filepath.Base(localPath), + To: containerPath, + }) + } + + // Render our Dockerfile into our directory + dockerfilePath := filepath.Join(td, "Dockerfile") + f, err := os.Create(dockerfilePath) + if err != nil { + return "", err + } + tpl := template.Must(template.New("dockerfile").Parse(dockerfileTemplate)) + err = tpl.Execute(f, &tplData) + f.Close() + if err != nil { + return "", err + } + + // Execute the build + cmd = exec.CommandContext(ctx, + "img", + "build", + "-f", dockerfilePath, + "-t", name, + ".", + ) + cmd.Dir = td + cmd.Stdout = &buf + cmd.Stderr = cmd.Stdout + buf.Reset() + L.Debug("executing img build for injection", "args", cmd.Args) + if err := cmd.Run(); err != nil { + L.Warn("failed to inject", "err", err, "output", buf.String()) + return "", err + } + + // Retag to the final name + cmd = exec.CommandContext(ctx, + "img", + "tag", + name, + newEp.NewImage, + ) + cmd.Dir = td + cmd.Stdout = &buf + cmd.Stderr = cmd.Stdout + buf.Reset() + L.Debug("executing img tag to rename", "args", cmd.Args) + if err := cmd.Run(); err != nil { + L.Warn("failed to tag", "err", err, "output", buf.String()) + return "", err + } + + // Remove the temporary image + cmd = exec.CommandContext(ctx, + "img", + "rm", + name, + ) + cmd.Dir = td + cmd.Stdout = &buf + cmd.Stderr = cmd.Stdout + buf.Reset() + L.Debug("removing build image", "args", cmd.Args) + if err := cmd.Run(); err != nil { + L.Warn("failed to remove build image", "err", err, "output", buf.String()) + return "", err + } + + // Create the image + return newEp.NewImage, nil +} + +const dockerfileTemplate = ` +FROM {{.Base}} + +{{range $file := .Copy}} +COPY {{.From}} {{.To}} +{{end}} + +{{if .Entrypoint}} +ENTRYPOINT {{.Entrypoint}} +{{end}} +` + +type tplData struct { + Base string // Base image + Copy []tplDataFile + Entrypoint string +} + +type tplDataFile struct { + From string + To string +} diff --git a/internal/runner/ui.go b/internal/runner/ui.go index 571d175a927..dddf31b6e2d 100644 --- a/internal/runner/ui.go +++ b/internal/runner/ui.go @@ -20,6 +20,11 @@ type runnerUI struct { mu *sync.Mutex evc pb.Waypoint_RunnerJobStreamClient + // stepIdx keeps track of the current step "ID" used when talking to + // the server. Each new stepgroup step gets a new monotonically increasing + // ID. You must never reuse an old ID. + stepIdx int32 + stdSetup sync.Once stdout, stderr io.Writer } @@ -548,9 +553,12 @@ func (u *runnerUISG) Add(str string, args ...interface{}) terminal.Step { u.wg.Add(1) + stepIdx := u.ui.stepIdx + u.ui.stepIdx++ + step := &runnerUISGStep{ sg: u, - id: int32(len(u.steps)), + id: stepIdx, } u.steps = append(u.steps, step) @@ -601,6 +609,9 @@ func (u *runnerUISG) Wait() { } func (u *runnerUI) StepGroup() terminal.StepGroup { + u.mu.Lock() + defer u.mu.Unlock() + ctx, cancel := context.WithCancel(u.ctx) sg := &runnerUISG{ diff --git a/internal/serverinstall/docker.go b/internal/serverinstall/docker.go index 6c4a1ec9b49..7a1f618b13d 100644 --- a/internal/serverinstall/docker.go +++ b/internal/serverinstall/docker.go @@ -291,7 +291,14 @@ func (i *DockerInstaller) InstallRunner( Labels: map[string]string{ "waypoint-type": "runner", }, - }, nil, &network.NetworkingConfig{ + }, &container.HostConfig{ + // These security options are required for the runner so that + // Docker daemonless image building works properly. + SecurityOpt: []string{ + "seccomp=unconfined", + "apparmor=unconfined", + }, + }, &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ "waypoint": {}, },