From 1cfbf1059b1bd142a6ea3d3b8e03b815cde4305a Mon Sep 17 00:00:00 2001 From: Yan Song Date: Fri, 14 Oct 2022 09:05:07 +0000 Subject: [PATCH] iohelper: move some public codes into Signed-off-by: Yan Song --- cache/converter.go | 11 +----- cache/manager_test.go | 5 ++- util/compression/compression.go | 51 +------------------------- util/compression/estargz.go | 5 ++- util/compression/uncompressed.go | 11 +----- util/iohelper/helper.go | 63 ++++++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 71 deletions(-) create mode 100644 util/iohelper/helper.go diff --git a/cache/converter.go b/cache/converter.go index 326af241e408..8b9c22924b89 100644 --- a/cache/converter.go +++ b/cache/converter.go @@ -14,6 +14,7 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/util/iohelper" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -76,7 +77,7 @@ func (c *conversion) convert(ctx context.Context, cs content.Store, desc ocispec bufW = bufio.NewWriterSize(w, 128*1024) } defer bufioPool.Put(bufW) - zw, err := c.compress(&nopWriteCloser{bufW}, c.target.Type.MediaType()) + zw, err := c.compress(&iohelper.NopWriteCloser{Writer: bufW}, c.target.Type.MediaType()) if err != nil { return nil, err } @@ -128,14 +129,6 @@ func (c *conversion) convert(ctx context.Context, cs content.Store, desc ocispec return &newDesc, nil } -type nopWriteCloser struct { - io.Writer -} - -func (w *nopWriteCloser) Close() error { - return nil -} - type onceWriteCloser struct { io.WriteCloser closeOnce sync.Once diff --git a/cache/manager_test.go b/cache/manager_test.go index f052ec5c5472..e7677cb0fa45 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -42,6 +42,7 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" + "github.com/moby/buildkit/util/iohelper" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/winlayers" digest "github.com/opencontainers/go-digest" @@ -1481,7 +1482,7 @@ func getCompressor(w io.Writer, compressionType compression.Type, customized boo } pr.Close() }() - return &compression.WriteCloser{WriteCloser: pw, CloseFunc: func() error { <-done; return nil }}, nil + return &iohelper.WriteCloser{WriteCloser: pw, CloseFunc: func() error { <-done; return nil }}, nil case compression.Zstd: if customized { skippableFrameMagic := []byte{0x50, 0x2a, 0x4d, 0x18} @@ -1998,7 +1999,7 @@ func checkDescriptor(ctx context.Context, t *testing.T, cs content.Store, desc o } // Check annotation values are valid - c := new(compression.Counter) + c := new(iohelper.Counter) ra, err := cs.ReaderAt(ctx, desc) if err != nil && errdefs.IsNotFound(err) { return // lazy layer diff --git a/util/compression/compression.go b/util/compression/compression.go index 8f040fb31f72..9559c95b6c10 100644 --- a/util/compression/compression.go +++ b/util/compression/compression.go @@ -5,12 +5,12 @@ import ( "context" "fmt" "io" - "sync" cdcompression "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/moby/buildkit/util/iohelper" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -253,52 +253,5 @@ func decompress(ctx context.Context, cs content.Store, desc ocispecs.Descriptor) return nil, err } } - return &readCloser{r, ra.Close}, nil -} - -type readCloser struct { - io.ReadCloser - closeFunc func() error -} - -func (rc *readCloser) Close() error { - err1 := rc.ReadCloser.Close() - err2 := rc.closeFunc() - if err1 != nil { - return errors.Wrapf(err1, "failed to close: %v", err2) - } - return err2 -} - -type WriteCloser struct { - io.WriteCloser - CloseFunc func() error -} - -func (wc *WriteCloser) Close() error { - err1 := wc.WriteCloser.Close() - err2 := wc.CloseFunc() - if err1 != nil { - return errors.Wrapf(err1, "failed to close: %v", err2) - } - return err2 -} - -type Counter struct { - n int64 - mu sync.Mutex -} - -func (c *Counter) Write(p []byte) (n int, err error) { - c.mu.Lock() - c.n += int64(len(p)) - c.mu.Unlock() - return len(p), nil -} - -func (c *Counter) Size() (n int64) { - c.mu.Lock() - n = c.n - c.mu.Unlock() - return + return &iohelper.ReadCloser{ReadCloser: r, CloseFunc: ra.Close}, nil } diff --git a/util/compression/estargz.go b/util/compression/estargz.go index aa9bb107e84d..358fc6b317ea 100644 --- a/util/compression/estargz.go +++ b/util/compression/estargz.go @@ -13,6 +13,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/containerd/stargz-snapshotter/estargz" + "github.com/moby/buildkit/util/iohelper" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -81,7 +82,7 @@ func (c estargzType) Compress(comp Config) (compressorFunc Compressor, finalize pr.Close() return nil }() - return &WriteCloser{pw, func() error { + return &iohelper.WriteCloser{WriteCloser: pw, CloseFunc: func() error { <-done // wait until the write completes return nil }}, nil @@ -228,7 +229,7 @@ func calculateBlobInfo() (io.WriteCloser, chan blobInfo) { pr, pw := io.Pipe() go func() { defer pr.Close() - c := new(Counter) + c := new(iohelper.Counter) dgstr := digest.Canonical.Digester() diffID := digest.Canonical.Digester() decompressR, err := cdcompression.DecompressStream(io.TeeReader(pr, dgstr.Hash())) diff --git a/util/compression/uncompressed.go b/util/compression/uncompressed.go index 476582628bd5..b2e7197c909c 100644 --- a/util/compression/uncompressed.go +++ b/util/compression/uncompressed.go @@ -7,20 +7,13 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" "github.com/docker/docker/pkg/ioutils" + "github.com/moby/buildkit/util/iohelper" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) -type nopWriteCloser struct { - io.Writer -} - -func (w *nopWriteCloser) Close() error { - return nil -} - func (c uncompressedType) Compress(comp Config) (compressorFunc Compressor, finalize Finalizer) { return func(dest io.Writer, mediaType string) (io.WriteCloser, error) { - return &nopWriteCloser{dest}, nil + return &iohelper.NopWriteCloser{Writer: dest}, nil }, nil } diff --git a/util/iohelper/helper.go b/util/iohelper/helper.go new file mode 100644 index 000000000000..e0ebaf9bb584 --- /dev/null +++ b/util/iohelper/helper.go @@ -0,0 +1,63 @@ +package iohelper + +import ( + "io" + "sync" + + "github.com/pkg/errors" +) + +type NopWriteCloser struct { + io.Writer +} + +func (w *NopWriteCloser) Close() error { + return nil +} + +type ReadCloser struct { + io.ReadCloser + CloseFunc func() error +} + +func (rc *ReadCloser) Close() error { + err1 := rc.ReadCloser.Close() + err2 := rc.CloseFunc() + if err1 != nil { + return errors.Wrapf(err1, "failed to close: %v", err2) + } + return err2 +} + +type WriteCloser struct { + io.WriteCloser + CloseFunc func() error +} + +func (wc *WriteCloser) Close() error { + err1 := wc.WriteCloser.Close() + err2 := wc.CloseFunc() + if err1 != nil { + return errors.Wrapf(err1, "failed to close: %v", err2) + } + return err2 +} + +type Counter struct { + n int64 + mu sync.Mutex +} + +func (c *Counter) Write(p []byte) (n int, err error) { + c.mu.Lock() + c.n += int64(len(p)) + c.mu.Unlock() + return len(p), nil +} + +func (c *Counter) Size() (n int64) { + c.mu.Lock() + n = c.n + c.mu.Unlock() + return +}