Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor backups #9589

Merged
merged 3 commits into from
Apr 29, 2022
Merged

Refactor backups #9589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d05c64d
Refactor backup creation
aledbf Apr 27, 2022
6acc82f
Refactor backup restauration
aledbf Apr 28, 2022
c82c3c1
Remove unused dependencies
aledbf Apr 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions components/content-service/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ go 1.18

require (
cloud.google.com/go/storage v1.15.0
github.com/Microsoft/hcsshim v0.8.17 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/content-service/api v0.0.0-00010101000000-000000000000
Expand All @@ -13,8 +12,6 @@ require (
github.com/google/go-cmp v0.5.7
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/minio/minio-go/v7 v7.0.11
github.com/moby/moby v20.10.7+incompatible
github.com/moby/sys/mount v0.2.0 // indirect
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2
github.com/opentracing/opentracing-go v1.2.0
Expand All @@ -28,6 +25,8 @@ require (
google.golang.org/protobuf v1.28.0
)

require github.com/containers/storage v1.39.0

require (
cloud.google.com/go v0.83.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
Expand All @@ -42,23 +41,29 @@ require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/moby/sys/mountinfo v0.4.1 // indirect
github.com/moby/sys/mountinfo v0.6.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/opencontainers/runc v1.1.0 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/rs/xid v1.2.1 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
go.opencensus.io v0.23.0 // indirect
go.uber.org/atomic v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
Expand Down
87 changes: 76 additions & 11 deletions components/content-service/go.sum
View file
Open in desktop

Large diffs are not rendered by default.

157 changes: 21 additions & 136 deletions components/content-service/pkg/archive/tar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,13 @@
package archive

import (
"archive/tar"
"context"
"io"
"os"
"os/exec"
"path"
"sort"
"syscall"
"time"

"github.com/moby/moby/pkg/system"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/idtools"
"github.com/opentracing/opentracing-go"
"golang.org/x/xerrors"

"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/common-go/tracing"
Expand Down Expand Up @@ -66,138 +60,29 @@ func ExtractTarbal(ctx context.Context, src io.Reader, dst string, opts ...TarOp
opt(&cfg)
}

pr, pw := io.Pipe()
src = io.TeeReader(src, pw)
tarReader := tar.NewReader(pr)

type Info struct {
UID, GID int
IsSymlink bool
Xattrs map[string]string
}

finished := make(chan bool)
m := make(map[string]Info)

go func() {
defer close(finished)
for {
hdr, err := tarReader.Next()
if err == io.EOF {
finished <- true
return
}

if err != nil {
log.WithError(err).Error("error reading tar")
return
}

m[hdr.Name] = Info{
UID: hdr.Uid,
GID: hdr.Gid,
IsSymlink: (hdr.Linkname != ""),
//nolint:staticcheck
Xattrs: hdr.Xattrs,
}
uidMaps := make([]idtools.IDMap, len(cfg.UIDMaps))
for i, m := range cfg.UIDMaps {
uidMaps[i] = idtools.IDMap{
ContainerID: m.ContainerID,
HostID: m.HostID,
Size: m.Size,
}
}()

// Be explicit about the tar flags. We want to restore the exact content without changes
tarcmd := exec.Command(
"tar",
"--extract",
"--preserve-permissions",
"--xattrs", "--xattrs-include=security.capability",
)
tarcmd.Dir = dst
tarcmd.Stdin = src

var msg []byte
msg, err = tarcmd.CombinedOutput()
if err != nil {
return xerrors.Errorf("tar %s: %s", dst, err.Error()+";"+string(msg))
}
<-finished

// lets create a sorted list of pathes and chown depth first.
paths := make([]string, 0, len(m))
for path := range m {
paths = append(paths, path)
}
sort.Sort(sort.Reverse(sort.StringSlice(paths)))

// We need to remap the UID and GID between the host and the container to avoid permission issues.
for _, p := range paths {
v := m[p]
uid := toHostID(v.UID, cfg.UIDMaps)
gid := toHostID(v.GID, cfg.GIDMaps)

if v.IsSymlink {
continue
}

err = remapFile(path.Join(dst, p), uid, gid, v.Xattrs)
if err != nil {
log.WithError(err).WithField("uid", uid).WithField("gid", gid).WithField("path", p).Warn("cannot chown")
gidMaps := make([]idtools.IDMap, len(cfg.GIDMaps))
for i, m := range cfg.GIDMaps {
gidMaps[i] = idtools.IDMap{
ContainerID: m.ContainerID,
HostID: m.HostID,
Size: m.Size,
}
}

log.WithField("duration", time.Since(start).Milliseconds()).Debug("untar complete")
return nil
}
err = archive.Untar(src, dst, &archive.TarOptions{
UIDMaps: uidMaps,
GIDMaps: gidMaps,
Compression: archive.Uncompressed,
})

func toHostID(containerID int, idMap []IDMapping) int {
for _, m := range idMap {
if (containerID >= m.ContainerID) && (containerID <= (m.ContainerID + m.Size - 1)) {
hostID := m.HostID + (containerID - m.ContainerID)
return hostID
}
}
return containerID
}

// remapFile changes the UID and GID of a file preserving existing file mode bits.
func remapFile(name string, uid, gid int, xattrs map[string]string) error {
// current info of the file before any change
fileInfo, err := os.Stat(name)
if err != nil {
return err
}

// nothing to do for symlinks
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
return nil
}

// changing UID or GID can break files with suid/sgid
err = os.Lchown(name, uid, gid)
if err != nil {
return err
}

// restore original permissions
err = os.Chmod(name, fileInfo.Mode())
if err != nil {
return err
}

for key, value := range xattrs {
if err := system.Lsetxattr(name, key, []byte(value), 0); err != nil {
log.WithField("name", key).WithField("value", value).WithField("file", name).WithError(err).Error("restoring extended attributes")
if err == syscall.ENOTSUP || err == syscall.EPERM {
continue
}

return err
}
}

// restore file times
fileTime := fileInfo.Sys().(*syscall.Stat_t)
return os.Chtimes(name, timespecToTime(fileTime.Atim), timespecToTime(fileTime.Mtim))
}

func timespecToTime(ts syscall.Timespec) time.Time {
return time.Unix(int64(ts.Sec), int64(ts.Nsec))
log.WithField("duration", time.Since(start).Milliseconds()).Debug("untar complete")
return
}
13 changes: 8 additions & 5 deletions components/supervisor/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,10 @@ require golang.org/x/net v0.0.0-20211209124913-491a49abca63
require (
cloud.google.com/go v0.83.0 // indirect
cloud.google.com/go/storage v1.15.0 // indirect
github.com/Microsoft/hcsshim v0.8.17 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/containers/storage v1.39.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible // indirect
Expand All @@ -55,20 +55,21 @@ require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/klauspost/compress v1.11.13 // indirect
github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/minio-go/v7 v7.0.11 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/moby/moby v20.10.7+incompatible // indirect
github.com/moby/sys/mount v0.2.0 // indirect
github.com/moby/sys/mountinfo v0.4.1 // indirect
github.com/moby/sys/mountinfo v0.6.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.1.0 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
Expand All @@ -78,8 +79,10 @@ require (
github.com/segmentio/backo-go v0.0.0-20200129164019-23eae7c10bd3 // indirect
github.com/sourcegraph/jsonrpc2 v0.0.0-20200429184054-15c2290dcb37 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
go.opencensus.io v0.23.0 // indirect
go.uber.org/atomic v1.8.0 // indirect
Expand Down
Loading