diff --git a/CODEOWNERS b/CODEOWNERS index d5f9420..3f795a2 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -9,16 +9,4 @@ # The format is described: https://github.blog/2017-07-06-introducing-code-owners/ # These owners will be the default owners for everything in the repo. -* @alecthomas @js-murph - - -# ----------------------------------------------- -# BELOW THIS LINE ARE TEMPLATES, UNUSED -# ----------------------------------------------- -# Order is important. The last matching pattern has the most precedence. -# So if a pull request only touches javascript files, only these owners -# will be requested to review. -# *.js @octocat @github/js - -# You can also use email addresses if you prefer. -# docs/* docs@example.com +* @block/cachew-team diff --git a/cmd/cachew/main.go b/cmd/cachew/main.go index 52bd3b0..b2bcfac 100644 --- a/cmd/cachew/main.go +++ b/cmd/cachew/main.go @@ -1,15 +1,202 @@ package main import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/alecthomas/errors" "github.com/alecthomas/kong" + "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/logging" + "github.com/block/cachew/internal/snapshot" ) -var cli struct { - logging.Config +type CLI struct { + LoggingConfig logging.Config `embed:"" prefix:"log-"` + + URL string `help:"Remote cache server URL." default:"http://127.0.0.1:8080"` + Platform bool `help:"Prefix keys with platform ($${os}-$${arch}-)."` + + Get GetCmd `cmd:"" help:"Download object from cache." group:"Operations:"` + Stat StatCmd `cmd:"" help:"Show metadata for cached object." group:"Operations:"` + Put PutCmd `cmd:"" help:"Upload object to cache." group:"Operations:"` + Delete DeleteCmd `cmd:"" help:"Remove object from cache." group:"Operations:"` + + Snapshot SnapshotCmd `cmd:"" help:"Create compressed archive of directory and upload." group:"Snapshots:"` + Restore RestoreCmd `cmd:"" help:"Download and extract archive to directory." group:"Snapshots:"` } func main() { - kong.Parse(&cli) + cli := CLI{} + kctx := kong.Parse(&cli, kong.UsageOnError(), kong.HelpOptions{Compact: true}, kong.DefaultEnvars("CACHEW"), kong.Bind(&cli)) + ctx := context.Background() + _, ctx = logging.Configure(ctx, cli.LoggingConfig) + + remote := cache.NewRemote(cli.URL) + defer remote.Close() + + kctx.BindTo(ctx, (*context.Context)(nil)) + kctx.BindTo(remote, (*cache.Cache)(nil)) + kctx.FatalIfErrorf(kctx.Run(ctx)) +} + +type GetCmd struct { + Key PlatformKey `arg:"" help:"Object key (hex or string)."` + Output *os.File `short:"o" help:"Output file (default: stdout)." default:"-"` +} + +func (c *GetCmd) Run(ctx context.Context, cache cache.Cache) error { + defer c.Output.Close() + + rc, headers, err := cache.Open(ctx, c.Key.Key()) + if err != nil { + return errors.Wrap(err, "failed to open object") + } + defer rc.Close() + + for key, values := range headers { + for _, value := range values { + fmt.Fprintf(os.Stderr, "%s: %s\n", key, value) //nolint:forbidigo + } + } + + _, err = io.Copy(c.Output, rc) + return errors.Wrap(err, "failed to copy data") +} + +type StatCmd struct { + Key PlatformKey `arg:"" help:"Object key (hex or string)."` +} + +func (c *StatCmd) Run(ctx context.Context, cache cache.Cache) error { + headers, err := cache.Stat(ctx, c.Key.Key()) + if err != nil { + return errors.Wrap(err, "failed to stat object") + } + + for key, values := range headers { + for _, value := range values { + fmt.Printf("%s: %s\n", key, value) //nolint:forbidigo + } + } + + return nil +} + +type PutCmd struct { + Key PlatformKey `arg:"" help:"Object key (hex or string)."` + Input *os.File `arg:"" help:"Input file (default: stdin)." default:"-"` + TTL time.Duration `help:"Time to live for the object."` + Headers map[string]string `short:"H" help:"Additional headers (key=value)."` +} + +func (c *PutCmd) Run(ctx context.Context, cache cache.Cache) error { + defer c.Input.Close() + + headers := make(http.Header) + for key, value := range c.Headers { + headers.Set(key, value) + } + + if filename := getFilename(c.Input); filename != "" { + headers.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filepath.Base(filename))) //nolint:perfsprint + } + + wc, err := cache.Create(ctx, c.Key.Key(), headers, c.TTL) + if err != nil { + return errors.Wrap(err, "failed to create object") + } + + if _, err := io.Copy(wc, c.Input); err != nil { + return errors.Join(errors.Wrap(err, "failed to copy data"), wc.Close()) + } + + return errors.Wrap(wc.Close(), "failed to close writer") +} + +type DeleteCmd struct { + Key PlatformKey `arg:"" help:"Object key (hex or string)."` +} + +func (c *DeleteCmd) Run(ctx context.Context, cache cache.Cache) error { + return errors.Wrap(cache.Delete(ctx, c.Key.Key()), "failed to delete object") +} + +type SnapshotCmd struct { + Key PlatformKey `arg:"" help:"Object key (hex or string)."` + Directory string `arg:"" help:"Directory to archive." type:"path"` + TTL time.Duration `help:"Time to live for the object."` + Exclude []string `help:"Patterns to exclude (tar --exclude syntax)."` +} + +func (c *SnapshotCmd) Run(ctx context.Context, cache cache.Cache) error { + fmt.Fprintf(os.Stderr, "Archiving %s...\n", c.Directory) //nolint:forbidigo + if err := snapshot.Create(ctx, cache, c.Key.Key(), c.Directory, c.TTL, c.Exclude); err != nil { + return errors.Wrap(err, "failed to create snapshot") + } + + fmt.Fprintf(os.Stderr, "Snapshot uploaded: %s\n", c.Key.String()) //nolint:forbidigo + return nil +} + +type RestoreCmd struct { + Key PlatformKey `arg:"" help:"Object key (hex or string)."` + Directory string `arg:"" help:"Target directory for extraction." type:"path"` +} + +func (c *RestoreCmd) Run(ctx context.Context, cache cache.Cache) error { + fmt.Fprintf(os.Stderr, "Restoring to %s...\n", c.Directory) //nolint:forbidigo + if err := snapshot.Restore(ctx, cache, c.Key.Key(), c.Directory); err != nil { + return errors.Wrap(err, "failed to restore snapshot") + } + + fmt.Fprintf(os.Stderr, "Snapshot restored: %s\n", c.Key.String()) //nolint:forbidigo + return nil +} + +func getFilename(f *os.File) string { + info, err := f.Stat() + if err != nil { + return "" + } + + if !info.Mode().IsRegular() { + return "" + } + + return f.Name() +} + +// PlatformKey wraps a cache.Key and stores the original input for platform prefixing. +type PlatformKey struct { + raw string + key cache.Key +} + +func (pk *PlatformKey) UnmarshalText(text []byte) error { + pk.raw = string(text) + return errors.WithStack(pk.key.UnmarshalText(text)) +} + +func (pk *PlatformKey) Key() cache.Key { + return pk.key +} + +func (pk *PlatformKey) String() string { + return pk.key.String() +} + +func (pk *PlatformKey) AfterApply(cli *CLI) error { + if !cli.Platform { + return nil + } + prefixed := fmt.Sprintf("%s-%s-%s", runtime.GOOS, runtime.GOARCH, pk.raw) + return errors.WithStack(pk.key.UnmarshalText([]byte(prefixed))) } diff --git a/internal/cache/api.go b/internal/cache/api.go index 2dbdaaa..d794872 100644 --- a/internal/cache/api.go +++ b/internal/cache/api.go @@ -80,14 +80,16 @@ func NewKey(url string) Key { return Key(sha256.Sum256([]byte(url))) } func (k *Key) String() string { return hex.EncodeToString(k[:]) } func (k *Key) UnmarshalText(text []byte) error { - bytes, err := hex.DecodeString(string(text)) - if err != nil { - return errors.WithStack(err) - } - if len(bytes) != len(*k) { - return errors.New("invalid key length") + // Try to decode as SHA256 hex encoded string + if len(text) == 64 { + bytes, err := hex.DecodeString(string(text)) + if err == nil && len(bytes) == len(*k) { + copy(k[:], bytes) + return nil + } } - copy(k[:], bytes) + // If not valid hex, treat as string and SHA256 it + *k = NewKey(string(text)) return nil } diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go new file mode 100644 index 0000000..2e8f5bb --- /dev/null +++ b/internal/snapshot/snapshot.go @@ -0,0 +1,140 @@ +// Package snapshot provides streaming directory archival and restoration using tar and zstd. +package snapshot + +import ( + "bytes" + "context" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/alecthomas/errors" + + "github.com/block/cachew/internal/cache" +) + +// Create archives a directory using tar with zstd compression, then uploads to the cache. +// +// The archive preserves all file permissions, ownership, and symlinks. +// The operation is fully streaming - no temporary files are created. +// Exclude patterns use tar's --exclude syntax. +func Create(ctx context.Context, remote cache.Cache, key cache.Key, directory string, ttl time.Duration, excludePatterns []string) error { + // Verify directory exists + if info, err := os.Stat(directory); err != nil { + return errors.Wrap(err, "failed to stat directory") + } else if !info.IsDir() { + return errors.Errorf("not a directory: %s", directory) + } + + headers := make(http.Header) + headers.Set("Content-Type", "application/zstd") + headers.Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filepath.Base(directory)+".tar.zst")) + + wc, err := remote.Create(ctx, key, headers, ttl) + if err != nil { + return errors.Wrap(err, "failed to create object") + } + + tarArgs := []string{"-cpf", "-", "-C", directory} + for _, pattern := range excludePatterns { + tarArgs = append(tarArgs, "--exclude", pattern) + } + tarArgs = append(tarArgs, ".") + + tarCmd := exec.CommandContext(ctx, "tar", tarArgs...) + zstdCmd := exec.CommandContext(ctx, "zstd", "-c", "-T0") + + tarStdout, err := tarCmd.StdoutPipe() + if err != nil { + return errors.Join(errors.Wrap(err, "failed to create tar stdout pipe"), wc.Close()) + } + + var tarStderr, zstdStderr bytes.Buffer + tarCmd.Stderr = &tarStderr + + zstdCmd.Stdin = tarStdout + zstdCmd.Stdout = wc + zstdCmd.Stderr = &zstdStderr + + if err := tarCmd.Start(); err != nil { + return errors.Join(errors.Wrap(err, "failed to start tar"), wc.Close()) + } + + if err := zstdCmd.Start(); err != nil { + return errors.Join(errors.Wrap(err, "failed to start zstd"), tarCmd.Wait(), wc.Close()) + } + + tarErr := tarCmd.Wait() + zstdErr := zstdCmd.Wait() + closeErr := wc.Close() + + var errs []error + if tarErr != nil { + errs = append(errs, errors.Errorf("tar failed: %w: %s", tarErr, tarStderr.String())) + } + if zstdErr != nil { + errs = append(errs, errors.Errorf("zstd failed: %w: %s", zstdErr, zstdStderr.String())) + } + if closeErr != nil { + errs = append(errs, errors.Wrap(closeErr, "failed to close writer")) + } + + return errors.Join(errs...) +} + +// Restore downloads an archive from the cache and extracts it to a directory. +// +// The archive is decompressed with zstd and extracted with tar, preserving +// all file permissions, ownership, and symlinks. +// The operation is fully streaming - no temporary files are created. +func Restore(ctx context.Context, remote cache.Cache, key cache.Key, directory string) error { + rc, _, err := remote.Open(ctx, key) + if err != nil { + return errors.Wrap(err, "failed to open object") + } + defer rc.Close() + + // Create target directory if it doesn't exist + if err := os.MkdirAll(directory, 0o750); err != nil { + return errors.Wrap(err, "failed to create target directory") + } + + zstdCmd := exec.CommandContext(ctx, "zstd", "-dc", "-T0") + tarCmd := exec.CommandContext(ctx, "tar", "-xpf", "-", "-C", directory) + + zstdCmd.Stdin = rc + zstdStdout, err := zstdCmd.StdoutPipe() + if err != nil { + return errors.Wrap(err, "failed to create zstd stdout pipe") + } + + var zstdStderr, tarStderr bytes.Buffer + zstdCmd.Stderr = &zstdStderr + + tarCmd.Stdin = zstdStdout + tarCmd.Stderr = &tarStderr + + if err := zstdCmd.Start(); err != nil { + return errors.Wrap(err, "failed to start zstd") + } + + if err := tarCmd.Start(); err != nil { + return errors.Join(errors.Wrap(err, "failed to start tar"), zstdCmd.Wait()) + } + + zstdErr := zstdCmd.Wait() + tarErr := tarCmd.Wait() + + var errs []error + if zstdErr != nil { + errs = append(errs, errors.Errorf("zstd failed: %w: %s", zstdErr, zstdStderr.String())) + } + if tarErr != nil { + errs = append(errs, errors.Errorf("tar failed: %w: %s", tarErr, tarStderr.String())) + } + + return errors.Join(errs...) +} diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go new file mode 100644 index 0000000..b691947 --- /dev/null +++ b/internal/snapshot/snapshot_test.go @@ -0,0 +1,287 @@ +package snapshot_test + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "testing" + "time" + + "github.com/alecthomas/assert/v2" + + "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/logging" + "github.com/block/cachew/internal/snapshot" +) + +func TestCreateAndRestoreRoundTrip(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "file1.txt"), []byte("content1"), 0o644)) + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "file2.txt"), []byte("content2"), 0o600)) + assert.NoError(t, os.Mkdir(filepath.Join(srcDir, "subdir"), 0o755)) + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "subdir", "file3.txt"), []byte("content3"), 0o644)) + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + headers, err := mem.Stat(ctx, key) + assert.NoError(t, err) + assert.Equal(t, "application/zstd", headers.Get("Content-Type")) + + dstDir := t.TempDir() + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.NoError(t, err) + + content1, err := os.ReadFile(filepath.Join(dstDir, "file1.txt")) + assert.NoError(t, err) + assert.Equal(t, "content1", string(content1)) + + content2, err := os.ReadFile(filepath.Join(dstDir, "file2.txt")) + assert.NoError(t, err) + assert.Equal(t, "content2", string(content2)) + + content3, err := os.ReadFile(filepath.Join(dstDir, "subdir", "file3.txt")) + assert.NoError(t, err) + assert.Equal(t, "content3", string(content3)) + + info2, err := os.Stat(filepath.Join(dstDir, "file2.txt")) + assert.NoError(t, err) + assert.Equal(t, os.FileMode(0o600), info2.Mode().Perm()) +} + +func TestCreateWithExcludePatterns(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "include.txt"), []byte("included"), 0o644)) + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "exclude.log"), []byte("excluded"), 0o644)) + assert.NoError(t, os.Mkdir(filepath.Join(srcDir, "logs"), 0o755)) + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "logs", "app.log"), []byte("excluded"), 0o644)) + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, []string{"*.log", "logs"}) + assert.NoError(t, err) + + dstDir := t.TempDir() + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.NoError(t, err) + + _, err = os.Stat(filepath.Join(dstDir, "include.txt")) + assert.NoError(t, err) + + _, err = os.Stat(filepath.Join(dstDir, "exclude.log")) + assert.IsError(t, err, os.ErrNotExist) + + _, err = os.Stat(filepath.Join(dstDir, "logs")) + assert.IsError(t, err, os.ErrNotExist) +} + +func TestCreatePreservesSymlinks(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "target.txt"), []byte("target"), 0o644)) + assert.NoError(t, os.Symlink("target.txt", filepath.Join(srcDir, "link.txt"))) + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + dstDir := t.TempDir() + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.NoError(t, err) + + info, err := os.Lstat(filepath.Join(dstDir, "link.txt")) + assert.NoError(t, err) + assert.Equal(t, os.ModeSymlink, info.Mode()&os.ModeSymlink) + + target, err := os.Readlink(filepath.Join(dstDir, "link.txt")) + assert.NoError(t, err) + assert.Equal(t, "target.txt", target) +} + +func TestCreateNonexistentDirectory(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + err = snapshot.Create(ctx, mem, key, "/nonexistent/directory", time.Hour, nil) + assert.Error(t, err) +} + +func TestCreateNotADirectory(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + tmpFile := filepath.Join(t.TempDir(), "file.txt") + assert.NoError(t, os.WriteFile(tmpFile, []byte("content"), 0o644)) + + err = snapshot.Create(ctx, mem, key, tmpFile, time.Hour, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not a directory") +} + +func TestCreateContextCancellation(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + for i := range 100 { + content := bytes.Repeat([]byte("data"), 10000) + filename := filepath.Join(srcDir, fmt.Sprintf("file%d.txt", i)) + assert.NoError(t, os.WriteFile(filename, content, 0o644)) + } + + cancelCtx, cancel := context.WithCancel(context.Background()) + cancel() + + err = snapshot.Create(cancelCtx, mem, key, srcDir, time.Hour, nil) + assert.Error(t, err) +} + +func TestRestoreNonexistentKey(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + dstDir := t.TempDir() + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.Error(t, err) +} + +func TestRestoreCreatesTargetDirectory(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "file.txt"), []byte("content"), 0o644)) + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + dstDir := filepath.Join(t.TempDir(), "nested", "target") + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.NoError(t, err) + + content, err := os.ReadFile(filepath.Join(dstDir, "file.txt")) + assert.NoError(t, err) + assert.Equal(t, "content", string(content)) +} + +func TestRestoreContextCancellation(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + for i := range 100 { + content := bytes.Repeat([]byte("data"), 10000) + filename := filepath.Join(srcDir, fmt.Sprintf("file%d.txt", i)) + assert.NoError(t, os.WriteFile(filename, content, 0o644)) + } + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + cancelCtx, cancel := context.WithCancel(context.Background()) + cancel() + + dstDir := t.TempDir() + err = snapshot.Restore(cancelCtx, mem, key, dstDir) + assert.Error(t, err) +} + +func TestCreateEmptyDirectory(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + dstDir := t.TempDir() + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.NoError(t, err) + + entries, err := os.ReadDir(dstDir) + assert.NoError(t, err) + assert.Equal(t, 0, len(entries)) +} + +func TestCreateWithNestedDirectories(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + deepPath := filepath.Join(srcDir, "a", "b", "c", "d", "e") + assert.NoError(t, os.MkdirAll(deepPath, 0o755)) + assert.NoError(t, os.WriteFile(filepath.Join(deepPath, "deep.txt"), []byte("deep content"), 0o644)) + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + dstDir := t.TempDir() + err = snapshot.Restore(ctx, mem, key, dstDir) + assert.NoError(t, err) + + content, err := os.ReadFile(filepath.Join(dstDir, "a", "b", "c", "d", "e", "deep.txt")) + assert.NoError(t, err) + assert.Equal(t, "deep content", string(content)) +} + +func TestCreateSetsCorrectHeaders(t *testing.T) { + ctx := logging.ContextWithLogger(context.Background(), slog.Default()) + mem, err := cache.NewMemory(ctx, cache.MemoryConfig{LimitMB: 100, MaxTTL: time.Hour}) + assert.NoError(t, err) + defer mem.Close() + key := cache.Key{1, 2, 3} + + srcDir := t.TempDir() + assert.NoError(t, os.WriteFile(filepath.Join(srcDir, "file.txt"), []byte("content"), 0o644)) + + err = snapshot.Create(ctx, mem, key, srcDir, time.Hour, nil) + assert.NoError(t, err) + + headers, err := mem.Stat(ctx, key) + assert.NoError(t, err) + assert.Equal(t, "application/zstd", headers.Get("Content-Type")) + assert.Contains(t, headers.Get("Content-Disposition"), "attachment") + assert.Contains(t, headers.Get("Content-Disposition"), ".tar.zst") +} diff --git a/internal/strategy/git/integration_test.go b/internal/strategy/git/integration_test.go index f453b2c..861bfa3 100644 --- a/internal/strategy/git/integration_test.go +++ b/internal/strategy/git/integration_test.go @@ -54,10 +54,10 @@ func TestIntegrationGitCloneViaProxy(t *testing.T) { // Create the git strategy mux := http.NewServeMux() - strategy, err := git.New(ctx, jobscheduler.New(ctx, jobscheduler.Config{}), git.Config{ + strategy, err := git.New(ctx, git.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }, nil, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux) assert.NoError(t, err) assert.NotZero(t, strategy) @@ -67,7 +67,7 @@ func TestIntegrationGitCloneViaProxy(t *testing.T) { // Clone a small public repository through the proxy // Using a small test repo to keep the test fast - repoURL := fmt.Sprintf("%s/github.com/octocat/Hello-World", server.URL) + repoURL := fmt.Sprintf("%s/git/github.com/octocat/Hello-World", server.URL) // First clone - should forward to upstream and start background clone cmd := exec.Command("git", "clone", repoURL, filepath.Join(workDir, "repo1")) @@ -132,16 +132,16 @@ func TestIntegrationGitFetchViaProxy(t *testing.T) { assert.NoError(t, err) mux := http.NewServeMux() - _, err = git.New(ctx, jobscheduler.New(ctx, jobscheduler.Config{}), git.Config{ + _, err = git.New(ctx, git.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }, nil, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux) assert.NoError(t, err) server := testServerWithLogging(ctx, mux) defer server.Close() - repoURL := fmt.Sprintf("%s/github.com/octocat/Hello-World", server.URL) + repoURL := fmt.Sprintf("%s/git/github.com/octocat/Hello-World", server.URL) // Clone first cmd := exec.Command("git", "clone", repoURL, filepath.Join(workDir, "repo")) @@ -211,10 +211,10 @@ func TestIntegrationPushForwardsToUpstream(t *testing.T) { defer upstreamServer.Close() mux := http.NewServeMux() - _, err = git.New(ctx, jobscheduler.New(ctx, jobscheduler.Config{}), git.Config{ + _, err = git.New(ctx, git.Config{ MirrorRoot: clonesDir, FetchInterval: 15, - }, nil, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux) assert.NoError(t, err) server := testServerWithLogging(ctx, mux) @@ -255,7 +255,7 @@ func TestIntegrationPushForwardsToUpstream(t *testing.T) { // Try to push through the proxy - this will fail but should forward to upstream // We're just verifying the forwarding logic, not actual push success - proxyURL := fmt.Sprintf("%s/localhost/test/repo", server.URL) + proxyURL := fmt.Sprintf("%s/git/localhost/test/repo", server.URL) cmd = exec.Command("git", "-C", repoPath, "push", proxyURL, "HEAD:main") cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") _, _ = cmd.CombinedOutput()