Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor local storage to avoid duplicating blobs #390

Merged
merged 16 commits into from
Jul 9, 2024
Merged

Refactor local storage to avoid duplicating blobs #390

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7b82b21
Move version vars to pkg/lib
amisevsk Jun 27, 2024
5145695
Move 'resolve kitfile' functions out of cmd/info and into pkg/lib
amisevsk Jun 27, 2024
f08d7cb
Refactor kitfile, repo, and local storage packages to better split them
amisevsk Jun 28, 2024
751b8eb
Add new implementation for local modelkit storage
amisevsk Jun 28, 2024
65e97ac
Switch to new local storage implementation for kit list
amisevsk Jul 2, 2024
0ade319
Use new storage implementation for info/inspect/pack/unpack/remove
amisevsk Jul 2, 2024
349536f
Use new storage implementation for local tagging
amisevsk Jul 2, 2024
0ed6c16
Use new local storage backend for push and pull operations
amisevsk Jul 8, 2024
f010c40
Rework how tagging works in local storage
amisevsk Jul 3, 2024
926df39
Fix deletion on local storage to not remove manifests in other repos
amisevsk Jul 3, 2024
5f198ec
Add automatic migration to new local storage layout
amisevsk Jul 3, 2024
1b4bccb
Add progress bar for storage migration
amisevsk Jul 8, 2024
4a9f7e0
Fix listing repositories when local storage does not exist
amisevsk Jul 8, 2024
f8b8ef2
Clean up minor issues in local storage
amisevsk Jul 8, 2024
e7ff03e
Incrementally clean up storage while migrating to avoid storage issues
amisevsk Jul 8, 2024
9ad4d76
Fix resolving local manfiests by digest and fixup tests
amisevsk Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .goreleaser.darwin.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ builds:
- arm64
binary: kit
ldflags:
- -s -w -X kitops/pkg/cmd/version.Version={{.Version}} -X kitops/pkg/cmd/version.GitCommit={{.Commit}} -X kitops/pkg/cmd/version.BuildTime={{.CommitDate}}
- -s -w -X kitops/pkg/lib/constants.Version={{.Version}} -X kitops/pkg/lib/constants.GitCommit={{.Commit}} -X kitops/pkg/lib/constants.BuildTime={{.CommitDate}}
hooks:
post:
- cmd: ./build/scripts/sign '{{ .Path }}'
Expand Down
2 changes: 1 addition & 1 deletion .goreleaser.linux.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ builds:
- linux
binary: kit
ldflags:
- -s -w -X kitops/pkg/cmd/version.Version={{.Version}} -X kitops/pkg/cmd/version.GitCommit={{.Commit}} -X kitops/pkg/cmd/version.BuildTime={{.CommitDate}}
- -s -w -X kitops/pkg/lib/constants.Version={{.Version}} -X kitops/pkg/lib/constants.GitCommit={{.Commit}} -X kitops/pkg/lib/constants.BuildTime={{.CommitDate}}

archives:
- id: kit-archive
Expand Down
2 changes: 1 addition & 1 deletion .goreleaser.windows.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ builds:
- windows
binary: kit
ldflags:
- -s -w -X kitops/pkg/cmd/version.Version={{.Version}} -X kitops/pkg/cmd/version.GitCommit={{.Commit}} -X kitops/pkg/cmd/version.BuildTime={{.CommitDate}}
- -s -w -X kitops/pkg/lib/constants.Version={{.Version}} -X kitops/pkg/lib/constants.GitCommit={{.Commit}} -X kitops/pkg/lib/constants.BuildTime={{.CommitDate}}

archives:
- id: kit-archive
Expand Down
2 changes: 1 addition & 1 deletion build/dockerfiles/KServe/kserve.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ COPY . .
RUN \
CGO_ENABLED=0 go build \
-o kit \
-ldflags="-s -w -X kitops/pkg/cmd/version.Version=${version} -X kitops/pkg/cmd/version.GitCommit=$gitCommit -X kitops/pkg/cmd/version.BuildTime=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
-ldflags="-s -w -X kitops/pkg/lib/constants.Version=${version} -X kitops/pkg/lib/constants.GitCommit=$gitCommit -X kitops/pkg/lib/constants.BuildTime=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really relevant to this PR but we probably need to add the go generate ./... step.


FROM docker.io/library/alpine
ENV USER_ID=1001 \
Expand Down
12 changes: 12 additions & 0 deletions cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"kitops/pkg/cmd/unpack"
"kitops/pkg/cmd/version"
"kitops/pkg/lib/constants"
"kitops/pkg/lib/repo/local"
"kitops/pkg/output"

"github.com/spf13/cobra"
Expand Down Expand Up @@ -87,6 +88,17 @@ func RunCommand() *cobra.Command {
// returning
cmd.SilenceErrors = true
cmd.SilenceUsage = true

storagePath := constants.StoragePath(configHome)
needsMigration, err := local.NeedsMigrate(storagePath)
if err != nil {
return output.Fatalf("Failed to determine if local modelkit needs to be migrated")
} else if needsMigration {
output.Infof("Migrating local storage to new format")
if err := local.MigrateStorage(ctx, storagePath); err != nil {
return output.Fatalf("Error migrating storage: %s", err)
}
}
return nil
},
}
Expand Down
3 changes: 2 additions & 1 deletion docs/src/docs/cli/gen-doc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ package main
import (
"log"

"github.com/spf13/cobra/doc"
"kitops/cmd"

"github.com/spf13/cobra/doc"
)

func main() {
Expand Down
5 changes: 3 additions & 2 deletions pkg/cmd/dev/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,14 @@
package dev

import (
"kitops/pkg/lib/harness"
"kitops/pkg/output"
"net"
"os"
"runtime"
"strconv"

"kitops/pkg/lib/harness"
"kitops/pkg/output"

"github.com/spf13/cobra"
)

Expand Down
6 changes: 4 additions & 2 deletions pkg/cmd/dev/dev.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ package dev
import (
"context"
"fmt"
"os"

"kitops/pkg/artifact"
"kitops/pkg/lib/filesystem"
"kitops/pkg/lib/harness"
kfutils "kitops/pkg/lib/kitfile"
"kitops/pkg/lib/repo/util"
"kitops/pkg/output"
"os"
)

func runDev(ctx context.Context, options *DevStartOptions) error {
Expand All @@ -39,7 +41,7 @@ func runDev(ctx context.Context, options *DevStartOptions) error {
return err
}
output.Infof("Loaded Kitfile: %s", options.modelFile)
if kfutils.IsModelKitReference(kitfile.Model.Path) {
if util.IsModelKitReference(kitfile.Model.Path) {
resolvedKitfile, err := kfutils.ResolveKitfile(ctx, options.configHome, kitfile.Model.Path, kitfile.Model.Path)
if err != nil {
return fmt.Errorf("Failed to resolve referenced modelkit %s: %w", kitfile.Model.Path, err)
Expand Down
1 change: 1 addition & 0 deletions pkg/cmd/dev/opts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package dev
import (
"context"
"fmt"

"kitops/pkg/lib/constants"
"kitops/pkg/lib/filesystem"
"kitops/pkg/output"
Expand Down
13 changes: 7 additions & 6 deletions pkg/cmd/info/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ import (
"context"
"errors"
"fmt"
"strings"

"kitops/pkg/cmd/options"
"kitops/pkg/lib/constants"
"kitops/pkg/lib/repo"
"kitops/pkg/lib/repo/util"
"kitops/pkg/output"
"strings"

"github.com/spf13/cobra"
"oras.land/oras-go/v2/errdef"
Expand Down Expand Up @@ -80,7 +81,7 @@ func runCommand(opts *infoOptions) func(*cobra.Command, []string) error {
config, err := getInfo(cmd.Context(), opts)
if err != nil {
if errors.Is(err, errdef.ErrNotFound) {
return output.Fatalf("Could not find modelkit %s", repo.FormatRepositoryForDisplay(opts.modelRef.String()))
return output.Fatalf("Could not find modelkit %s", util.FormatRepositoryForDisplay(opts.modelRef.String()))
}
return output.Fatalf("Error resolving modelkit: %s", err)
}
Expand All @@ -100,7 +101,7 @@ func (opts *infoOptions) complete(ctx context.Context, args []string) error {
}
opts.configHome = configHome

ref, extraTags, err := repo.ParseReference(args[0])
ref, extraTags, err := util.ParseReference(args[0])
if err != nil {
return err
}
Expand All @@ -109,8 +110,8 @@ func (opts *infoOptions) complete(ctx context.Context, args []string) error {
}
opts.modelRef = ref

if opts.modelRef.Registry == repo.DefaultRegistry && opts.checkRemote {
return fmt.Errorf("can not check remote: %s does not contain registry", repo.FormatRepositoryForDisplay(opts.modelRef.String()))
if opts.modelRef.Registry == util.DefaultRegistry && opts.checkRemote {
return fmt.Errorf("can not check remote: %s does not contain registry", util.FormatRepositoryForDisplay(opts.modelRef.String()))
}

return nil
Expand Down
13 changes: 8 additions & 5 deletions pkg/cmd/info/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@ package info
import (
"context"
"fmt"

"kitops/pkg/artifact"
"kitops/pkg/lib/constants"
"kitops/pkg/lib/repo"
"kitops/pkg/lib/repo/local"
"kitops/pkg/lib/repo/remote"
"kitops/pkg/lib/repo/util"
)

func getInfo(ctx context.Context, opts *infoOptions) (*artifact.KitFile, error) {
Expand All @@ -34,27 +37,27 @@ func getInfo(ctx context.Context, opts *infoOptions) (*artifact.KitFile, error)

func getLocalConfig(ctx context.Context, opts *infoOptions) (*artifact.KitFile, error) {
storageRoot := constants.StoragePath(opts.configHome)
store, err := repo.NewLocalStore(storageRoot, opts.modelRef)
localRepo, err := local.NewLocalRepo(storageRoot, opts.modelRef)
if err != nil {
return nil, fmt.Errorf("failed to read local storage: %w", err)
}
_, _, config, err := repo.ResolveManifestAndConfig(ctx, store, opts.modelRef.Reference)
_, _, config, err := util.ResolveManifestAndConfig(ctx, localRepo, opts.modelRef.Reference)
if err != nil {
return nil, err
}
return config, nil
}

func getRemoteConfig(ctx context.Context, opts *infoOptions) (*artifact.KitFile, error) {
repository, err := repo.NewRepository(ctx, opts.modelRef.Registry, opts.modelRef.Repository, &repo.RegistryOptions{
repository, err := remote.NewRepository(ctx, opts.modelRef.Registry, opts.modelRef.Repository, &remote.RegistryOptions{
PlainHTTP: opts.PlainHTTP,
SkipTLSVerify: !opts.TlsVerify,
CredentialsPath: constants.CredentialsPath(opts.configHome),
})
if err != nil {
return nil, err
}
_, _, config, err := repo.ResolveManifestAndConfig(ctx, repository, opts.modelRef.Reference)
_, _, config, err := util.ResolveManifestAndConfig(ctx, repository, opts.modelRef.Reference)
if err != nil {
return nil, err
}
Expand Down
56 changes: 0 additions & 56 deletions pkg/cmd/info/util.go
View file
Open in desktop

This file was deleted.

13 changes: 7 additions & 6 deletions pkg/cmd/inspect/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,12 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"

"kitops/pkg/cmd/options"
"kitops/pkg/lib/constants"
"kitops/pkg/lib/repo"
"kitops/pkg/lib/repo/util"
"kitops/pkg/output"
"strings"

"github.com/spf13/cobra"
"oras.land/oras-go/v2/errdef"
Expand Down Expand Up @@ -80,7 +81,7 @@ func runCommand(opts *inspectOptions) func(*cobra.Command, []string) error {
inspectInfo, err := inspectReference(cmd.Context(), opts)
if err != nil {
if errors.Is(err, errdef.ErrNotFound) {
return output.Fatalf("Could not find modelkit %s", repo.FormatRepositoryForDisplay(opts.modelRef.String()))
return output.Fatalf("Could not find modelkit %s", util.FormatRepositoryForDisplay(opts.modelRef.String()))
}
return output.Fatalf("Error resolving modelkit: %s", err)
}
Expand All @@ -100,7 +101,7 @@ func (opts *inspectOptions) complete(ctx context.Context, args []string) error {
}
opts.configHome = configHome

ref, extraTags, err := repo.ParseReference(args[0])
ref, extraTags, err := util.ParseReference(args[0])
if err != nil {
return err
}
Expand All @@ -109,8 +110,8 @@ func (opts *inspectOptions) complete(ctx context.Context, args []string) error {
}
opts.modelRef = ref

if opts.modelRef.Registry == repo.DefaultRegistry && opts.checkRemote {
return fmt.Errorf("can not check remote: %s does not contain registry", repo.FormatRepositoryForDisplay(opts.modelRef.String()))
if opts.modelRef.Registry == util.DefaultRegistry && opts.checkRemote {
return fmt.Errorf("can not check remote: %s does not contain registry", util.FormatRepositoryForDisplay(opts.modelRef.String()))
}

return nil
Expand Down
13 changes: 8 additions & 5 deletions pkg/cmd/inspect/inspect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,12 @@ package inspect
import (
"context"
"fmt"

"kitops/pkg/artifact"
"kitops/pkg/lib/constants"
"kitops/pkg/lib/repo"
"kitops/pkg/lib/repo/local"
"kitops/pkg/lib/repo/remote"
"kitops/pkg/lib/repo/util"

"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
Expand All @@ -46,15 +49,15 @@ func inspectReference(ctx context.Context, opts *inspectOptions) (*inspectInfo,

func getLocalInspect(ctx context.Context, opts *inspectOptions) (*inspectInfo, error) {
storageRoot := constants.StoragePath(opts.configHome)
store, err := repo.NewLocalStore(storageRoot, opts.modelRef)
localRepo, err := local.NewLocalRepo(storageRoot, opts.modelRef)
if err != nil {
return nil, fmt.Errorf("failed to read local storage: %w", err)
}
return getInspectInfo(ctx, store, opts.modelRef.Reference)
return getInspectInfo(ctx, localRepo, opts.modelRef.Reference)
}

func getRemoteInspect(ctx context.Context, opts *inspectOptions) (*inspectInfo, error) {
repository, err := repo.NewRepository(ctx, opts.modelRef.Registry, opts.modelRef.Repository, &repo.RegistryOptions{
repository, err := remote.NewRepository(ctx, opts.modelRef.Registry, opts.modelRef.Repository, &remote.RegistryOptions{
PlainHTTP: opts.PlainHTTP,
SkipTLSVerify: !opts.TlsVerify,
CredentialsPath: constants.CredentialsPath(opts.configHome),
Expand All @@ -66,7 +69,7 @@ func getRemoteInspect(ctx context.Context, opts *inspectOptions) (*inspectInfo,
}

func getInspectInfo(ctx context.Context, repository oras.Target, ref string) (*inspectInfo, error) {
desc, manifest, kitfile, err := repo.ResolveManifestAndConfig(ctx, repository, ref)
desc, manifest, kitfile, err := util.ResolveManifestAndConfig(ctx, repository, ref)
if err != nil {
return nil, err
}
Expand Down
7 changes: 4 additions & 3 deletions pkg/cmd/list/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ import (
"context"
"fmt"
"io"
"text/tabwriter"

"kitops/pkg/cmd/options"
"kitops/pkg/lib/constants"
"kitops/pkg/lib/repo"
"kitops/pkg/lib/repo/util"
"kitops/pkg/output"
"text/tabwriter"

"github.com/spf13/cobra"
"oras.land/oras-go/v2/registry"
Expand Down Expand Up @@ -69,7 +70,7 @@ func (opts *listOptions) complete(ctx context.Context, args []string) error {
}
opts.configHome = configHome
if len(args) > 0 {
remoteRef, extraTags, err := repo.ParseReference(args[0])
remoteRef, extraTags, err := util.ParseReference(args[0])
if err != nil {
return fmt.Errorf("invalid reference: %w", err)
}
Expand Down
Loading