From 5f5f1bbc897f06bdd1302fa27051815b71cbdd2e Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 21 Nov 2022 15:58:57 +0000 Subject: [PATCH 01/11] Add Cargo registry. --- custom/conf/app.example.ini | 2 + docs/content/doc/packages/cargo.en-us.md | 109 +++++++ docs/content/doc/packages/overview.en-us.md | 1 + models/packages/descriptor.go | 3 + models/packages/package.go | 6 + models/packages/package_property.go | 6 + modules/packages/cargo/parser.go | 144 +++++++++ modules/packages/cargo/parser_test.go | 87 ++++++ modules/repository/create.go | 1 + modules/setting/packages.go | 2 + options/locale/locale_en-US.ini | 14 + public/img/svg/gitea-cargo.svg | 1 + routers/api/packages/api.go | 15 + routers/api/packages/cargo/cargo.go | 273 ++++++++++++++++ routers/api/v1/packages/package.go | 2 +- routers/web/org/setting_packages.go | 20 ++ routers/web/shared/packages/packages.go | 22 ++ routers/web/user/setting/packages.go | 18 ++ routers/web/web.go | 8 + services/forms/package_form.go | 2 +- services/packages/cargo/index.go | 283 +++++++++++++++++ services/packages/packages.go | 30 +- templates/admin/packages/list.tmpl | 1 + templates/org/settings/packages.tmpl | 1 + templates/package/content/cargo.tmpl | 62 ++++ templates/package/metadata/cargo.tmpl | 7 + templates/package/shared/cargo.tmpl | 24 ++ templates/package/shared/list.tmpl | 1 + templates/package/view.tmpl | 2 + templates/swagger/v1_json.tmpl | 1 + templates/user/settings/packages.tmpl | 1 + tests/integration/api_packages_cargo_test.go | 310 +++++++++++++++++++ web_src/svg/gitea-cargo.svg | 3 + 33 files changed, 1455 insertions(+), 7 deletions(-) create mode 100644 docs/content/doc/packages/cargo.en-us.md create mode 100644 modules/packages/cargo/parser.go create mode 100644 modules/packages/cargo/parser_test.go create mode 100644 public/img/svg/gitea-cargo.svg create mode 100644 routers/api/packages/cargo/cargo.go create mode 100644 services/packages/cargo/index.go create mode 100644 templates/package/content/cargo.tmpl create mode 100644 templates/package/metadata/cargo.tmpl create mode 100644 templates/package/shared/cargo.tmpl create mode 100644 tests/integration/api_packages_cargo_test.go create mode 100644 web_src/svg/gitea-cargo.svg diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 8e85394d34820..a280a3c2e8391 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2377,6 +2377,8 @@ ROUTER = console ;LIMIT_TOTAL_OWNER_COUNT = -1 ;; Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_TOTAL_OWNER_SIZE = -1 +;; Maxmimum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_CARGO = -1 ;; Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_COMPOSER = -1 ;; Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/packages/cargo.en-us.md b/docs/content/doc/packages/cargo.en-us.md new file mode 100644 index 0000000000000..1f90d939d1f7e --- /dev/null +++ b/docs/content/doc/packages/cargo.en-us.md @@ -0,0 +1,109 @@ +--- +date: "2022-11-20T00:00:00+00:00" +title: "Cargo Packages Repository" +slug: "packages/cargo" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "Cargo" + weight: 5 + identifier: "cargo" +--- + +# Cargo Packages Repository + +Publish [Cargo](https://doc.rust-lang.org/stable/cargo/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Cargo package registry, you need [Rust and Cargo](https://www.rust-lang.org/tools/install). + +Cargo stores informations about the available packages in a package index stored in a git repository. +This repository is needed to work with the registry. +The following section describes how to create it. + +## Index Repository + +Cargo stores informations about the available packages in a package index stored in a git repository. +In Gitea this repository has the special name `_cargo-index`. +After a package was uploaded, its metadata is automatically written to the index. +The content of this repository should not be manually modified. + +The user or organization package settings page allows to create the index repository along with the configuration file. +If needed this action will rewrite the configuration file. +This can be useful if for example the Gitea instance domain was changed. + +If the case arises where the packages stored in Gitea and the information in the index repository are out of sync, the settings page allows to rebuild the index repository. +This action iterates all packages in the registry and writes their information to the index. +If there are lot of packages this process may take some time. + +## Configuring the package registry + +To register the package registry the Cargo configuration must be updated. +Add the following text to the configuration file located in the current users home directory (for example `~/.cargo/config.toml`): + +``` +[registry] +default = "gitea" + +[registries.gitea] +index = "https://gitea.example.com/{owner}/_cargo-index.git" + +[net] +git-fetch-with-cli = true +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package. | + +If the registry is private or you want to publish new packages, you have to configure your credentials. +Add the credentials section to the credentials file located in the current users home directory (for example `~/.cargo/credentials.toml`): + +``` +[registries.gitea] +token = "Bearer {token}" +``` + +| Parameter | Description | +| --------- | ----------- | +| `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) | + +## Publish a package + +Publish a package by running the following command in your project: + +```shell +cargo publish +``` + +You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first. + +## Install a package + +To install a package from the package registry, execute the following command: + +```shell +cargo add {package_name} +``` + +| Parameter | Description | +| -------------- | ----------- | +| `package_name` | The package name. | + +## Supported commands + +``` +cargo publish +cargo add +cargo install +cargo yank +cargo unyank +cargo search +``` diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 0abb054b0f1ba..17fae496750a4 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -26,6 +26,7 @@ The following package managers are currently supported: | Name | Language | Package client | | ---- | -------- | -------------- | +| [Cargo]({{< relref "doc/packages/cargo.en-us.md" >}}) | Rust | `cargo` | | [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` | | [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` | | [Container]({{< relref "doc/packages/container.en-us.md" >}}) | - | any OCI compliant client | diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 357574a706c2d..d27cc85e2a928 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -12,6 +12,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/packages/cargo" "code.gitea.io/gitea/modules/packages/composer" "code.gitea.io/gitea/modules/packages/conan" "code.gitea.io/gitea/modules/packages/container" @@ -129,6 +130,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc var metadata interface{} switch p.Type { + case TypeCargo: + metadata = &cargo.Metadata{} case TypeComposer: metadata = &composer.Metadata{} case TypeConan: diff --git a/models/packages/package.go b/models/packages/package.go index cea04a0957955..a52f6d307f307 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -31,6 +31,7 @@ type Type string // List of supported packages const ( + TypeCargo Type = "cargo" TypeComposer Type = "composer" TypeConan Type = "conan" TypeContainer Type = "container" @@ -46,6 +47,7 @@ const ( ) var TypeList = []Type{ + TypeCargo, TypeComposer, TypeConan, TypeContainer, @@ -63,6 +65,8 @@ var TypeList = []Type{ // Name gets the name of the package type func (pt Type) Name() string { switch pt { + case TypeCargo: + return "Cargo" case TypeComposer: return "Composer" case TypeConan: @@ -94,6 +98,8 @@ func (pt Type) Name() string { // SVGName gets the name of the package type svg image func (pt Type) SVGName() string { switch pt { + case TypeCargo: + return "gitea-cargo" case TypeComposer: return "gitea-composer" case TypeConan: diff --git a/models/packages/package_property.go b/models/packages/package_property.go index fc10713801947..fff8eab1d3ba2 100644 --- a/models/packages/package_property.go +++ b/models/packages/package_property.go @@ -59,6 +59,12 @@ func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64, return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps) } +// UpdateProperty updates a property +func UpdateProperty(ctx context.Context, pp *PackageProperty) error { + _, err := db.GetEngine(ctx).ID(pp.ID).Update(pp) + return err +} + // DeleteAllProperties deletes all properties of a ref func DeleteAllProperties(ctx context.Context, refType PropertyType, refID int64) error { _, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ?", refType, refID).Delete(&PackageProperty{}) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go new file mode 100644 index 0000000000000..c241fd1c36ccc --- /dev/null +++ b/modules/packages/cargo/parser.go @@ -0,0 +1,144 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cargo + +import ( + "encoding/binary" + "errors" + "io" + "regexp" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" + + "github.com/hashicorp/go-version" +) + +const PropertyYanked = "cargo.yanked" + +var ( + ErrInvalidName = errors.New("package name is invalid") + ErrInvalidVersion = errors.New("package version is invalid") +) + +// Package represents a Cargo package +type Package struct { + Name string + Version string + Metadata *Metadata + Content io.Reader +} + +// Metadata represents the metadata of a Cargo package +type Metadata struct { + Dependencies []*Dependency `json:"dependencies,omitempty"` + Features map[string][]string `json:"features,omitempty"` + Authors []string `json:"authors,omitempty"` + Description string `json:"description,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Readme string `json:"readme,omitempty"` + Keywords []string `json:"keywords,omitempty"` + Categories []string `json:"categories,omitempty"` + License string `json:"license,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + Links string `json:"links,omitempty"` +} + +type Dependency struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target string `json:"target"` + Kind string `json:"kind"` + Registry string `json:"registry"` + ExplicitNameInToml string `json:"explicit_name_in_toml"` +} + +var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`) + +// ParsePackage reads the metadata and content of a package +func ParsePackage(r io.Reader) (*Package, error) { + var size uint32 + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p, err := parsePackage(io.LimitReader(r, int64(size))) + if err != nil { + return nil, err + } + + if err := binary.Read(r, binary.LittleEndian, &size); err != nil { + return nil, err + } + + p.Content = io.LimitReader(r, int64(size)) + + return p, nil +} + +func parsePackage(r io.Reader) (*Package, error) { + var meta struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []*Dependency `json:"deps"` + Features map[string][]string `json:"features"` + Authors []string `json:"authors"` + Description string `json:"description"` + Documentation string `json:"documentation"` + Homepage string `json:"homepage"` + Readme string `json:"readme"` + ReadmeFile string `json:"readme_file"` + Keywords []string `json:"keywords"` + Categories []string `json:"categories"` + License string `json:"license"` + LicenseFile string `json:"license_file"` + Repository string `json:"repository"` + Links string `json:"links"` + } + if err := json.NewDecoder(r).Decode(&meta); err != nil { + return nil, err + } + + if !nameMatch.MatchString(meta.Name) { + return nil, ErrInvalidName + } + + if _, err := version.NewSemver(meta.Vers); err != nil { + return nil, ErrInvalidVersion + } + + if !validation.IsValidURL(meta.Homepage) { + meta.Homepage = "" + } + if !validation.IsValidURL(meta.Documentation) { + meta.Documentation = "" + } + if !validation.IsValidURL(meta.Repository) { + meta.Repository = "" + } + + return &Package{ + Name: meta.Name, + Version: meta.Vers, + Metadata: &Metadata{ + Dependencies: meta.Deps, + Features: meta.Features, + Authors: meta.Authors, + Description: meta.Description, + DocumentationURL: meta.Documentation, + ProjectURL: meta.Homepage, + Readme: meta.Readme, + Keywords: meta.Keywords, + Categories: meta.Categories, + License: meta.License, + RepositoryURL: meta.Repository, + Links: meta.Links, + }, + }, nil +} diff --git a/modules/packages/cargo/parser_test.go b/modules/packages/cargo/parser_test.go new file mode 100644 index 0000000000000..f60f5a3fe0a7a --- /dev/null +++ b/modules/packages/cargo/parser_test.go @@ -0,0 +1,87 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cargo + +import ( + "bytes" + "encoding/binary" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +const ( + description = "Package Description" + author = "KN4CK3R" + homepage = "https://gitea.io/" + license = "MIT" +) + +func TestParsePackage(t *testing.T) { + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + description + `", + "authors": ["` + author + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0" + } + ], + "homepage":"` + homepage + `", + "license":"` + license + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "0test", "-test", "_test", strings.Repeat("a", 65)} { + data := createPackage(name, "1.0.0") + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.", "-1.0", "1.0.0/1"} { + data := createPackage("test", version) + + cp, err := ParsePackage(data) + assert.Nil(t, cp) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + data := createPackage("test", "1.0.0") + + cp, err := ParsePackage(data) + assert.NotNil(t, cp) + assert.NoError(t, err) + + assert.Equal(t, "test", cp.Name) + assert.Equal(t, "1.0.0", cp.Version) + assert.Equal(t, description, cp.Metadata.Description) + assert.Equal(t, []string{author}, cp.Metadata.Authors) + assert.Len(t, cp.Metadata.Dependencies, 1) + assert.Equal(t, "dep", cp.Metadata.Dependencies[0].Name) + assert.Equal(t, homepage, cp.Metadata.ProjectURL) + assert.Equal(t, license, cp.Metadata.License) + content, _ := io.ReadAll(cp.Content) + assert.Equal(t, "test", string(content)) + }) +} diff --git a/modules/repository/create.go b/modules/repository/create.go index c43f1e09898a3..2758fdf704653 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -207,6 +207,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, IsMirror: opts.IsMirror, + DefaultBranch: opts.DefaultBranch, } var rollbackRepo *repo_model.Repository diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 62201032c7403..647cd26e05401 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -26,6 +26,7 @@ var ( LimitTotalOwnerCount int64 LimitTotalOwnerSize int64 + LimitSizeCargo int64 LimitSizeComposer int64 LimitSizeConan int64 LimitSizeContainer int64 @@ -65,6 +66,7 @@ func newPackages() { } Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") + Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ce93e92d34550..f0f333114392a 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3118,6 +3118,11 @@ versions.on = on versions.view_all = View all dependency.id = ID dependency.version = Version +cargo.registry = Setup this registry in the Cargo configuration file (for example ~/.cargo/config.toml): +cargo.install = To install the package using Cargo, run the following command: +cargo.documentation = For more information on the Cargo registry, see the documentation. +cargo.details.repository_site = Repository Site +cargo.details.documentation_site = Documentation Site composer.registry = Setup this registry in your ~/.composer/config.json file: composer.install = To install the package using Composer, run the following command: composer.documentation = For more information on the Composer registry, see the documentation. @@ -3189,6 +3194,15 @@ settings.delete.description = Deleting a package is permanent and cannot be undo settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure? settings.delete.success = The package has been deleted. settings.delete.error = Failed to delete the package. +owner.settings.cargo.title = Cargo Registry Index +owner.settings.cargo.initialize = Initialize Index +owner.settings.cargo.initialize.description = To use the Cargo registry a special index git repository is needed. Here you can (re)create it with the required config. +owner.settings.cargo.initialize.error = Failed to initialize Cargo index: %v +owner.settings.cargo.initialize.success = The Cargo index was successfully created. +owner.settings.cargo.rebuild = Rebuild Index +owner.settings.cargo.rebuild.description = If the index is out of sync with the cargo packages stored you can rebuild it here. +owner.settings.cargo.rebuild.error = Failed to rebuild Cargo index: %v +owner.settings.cargo.rebuild.success = The Cargo index was successfully rebuild. owner.settings.cleanuprules.title = Manage Cleanup Rules owner.settings.cleanuprules.add = Add Cleanup Rule owner.settings.cleanuprules.edit = Edit Cleanup Rule diff --git a/public/img/svg/gitea-cargo.svg b/public/img/svg/gitea-cargo.svg new file mode 100644 index 0000000000000..91d53941cad9f --- /dev/null +++ b/public/img/svg/gitea-cargo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 11e7e5d6a67e3..374c12a5becc0 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/packages/cargo" "code.gitea.io/gitea/routers/api/packages/composer" "code.gitea.io/gitea/routers/api/packages/conan" "code.gitea.io/gitea/routers/api/packages/container" @@ -64,6 +65,20 @@ func CommonRoutes(ctx gocontext.Context) *web.Route { }) r.Group("/{username}", func() { + r.Group("/cargo", func() { + r.Group("/api/v1/crates", func() { + r.Get("", cargo.SearchPackages) + r.Put("/new", reqPackageAccess(perm.AccessModeWrite), cargo.UploadPackage) + r.Group("/{package}", func() { + r.Group("/{version}", func() { + r.Get("/download", cargo.DownloadPackageFile) + r.Delete("/yank", reqPackageAccess(perm.AccessModeWrite), cargo.YankPackage) + r.Put("/unyank", reqPackageAccess(perm.AccessModeWrite), cargo.UnyankPackage) + }) + r.Get("/owners", cargo.ListOwners) + }) + }) + }, reqPackageAccess(perm.AccessModeRead)) r.Group("/composer", func() { r.Get("/packages.json", composer.ServiceIndex) r.Get("/search.json", composer.SearchPackages) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go new file mode 100644 index 0000000000000..cab399c3367e2 --- /dev/null +++ b/routers/api/packages/cargo/cargo.go @@ -0,0 +1,273 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cargo + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + cargo_service "code.gitea.io/gitea/services/packages/cargo" +) + +// https://doc.rust-lang.org/cargo/reference/registries.html#web-api +type StatusResponse struct { + OK bool `json:"ok"` + Errors []StatusMessage `json:"errors,omitempty"` +} + +type StatusMessage struct { + Message string `json:"detail"` +} + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, StatusResponse{ + OK: false, + Errors: []StatusMessage{ + { + Message: message, + }, + }, + }) + }) +} + +type SearchResult struct { + Crates []*SearchResultCrate `json:"crates"` + Meta SearchResultMeta `json:"meta"` +} + +type SearchResultCrate struct { + Name string `json:"name"` + LatestVersion string `json:"max_version"` + Description string `json:"description"` +} + +type SearchResultMeta struct { + Total int64 `json:"total"` +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#search +func SearchPackages(ctx *context.Context) { + page := ctx.FormInt("page") + if page < 1 { + page = 1 + } + perPage := ctx.FormInt("per_page") + paginator := db.ListOptions{ + Page: page, + PageSize: convert.ToCorrectPageSize(perPage), + } + + pvs, total, err := packages_model.SearchLatestVersions( + ctx, + &packages_model.PackageSearchOptions{ + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeCargo, + Name: packages_model.SearchValue{Value: ctx.FormTrim("q")}, + IsInternal: util.OptionalBoolFalse, + Paginator: &paginator, + }, + ) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + crates := make([]*SearchResultCrate, 0, len(pvs)) + for _, pd := range pds { + crates = append(crates, &SearchResultCrate{ + Name: pd.Package.Name, + LatestVersion: pd.Version.Version, + Description: pd.Metadata.(*cargo_module.Metadata).Description, + }) + } + + ctx.JSON(http.StatusOK, SearchResult{ + Crates: crates, + Meta: SearchResultMeta{ + Total: total, + }, + }) +} + +type Owners struct { + Users []OwnerUser `json:"users"` +} + +type OwnerUser struct { + ID int64 `json:"id"` + Login string `json:"login"` + Name string `json:"name"` +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#owners-list +func ListOwners(ctx *context.Context) { + ctx.JSON(http.StatusOK, Owners{ + Users: []OwnerUser{ + { + ID: ctx.Package.Owner.ID, + Login: ctx.Package.Owner.Name, + Name: ctx.Package.Owner.DisplayName(), + }, + }, + }) +} + +// DownloadPackageFile serves the content of a package +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeCargo, + Name: ctx.Params("package"), + Version: ctx.Params("version"), + }, + &packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", ctx.Params("package"), ctx.Params("version"))), + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#publish +func UploadPackage(ctx *context.Context) { + defer ctx.Req.Body.Close() + + cp, err := cargo_module.ParsePackage(ctx.Req.Body) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + + buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + pv, _, err := packages_service.CreatePackageAndAddFile( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeCargo, + Name: cp.Name, + Version: cp.Version, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: cp.Metadata, + VersionProperties: map[string]string{ + cargo_module.PropertyYanked: strconv.FormatBool(false), + }, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", cp.Name, cp.Version)), + }, + Data: buf, + IsLead: true, + }, + ) + if err != nil { + switch err { + case packages_model.ErrDuplicatePackageVersion: + apiError(ctx, http.StatusConflict, err) + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + + if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil { + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + log.Error("Rollback creation of package version: %v", err) + } + + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, StatusResponse{OK: true}) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#yank +func YankPackage(ctx *context.Context) { + yankPackage(ctx, true) +} + +// https://doc.rust-lang.org/cargo/reference/registries.html#unyank +func UnyankPackage(ctx *context.Context) { + yankPackage(ctx, false) +} + +func yankPackage(ctx *context.Context, yank bool) { + pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"), ctx.Params("version")) + if err != nil { + if err == packages_model.ErrPackageNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, cargo_module.PropertyYanked) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pps) == 0 { + apiError(ctx, http.StatusInternalServerError, "Property not found") + return + } + + pp := pps[0] + pp.Value = strconv.FormatBool(yank) + + if err := packages_model.UpdateProperty(ctx, pp); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.JSON(http.StatusOK, StatusResponse{OK: true}) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 29fa6c85a5b6f..4fccf7dde72c9 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -41,7 +41,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, conan, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] + // enum: [cargo, composer, conan, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/web/org/setting_packages.go b/routers/web/org/setting_packages.go index c7edf4a18531c..8da535053550e 100644 --- a/routers/web/org/setting_packages.go +++ b/routers/web/org/setting_packages.go @@ -85,3 +85,23 @@ func PackagesRulePreview(ctx *context.Context) { ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) } + +func InitializeCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.InitializeCargoIndex(ctx, ctx.ContextUser) + + ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name)) +} + +func RebuildCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsOrgSettings"] = true + ctx.Data["PageIsSettingsPackages"] = true + + shared.RebuildCargoIndex(ctx, ctx.ContextUser) + + ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name)) +} diff --git a/routers/web/shared/packages/packages.go b/routers/web/shared/packages/packages.go index 5e934d707ee41..99e86d048030c 100644 --- a/routers/web/shared/packages/packages.go +++ b/routers/web/shared/packages/packages.go @@ -14,9 +14,11 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/services/forms" + cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" ) @@ -224,3 +226,23 @@ func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *pack return nil } + +func InitializeCargoIndex(ctx *context.Context, owner *user_model.User) { + err := cargo_service.InitializeIndexRepository(ctx, owner, owner) + if err != nil { + log.Error("InitializeIndexRepository failed: %v", err) + ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.initialize.error", err)) + } else { + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.initialize.success")) + } +} + +func RebuildCargoIndex(ctx *context.Context, owner *user_model.User) { + err := cargo_service.RebuildIndex(ctx, owner, owner) + if err != nil { + log.Error("RebuildIndex failed: %v", err) + ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.rebuild.error", err)) + } else { + ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.rebuild.success")) + } +} diff --git a/routers/web/user/setting/packages.go b/routers/web/user/setting/packages.go index d44e904556d2b..5c7ea8513a34a 100644 --- a/routers/web/user/setting/packages.go +++ b/routers/web/user/setting/packages.go @@ -78,3 +78,21 @@ func PackagesRulePreview(ctx *context.Context) { ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview) } + +func InitializeCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.InitializeCargoIndex(ctx, ctx.Doer) + + ctx.Redirect(setting.AppSubURL + "/user/settings/packages") +} + +func RebuildCargoIndex(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("packages.title") + ctx.Data["PageIsSettingsPackages"] = true + + shared.RebuildCargoIndex(ctx, ctx.Doer) + + ctx.Redirect(setting.AppSubURL + "/user/settings/packages") +} diff --git a/routers/web/web.go b/routers/web/web.go index 142f2384eb3c4..864a45407e507 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -463,6 +463,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", user_setting.PackagesRulePreview) }) }) + m.Group("/cargo", func() { + m.Post("/initialize", user_setting.InitializeCargoIndex) + m.Post("/rebuild", user_setting.RebuildCargoIndex) + }) }, packagesEnabled) m.Get("/organization", user_setting.Organization) m.Get("/repos", user_setting.Repos) @@ -784,6 +788,10 @@ func RegisterRoutes(m *web.Route) { m.Get("/preview", org.PackagesRulePreview) }) }) + m.Group("/cargo", func() { + m.Post("/initialize", org.InitializeCargoIndex) + m.Post("/rebuild", org.RebuildCargoIndex) + }) }, packagesEnabled) }, func(ctx *context.Context) { ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable diff --git a/services/forms/package_form.go b/services/forms/package_form.go index 6c3ff52a9c01b..b0ebb7cb08dca 100644 --- a/services/forms/package_form.go +++ b/services/forms/package_form.go @@ -16,7 +16,7 @@ import ( type PackageCleanupRuleForm struct { ID int64 Enabled bool - Type string `binding:"Required;In(composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` + Type string `binding:"Required;In(cargo,composer,conan,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"` KeepCount int `binding:"In(0,1,5,10,25,50,100)"` KeepPattern string `binding:"RegexPattern"` RemoveDays int `binding:"In(0,7,14,30,60,90,180)"` diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go new file mode 100644 index 0000000000000..b6f4136ea3797 --- /dev/null +++ b/services/packages/cargo/index.go @@ -0,0 +1,283 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cargo + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "path" + "strconv" + "time" + + packages_model "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + repo_module "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + files_service "code.gitea.io/gitea/services/repository/files" +) + +const ( + IndexRepositoryName = "_cargo-index" + ConfigFileName = "config.json" +) + +// https://doc.rust-lang.org/cargo/reference/registries.html#index-format + +func BuildPackagePath(name string) string { + switch len(name) { + case 0: + panic("Cargo package name can not be empty") + case 1: + return path.Join("1", name) + case 2: + return path.Join("2", name) + case 3: + return path.Join("3", string(name[0]), name) + default: + return path.Join(name[0:2], name[2:4], name) + } +} + +func InitializeIndexRepository(ctx context.Context, doer, owner *user_model.User) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + if err := createOrUpdateConfigFile(ctx, repo, doer, owner); err != nil { + return fmt.Errorf("createOrUpdateConfigFile: %w", err) + } + + return nil +} + +func RebuildIndex(ctx context.Context, doer, owner *user_model.User) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeCargo) + if err != nil { + return fmt.Errorf("GetPackagesByType: %w", err) + } + + return alterRepositoryContent( + ctx, + doer, + repo, + "Rebuild Cargo Index", + func(t *files_service.TemporaryUploadRepository) error { + // Remove all existing content but the Cargo config + files, err := t.LsFiles() + if err != nil { + return err + } + for i, file := range files { + if file == ConfigFileName { + files[i] = files[len(files)-1] + files = files[:len(files)-1] + break + } + } + if err := t.RemoveFilesFromIndex(files...); err != nil { + return err + } + + // Add all packages + for _, p := range ps { + if err := addOrUpdatePackageIndex(ctx, t, p); err != nil { + return err + } + } + + return nil + }, + ) +} + +func AddOrUpdatePackageIndex(ctx context.Context, doer, owner *user_model.User, packageID int64) error { + repo, err := getOrCreateIndexRepository(ctx, doer, owner) + if err != nil { + return err + } + + p, err := packages_model.GetPackageByID(ctx, packageID) + if err != nil { + return fmt.Errorf("GetPackageByID[%d]: %w", packageID, err) + } + + return alterRepositoryContent( + ctx, + doer, + repo, + "Update "+p.Name, + func(t *files_service.TemporaryUploadRepository) error { + return addOrUpdatePackageIndex(ctx, t, p) + }, + ) +} + +type IndexVersionEntry struct { + Name string `json:"name"` + Version string `json:"vers"` + Dependencies []*cargo_module.Dependency `json:"deps,omitempty"` + FileChecksum string `json:"cksum"` + Features map[string][]string `json:"features,omitempty"` + Yanked bool `json:"yanked"` + Links string `json:"links,omitempty"` +} + +func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + }) + if err != nil { + return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) + } + if len(pvs) == 0 { + return nil + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err) + } + + var b bytes.Buffer + for _, pd := range pds { + metadata := pd.Metadata.(*cargo_module.Metadata) + yanked, _ := strconv.ParseBool(pd.VersionProperties.GetByName(cargo_module.PropertyYanked)) + entry, err := json.Marshal(&IndexVersionEntry{ + Name: pd.Package.Name, + Version: pd.Version.Version, + Dependencies: metadata.Dependencies, + FileChecksum: pd.Files[0].Blob.HashSHA256, + Features: metadata.Features, + Yanked: yanked, + Links: metadata.Links, + }) + if err != nil { + return err + } + + b.Write(entry) + b.WriteString("\n") + } + + return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b) +} + +func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { + repo, err := repo_model.GetRepositoryByOwnerAndNameCtx(ctx, owner.Name, IndexRepositoryName) + if err != nil { + if errors.Is(err, util.ErrNotExist) { + repo, err = repo_module.CreateRepository(doer, owner, repo_module.CreateRepoOptions{ + Name: IndexRepositoryName, + }) + if err != nil { + return nil, fmt.Errorf("CreateRepository: %w", err) + } + } else { + return nil, fmt.Errorf("GetRepositoryByOwnerAndNameCtx: %w", err) + } + } + + return repo, nil +} + +type CargoConfig struct { + DownloadURL string `json:"dl"` + APIURL string `json:"api"` +} + +func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error { + return alterRepositoryContent( + ctx, + doer, + repo, + "Initialize Cargo Config", + func(t *files_service.TemporaryUploadRepository) error { + var b bytes.Buffer + err := json.NewEncoder(&b).Encode(CargoConfig{ + DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", + APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", + }) + if err != nil { + return err + } + + return writeObjectToIndex(t, ConfigFileName, &b) + }, + ) +} + +// This is a shorter version of CreateOrUpdateRepoFile which allows to perform multiple actions on a git repository +func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, commitMessage string, fn func(*files_service.TemporaryUploadRepository) error) error { + t, err := files_service.NewTemporaryUploadRepository(ctx, repo) + if err != nil { + return err + } + defer t.Close() + + var lastCommitID string + if err := t.Clone(repo.DefaultBranch); err != nil { + if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { + return err + } + if err := t.Init(); err != nil { + return err + } + } else { + if err := t.SetDefaultIndex(); err != nil { + return err + } + + commit, err := t.GetBranchCommit(repo.DefaultBranch) + if err != nil { + return err + } + + lastCommitID = commit.ID.String() + } + + if err := fn(t); err != nil { + return err + } + + treeHash, err := t.WriteTree() + if err != nil { + return err + } + + now := time.Now() + commitHash, err := t.CommitTreeWithDate(lastCommitID, doer, doer, treeHash, commitMessage, false, now, now) + if err != nil { + return err + } + + if err := t.Push(doer, commitHash, repo.DefaultBranch); err != nil { + return err + } + + return nil +} + +func writeObjectToIndex(t *files_service.TemporaryUploadRepository, path string, r io.Reader) error { + hash, err := t.HashObject(r) + if err != nil { + return err + } + + return t.AddObjectToIndex("100644", hash, path) +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 7343ffc530d55..540a1a12c10c0 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -22,6 +22,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + cargo_service "code.gitea.io/gitea/services/packages/cargo" container_service "code.gitea.io/gitea/services/packages/container" ) @@ -331,6 +332,8 @@ func checkSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p var typeSpecificSize int64 switch packageType { + case packages_model.TypeCargo: + typeSpecificSize = setting.Packages.LimitSizeCargo case packages_model.TypeComposer: typeSpecificSize = setting.Packages.LimitSizeComposer case packages_model.TypeConan: @@ -478,12 +481,15 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { if err != nil { return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) } + versionDeleted := false for _, pv := range pvs { - if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) - } else if skip { - log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) - continue + if pcr.Type == packages_model.TypeContainer { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } } toMatch := pv.LowerVersion @@ -509,6 +515,20 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) } + + versionDeleted = true + } + + if versionDeleted { + if pcr.Type == packages_model.TypeCargo { + owner, err := user_model.GetUserByIDCtx(ctx, pcr.OwnerID) + if err != nil { + return fmt.Errorf("GetUserByID failed: %w", err) + } + if err := cargo_service.AddOrUpdatePackageIndex(ctx, owner, owner, p.ID); err != nil { + return fmt.Errorf("CleanupRule [%d]: cargo.AddOrUpdatePackageIndex failed: %w", pcr.ID, err) + } + } } } return nil diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 3aab2873c6b7f..be46920f3a68e 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -13,6 +13,7 @@ + diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index a5b2a2ef68a39..cf9bdc1f1118e 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -19,6 +19,7 @@
+ {{template "package/content/cargo" .}} {{template "package/content/composer" .}} {{template "package/content/conan" .}} {{template "package/content/container" .}} @@ -42,6 +43,7 @@ {{end}}
{{svg "octicon-calendar" 16 "mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}
{{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}
+ {{template "package/metadata/cargo" .}} {{template "package/metadata/composer" .}} {{template "package/metadata/conan" .}} {{template "package/metadata/container" .}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index ddafc146a159c..413232f153a69 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1952,6 +1952,7 @@ }, { "enum": [ + "cargo", "composer", "conan", "container", diff --git a/templates/user/settings/packages.tmpl b/templates/user/settings/packages.tmpl index 2612313454e88..866feba3960be 100644 --- a/templates/user/settings/packages.tmpl +++ b/templates/user/settings/packages.tmpl @@ -4,6 +4,7 @@
{{template "base/alert" .}} {{template "package/shared/cleanup_rules/list" .}} + {{template "package/shared/cargo" .}}
{{template "base/footer" .}} diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go new file mode 100644 index 0000000000000..64fca4a2f479d --- /dev/null +++ b/tests/integration/api_packages_cargo_test.go @@ -0,0 +1,310 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "net/http" + neturl "net/url" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/json" + cargo_module "code.gitea.io/gitea/modules/packages/cargo" + "code.gitea.io/gitea/modules/setting" + cargo_router "code.gitea.io/gitea/routers/api/packages/cargo" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" +) + +func TestPackageCargo(t *testing.T) { + onGiteaRun(t, testPackageCargo) +} + +func testPackageCargo(t *testing.T, _ *neturl.URL) { + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + packageName := "cargo-package" + packageVersion := "1.0.3" + packageDescription := "Package Description" + packageAuthor := "KN4CK3R" + packageHomepage := "https://gitea.io/" + packageLicense := "MIT" + + createPackage := func(name, version string) io.Reader { + metadata := `{ + "name":"` + name + `", + "vers":"` + version + `", + "description":"` + packageDescription + `", + "authors": ["` + packageAuthor + `"], + "deps":[ + { + "name":"dep", + "version_req":"1.0" + } + ], + "homepage":"` + packageHomepage + `", + "license":"` + packageLicense + `" +}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("test") + return &buf + } + + err := cargo_service.InitializeIndexRepository(db.DefaultContext, user, user) + assert.NoError(t, err) + + repo, err := repo_model.GetRepositoryByOwnerAndNameCtx(db.DefaultContext, user.Name, cargo_service.IndexRepositoryName) + assert.NotNil(t, repo) + assert.NoError(t, err) + + readGitContent := func(t *testing.T, path string) string { + gitRepo, err := git.OpenRepository(db.DefaultContext, repo.RepoPath()) + assert.NoError(t, err) + defer gitRepo.Close() + + commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + assert.NoError(t, err) + + blob, err := commit.GetBlobByPath(path) + assert.NoError(t, err) + + content, err := blob.GetBlobContent() + assert.NoError(t, err) + + return content + } + + root := fmt.Sprintf("%sapi/packages/%s/cargo", setting.AppURL, user.Name) + url := fmt.Sprintf("%s/api/v1/crates", root) + + t.Run("Index", func(t *testing.T) { + t.Run("Config", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.ConfigFileName) + + var config cargo_service.CargoConfig + err := json.Unmarshal([]byte(content), &config) + assert.NoError(t, err) + + assert.Equal(t, url, config.DownloadURL) + assert.Equal(t, root, config.APIURL) + }) + }) + + t.Run("Upload", func(t *testing.T) { + t.Run("InvalidNameOrVersion", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := createPackage("0test", "1.0.0") + + req := NewRequestWithBody(t, "PUT", url+"/new", content) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusBadRequest) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.False(t, status.OK) + + content = createPackage("test", "-1.0.0") + + req = NewRequestWithBody(t, "PUT", url+"/new", content) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusBadRequest) + + DecodeJSON(t, resp, &status) + assert.False(t, status.OK) + }) + + t.Run("Valid", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeCargo) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &cargo_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, fmt.Sprintf("%s-%s.crate", packageName, packageVersion), pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.EqualValues(t, 4, pb.Size) + + req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion)) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusConflict) + + t.Run("Index", func(t *testing.T) { + t.Run("Entry", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.Equal(t, packageName, entry.Name) + assert.Equal(t, packageVersion, entry.Version) + assert.Equal(t, pb.HashSHA256, entry.FileChecksum) + assert.False(t, entry.Yanked) + }) + + t.Run("Rebuild", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + err := cargo_service.RebuildIndex(db.DefaultContext, user, user) + assert.NoError(t, err) + + _ = readGitContent(t, cargo_service.BuildPackagePath(packageName)) + }) + }) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion) + assert.NoError(t, err) + assert.EqualValues(t, 0, pv.DownloadCount) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/download", url, neturl.PathEscape(packageName), neturl.PathEscape(pv.Version))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "test", resp.Body.String()) + + pv, err = packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion) + assert.NoError(t, err) + assert.EqualValues(t, 1, pv.DownloadCount) + }) + + t.Run("Search", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + cases := []struct { + Query string + Page int + PerPage int + ExpectedTotal int64 + ExpectedResults int + }{ + {"", 0, 0, 1, 1}, + {"", 1, 10, 1, 1}, + {"cargo", 1, 0, 1, 1}, + {"cargo", 1, 10, 1, 1}, + {"cargo", 2, 10, 1, 0}, + {"test", 0, 10, 0, 0}, + } + + for i, c := range cases { + req := NewRequest(t, "GET", fmt.Sprintf("%s?q=%s&page=%d&per_page=%d", url, c.Query, c.Page, c.PerPage)) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var result cargo_router.SearchResult + DecodeJSON(t, resp, &result) + + assert.Equal(t, c.ExpectedTotal, result.Meta.Total, "case %d: unexpected total hits", i) + assert.Len(t, result.Crates, c.ExpectedResults, "case %d: unexpected result count", i) + } + }) + + t.Run("Yank", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/yank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.True(t, entry.Yanked) + }) + + t.Run("Unyank", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "PUT", fmt.Sprintf("%s/%s/%s/unyank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion))) + req = AddBasicAuthHeader(req, user.Name) + resp := MakeRequest(t, req, http.StatusOK) + + var status cargo_router.StatusResponse + DecodeJSON(t, resp, &status) + assert.True(t, status.OK) + + content := readGitContent(t, cargo_service.BuildPackagePath(packageName)) + + var entry cargo_service.IndexVersionEntry + err := json.Unmarshal([]byte(content), &entry) + assert.NoError(t, err) + + assert.False(t, entry.Yanked) + }) + + t.Run("ListOwners", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/owners", url, neturl.PathEscape(packageName))) + resp := MakeRequest(t, req, http.StatusOK) + + var owners cargo_router.Owners + DecodeJSON(t, resp, &owners) + + assert.Len(t, owners.Users, 1) + assert.Equal(t, user.ID, owners.Users[0].ID) + assert.Equal(t, user.Name, owners.Users[0].Login) + assert.Equal(t, user.DisplayName(), owners.Users[0].Name) + }) +} diff --git a/web_src/svg/gitea-cargo.svg b/web_src/svg/gitea-cargo.svg new file mode 100644 index 0000000000000..dbec107ad0dcf --- /dev/null +++ b/web_src/svg/gitea-cargo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file From 9da9d58195a1c23e9dbc2a51443f94f6be940e22 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 21 Nov 2022 16:00:55 +0000 Subject: [PATCH 02/11] Add setting entry. --- docs/content/doc/advanced/config-cheat-sheet.en-us.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 4e7ef492f90b2..1f9533f0c03c6 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -1180,6 +1180,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload` - `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maxmimum count of package versions a single owner can have (`-1` means no limits) - `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maxmimum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_CARGO`: **-1**: Maxmimum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_COMPOSER`: **-1**: Maxmimum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONAN`: **-1**: Maxmimum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_CONTAINER`: **-1**: Maxmimum size of a Container upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) From ef668a7ac3b24722f4f623d5cae89883b6cfbb20 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 22 Nov 2022 09:01:33 +0000 Subject: [PATCH 03/11] Resolve import cycle. --- services/cron/tasks_basic.go | 4 +- services/packages/cargo/index.go | 10 +- services/packages/cleanup/cleanup.go | 155 +++++++++++++++++++ services/packages/packages.go | 137 ---------------- tests/integration/api_packages_cargo_test.go | 2 +- tests/integration/api_packages_test.go | 5 +- 6 files changed, 164 insertions(+), 149 deletions(-) create mode 100644 services/packages/cleanup/cleanup.go diff --git a/services/cron/tasks_basic.go b/services/cron/tasks_basic.go index ca35f5be57d05..578e55e257f01 100644 --- a/services/cron/tasks_basic.go +++ b/services/cron/tasks_basic.go @@ -17,7 +17,7 @@ import ( "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/migrations" mirror_service "code.gitea.io/gitea/services/mirror" - packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" repo_service "code.gitea.io/gitea/services/repository" archiver_service "code.gitea.io/gitea/services/repository/archiver" ) @@ -157,7 +157,7 @@ func registerCleanupPackages() { OlderThan: 24 * time.Hour, }, func(ctx context.Context, _ *user_model.User, config Config) error { realConfig := config.(*OlderThanConfig) - return packages_service.Cleanup(ctx, realConfig.OlderThan) + return packages_cleanup_service.Cleanup(ctx, realConfig.OlderThan) }) } diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index b6f4136ea3797..814cded26438b 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -196,7 +196,7 @@ func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.Use return repo, nil } -type CargoConfig struct { +type Config struct { DownloadURL string `json:"dl"` APIURL string `json:"api"` } @@ -209,7 +209,7 @@ func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, "Initialize Cargo Config", func(t *files_service.TemporaryUploadRepository) error { var b bytes.Buffer - err := json.NewEncoder(&b).Encode(CargoConfig{ + err := json.NewEncoder(&b).Encode(Config{ DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates", APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo", }) @@ -266,11 +266,7 @@ func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *re return err } - if err := t.Push(doer, commitHash, repo.DefaultBranch); err != nil { - return err - } - - return nil + return t.Push(doer, commitHash, repo.DefaultBranch) } func writeObjectToIndex(t *files_service.TemporaryUploadRepository, path string, r io.Reader) error { diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go new file mode 100644 index 0000000000000..76b51fb756914 --- /dev/null +++ b/services/packages/cleanup/cleanup.go @@ -0,0 +1,155 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package container + +import ( + "context" + "fmt" + "time" + + "code.gitea.io/gitea/models/db" + packages_model "code.gitea.io/gitea/models/packages" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/util" + packages_service "code.gitea.io/gitea/services/packages" + cargo_service "code.gitea.io/gitea/services/packages/cargo" + container_service "code.gitea.io/gitea/services/packages/container" +) + +// Cleanup removes expired package data +func Cleanup(taskCtx context.Context, olderThan time.Duration) error { + ctx, committer, err := db.TxContext(taskCtx) + if err != nil { + return err + } + defer committer.Close() + + err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { + select { + case <-taskCtx.Done(): + return db.ErrCancelledf("While processing package cleanup rules") + default: + } + + if err := pcr.CompiledPattern(); err != nil { + return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) + } + + olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) + + packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) + } + + for _, p := range packages { + pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ + PackageID: p.ID, + IsInternal: util.OptionalBoolFalse, + Sort: packages_model.SortCreatedDesc, + Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), + }) + if err != nil { + return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) + } + versionDeleted := false + for _, pv := range pvs { + if pcr.Type == packages_model.TypeContainer { + if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) + } else if skip { + log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) + continue + } + } + + toMatch := pv.LowerVersion + if pcr.MatchFullName { + toMatch = p.LowerName + "/" + pv.LowerVersion + } + + if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) + continue + } + if pv.CreatedUnix.AsLocalTime().After(olderThan) { + log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) + continue + } + if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { + log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) + continue + } + + log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) + + if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil { + return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) + } + + versionDeleted = true + } + + if versionDeleted { + if pcr.Type == packages_model.TypeCargo { + owner, err := user_model.GetUserByIDCtx(ctx, pcr.OwnerID) + if err != nil { + return fmt.Errorf("GetUserByID failed: %w", err) + } + if err := cargo_service.AddOrUpdatePackageIndex(ctx, owner, owner, p.ID); err != nil { + return fmt.Errorf("CleanupRule [%d]: cargo.AddOrUpdatePackageIndex failed: %w", pcr.ID, err) + } + } + } + } + return nil + }) + if err != nil { + return err + } + + if err := container_service.Cleanup(ctx, olderThan); err != nil { + return err + } + + ps, err := packages_model.FindUnreferencedPackages(ctx) + if err != nil { + return err + } + for _, p := range ps { + if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil { + return err + } + if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil { + return err + } + } + + pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) + if err != nil { + return err + } + + for _, pb := range pbs { + if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { + return err + } + } + + if err := committer.Commit(); err != nil { + return err + } + + contentStore := packages_module.NewContentStore() + for _, pb := range pbs { + if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { + log.Error("Error deleting package blob [%v]: %v", pb.ID, err) + } + } + + return nil +} diff --git a/services/packages/packages.go b/services/packages/packages.go index 540a1a12c10c0..81cdc8fe82b43 100644 --- a/services/packages/packages.go +++ b/services/packages/packages.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "strings" - "time" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -22,8 +21,6 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - cargo_service "code.gitea.io/gitea/services/packages/cargo" - container_service "code.gitea.io/gitea/services/packages/container" ) var ( @@ -445,140 +442,6 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro return packages_model.DeleteFileByID(ctx, pf.ID) } -// Cleanup removes expired package data -func Cleanup(taskCtx context.Context, olderThan time.Duration) error { - ctx, committer, err := db.TxContext(taskCtx) - if err != nil { - return err - } - defer committer.Close() - - err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error { - select { - case <-taskCtx.Done(): - return db.ErrCancelledf("While processing package cleanup rules") - default: - } - - if err := pcr.CompiledPattern(); err != nil { - return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err) - } - - olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays) - - packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err) - } - - for _, p := range packages { - pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ - PackageID: p.ID, - IsInternal: util.OptionalBoolFalse, - Sort: packages_model.SortCreatedDesc, - Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200), - }) - if err != nil { - return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err) - } - versionDeleted := false - for _, pv := range pvs { - if pcr.Type == packages_model.TypeContainer { - if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err) - } else if skip { - log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version) - continue - } - } - - toMatch := pv.LowerVersion - if pcr.MatchFullName { - toMatch = p.LowerName + "/" + pv.LowerVersion - } - - if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version) - continue - } - if pv.CreatedUnix.AsLocalTime().After(olderThan) { - log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version) - continue - } - if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) { - log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version) - continue - } - - log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version) - - if err := DeletePackageVersionAndReferences(ctx, pv); err != nil { - return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err) - } - - versionDeleted = true - } - - if versionDeleted { - if pcr.Type == packages_model.TypeCargo { - owner, err := user_model.GetUserByIDCtx(ctx, pcr.OwnerID) - if err != nil { - return fmt.Errorf("GetUserByID failed: %w", err) - } - if err := cargo_service.AddOrUpdatePackageIndex(ctx, owner, owner, p.ID); err != nil { - return fmt.Errorf("CleanupRule [%d]: cargo.AddOrUpdatePackageIndex failed: %w", pcr.ID, err) - } - } - } - } - return nil - }) - if err != nil { - return err - } - - if err := container_service.Cleanup(ctx, olderThan); err != nil { - return err - } - - ps, err := packages_model.FindUnreferencedPackages(ctx) - if err != nil { - return err - } - for _, p := range ps { - if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil { - return err - } - if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil { - return err - } - } - - pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan) - if err != nil { - return err - } - - for _, pb := range pbs { - if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil { - return err - } - } - - if err := committer.Commit(); err != nil { - return err - } - - contentStore := packages_module.NewContentStore() - for _, pb := range pbs { - if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil { - log.Error("Error deleting package blob [%v]: %v", pb.ID, err) - } - } - - return nil -} - // GetFileStreamByPackageNameAndVersion returns the content of the specific package file func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *packages_model.PackageFile, error) { log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey) diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index 64fca4a2f479d..507d505f717ea 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -100,7 +100,7 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { content := readGitContent(t, cargo_service.ConfigFileName) - var config cargo_service.CargoConfig + var config cargo_service.Config err := json.Unmarshal([]byte(content), &config) assert.NoError(t, err) diff --git a/tests/integration/api_packages_test.go b/tests/integration/api_packages_test.go index 8efb70848bf6b..755c5bb5085a3 100644 --- a/tests/integration/api_packages_test.go +++ b/tests/integration/api_packages_test.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" packages_service "code.gitea.io/gitea/services/packages" + packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" @@ -215,7 +216,7 @@ func TestPackageCleanup(t *testing.T) { _, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion) assert.NoError(t, err) - err = packages_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration) @@ -352,7 +353,7 @@ func TestPackageCleanup(t *testing.T) { pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule) assert.NoError(t, err) - err = packages_service.Cleanup(db.DefaultContext, duration) + err = packages_cleanup_service.Cleanup(db.DefaultContext, duration) assert.NoError(t, err) for _, v := range c.Versions { From f216281beb935a11e5f7f077a2e8b773293a1548 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 22 Nov 2022 09:53:45 +0000 Subject: [PATCH 04/11] Fix test. --- routers/api/packages/cargo/cargo.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index cab399c3367e2..93e826c555cef 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -197,8 +197,9 @@ func UploadPackage(ctx *context.Context) { PackageFileInfo: packages_service.PackageFileInfo{ Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", cp.Name, cp.Version)), }, - Data: buf, - IsLead: true, + Creator: ctx.Doer, + Data: buf, + IsLead: true, }, ) if err != nil { From 16a312842e49b171d0ad5fcea9760b4937a9794a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 8 Dec 2022 18:02:58 +0000 Subject: [PATCH 05/11] Adapt to merged methods. --- routers/api/packages/cargo/cargo.go | 5 ++++- services/packages/cargo/index.go | 4 ++-- services/packages/cleanup/cleanup.go | 2 +- tests/integration/api_packages_cargo_test.go | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index 93e826c555cef..055d7b94e4415 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -158,7 +158,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // https://doc.rust-lang.org/cargo/reference/registries.html#publish diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index 814cded26438b..e4dff1eba14d5 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -179,7 +179,7 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo } func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) { - repo, err := repo_model.GetRepositoryByOwnerAndNameCtx(ctx, owner.Name, IndexRepositoryName) + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner.Name, IndexRepositoryName) if err != nil { if errors.Is(err, util.ErrNotExist) { repo, err = repo_module.CreateRepository(doer, owner, repo_module.CreateRepoOptions{ @@ -189,7 +189,7 @@ func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.Use return nil, fmt.Errorf("CreateRepository: %w", err) } } else { - return nil, fmt.Errorf("GetRepositoryByOwnerAndNameCtx: %w", err) + return nil, fmt.Errorf("GetRepositoryByOwnerAndName: %w", err) } } diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 76b51fb756914..782994ee805c8 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -96,7 +96,7 @@ func Cleanup(taskCtx context.Context, olderThan time.Duration) error { if versionDeleted { if pcr.Type == packages_model.TypeCargo { - owner, err := user_model.GetUserByIDCtx(ctx, pcr.OwnerID) + owner, err := user_model.GetUserByID(ctx, pcr.OwnerID) if err != nil { return fmt.Errorf("GetUserByID failed: %w", err) } diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index 507d505f717ea..3381d217da7b8 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -70,7 +70,7 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { err := cargo_service.InitializeIndexRepository(db.DefaultContext, user, user) assert.NoError(t, err) - repo, err := repo_model.GetRepositoryByOwnerAndNameCtx(db.DefaultContext, user.Name, cargo_service.IndexRepositoryName) + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, user.Name, cargo_service.IndexRepositoryName) assert.NotNil(t, repo) assert.NoError(t, err) From 6b46a485c642b823f9a4a0a6ecb1637fe2a25515 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 9 Dec 2022 08:23:59 +0000 Subject: [PATCH 06/11] lint --- modules/packages/cargo/parser.go | 3 +-- modules/packages/cargo/parser_test.go | 3 +-- routers/api/packages/cargo/cargo.go | 3 +-- services/packages/cargo/index.go | 3 +-- services/packages/cleanup/cleanup.go | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go index c241fd1c36ccc..201279a6e5bad 100644 --- a/modules/packages/cargo/parser.go +++ b/modules/packages/cargo/parser.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package cargo diff --git a/modules/packages/cargo/parser_test.go b/modules/packages/cargo/parser_test.go index f60f5a3fe0a7a..2230a5b4999c9 100644 --- a/modules/packages/cargo/parser_test.go +++ b/modules/packages/cargo/parser_test.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package cargo diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index 055d7b94e4415..ee7eff79d2d41 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package cargo diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index e4dff1eba14d5..76f6b5a5de4cd 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package cargo diff --git a/services/packages/cleanup/cleanup.go b/services/packages/cleanup/cleanup.go index 782994ee805c8..2d62a028a4c6d 100644 --- a/services/packages/cleanup/cleanup.go +++ b/services/packages/cleanup/cleanup.go @@ -1,6 +1,5 @@ // Copyright 2022 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT package container From f9326c6d04981449db8672bd3920b3ecb6546032 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 15 Jan 2023 18:54:46 +0000 Subject: [PATCH 07/11] Check content size. --- modules/packages/cargo/parser.go | 10 ++++++---- routers/api/packages/cargo/cargo.go | 5 +++++ tests/integration/api_packages_cargo_test.go | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go index 201279a6e5bad..bdee75e2ac037 100644 --- a/modules/packages/cargo/parser.go +++ b/modules/packages/cargo/parser.go @@ -24,10 +24,11 @@ var ( // Package represents a Cargo package type Package struct { - Name string - Version string - Metadata *Metadata - Content io.Reader + Name string + Version string + Metadata *Metadata + Content io.Reader + ContentSize int64 } // Metadata represents the metadata of a Cargo package @@ -77,6 +78,7 @@ func ParsePackage(r io.Reader) (*Package, error) { } p.Content = io.LimitReader(r, int64(size)) + p.ContentSize = int64(size) return p, nil } diff --git a/routers/api/packages/cargo/cargo.go b/routers/api/packages/cargo/cargo.go index 85a15d2e0ee4b..e0bf5da13adb0 100644 --- a/routers/api/packages/cargo/cargo.go +++ b/routers/api/packages/cargo/cargo.go @@ -180,6 +180,11 @@ func UploadPackage(ctx *context.Context) { } defer buf.Close() + if buf.Size() != cp.ContentSize { + apiError(ctx, http.StatusBadRequest, "invalid content size") + return + } + pv, _, err := packages_service.CreatePackageAndAddFile( &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index 3381d217da7b8..3d8f429f15d12 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -133,6 +133,22 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { assert.False(t, status.OK) }) + t.Run("InvalidContent", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + metadata := `{"name":"test","vers":"1.0.0"}` + + var buf bytes.Buffer + binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) + buf.WriteString(metadata) + binary.Write(&buf, binary.LittleEndian, uint32(4)) + buf.WriteString("te") + + req := NewRequestWithBody(t, "PUT", url+"/new", &buf) + req = AddBasicAuthHeader(req, user.Name) + MakeRequest(t, req, http.StatusBadRequest) + }) + t.Run("Valid", func(t *testing.T) { defer tests.PrintCurrentTest(t)() From b5fcf448d56f5dbc07c89f141a2a6e0ae984b902 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 25 Jan 2023 11:17:28 +0000 Subject: [PATCH 08/11] Fix dependency json format. --- modules/packages/cargo/parser.go | 49 +++++++++++++++++++++++--------- services/packages/cargo/index.go | 1 + 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go index bdee75e2ac037..99a5686a3aab6 100644 --- a/modules/packages/cargo/parser.go +++ b/modules/packages/cargo/parser.go @@ -48,15 +48,14 @@ type Metadata struct { } type Dependency struct { - Name string `json:"name"` - VersionReq string `json:"version_req"` - Features []string `json:"features"` - Optional bool `json:"optional"` - DefaultFeatures bool `json:"default_features"` - Target string `json:"target"` - Kind string `json:"kind"` - Registry string `json:"registry"` - ExplicitNameInToml string `json:"explicit_name_in_toml"` + Name string `json:"name"` + Req string `json:"req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target string `json:"target,omitempty"` + Kind string `json:"kind"` + Registry string `json:"registry,omitempty"` } var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`) @@ -85,9 +84,19 @@ func ParsePackage(r io.Reader) (*Package, error) { func parsePackage(r io.Reader) (*Package, error) { var meta struct { - Name string `json:"name"` - Vers string `json:"vers"` - Deps []*Dependency `json:"deps"` + Name string `json:"name"` + Vers string `json:"vers"` + Deps []struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target string `json:"target"` + Kind string `json:"kind"` + Registry string `json:"registry"` + ExplicitNameInToml string `json:"explicit_name_in_toml"` + } `json:"deps"` Features map[string][]string `json:"features"` Authors []string `json:"authors"` Description string `json:"description"` @@ -124,11 +133,25 @@ func parsePackage(r io.Reader) (*Package, error) { meta.Repository = "" } + dependencies := make([]*Dependency, 0, len(meta.Deps)) + for _, dep := range meta.Deps { + dependencies = append(dependencies, &Dependency{ + Name: dep.Name, + Req: dep.VersionReq, + Features: dep.Features, + Optional: dep.Optional, + DefaultFeatures: dep.DefaultFeatures, + Target: dep.Target, + Kind: dep.Kind, + Registry: dep.Registry, + }) + } + return &Package{ Name: meta.Name, Version: meta.Vers, Metadata: &Metadata{ - Dependencies: meta.Deps, + Dependencies: dependencies, Features: meta.Features, Authors: meta.Authors, Description: meta.Description, diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index 76f6b5a5de4cd..f8e58c9d01994 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -140,6 +140,7 @@ type IndexVersionEntry struct { func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{ PackageID: p.ID, + Sort: packages_model.SortVersionAsc, }) if err != nil { return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err) From d391bdf642de6fc6eb22816852c4707928ca2c3a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 26 Jan 2023 11:58:53 +0000 Subject: [PATCH 09/11] Use pointers. --- modules/packages/cargo/parser.go | 9 +++++---- services/packages/cargo/index.go | 4 ++-- templates/package/content/cargo.tmpl | 2 +- tests/integration/api_packages_cargo_test.go | 17 ++++++++++++++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go index 99a5686a3aab6..36cd44df847aa 100644 --- a/modules/packages/cargo/parser.go +++ b/modules/packages/cargo/parser.go @@ -53,9 +53,10 @@ type Dependency struct { Features []string `json:"features"` Optional bool `json:"optional"` DefaultFeatures bool `json:"default_features"` - Target string `json:"target,omitempty"` + Target *string `json:"target"` Kind string `json:"kind"` - Registry string `json:"registry,omitempty"` + Registry *string `json:"registry"` + Package *string `json:"package"` } var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`) @@ -92,9 +93,9 @@ func parsePackage(r io.Reader) (*Package, error) { Features []string `json:"features"` Optional bool `json:"optional"` DefaultFeatures bool `json:"default_features"` - Target string `json:"target"` + Target *string `json:"target"` Kind string `json:"kind"` - Registry string `json:"registry"` + Registry *string `json:"registry"` ExplicitNameInToml string `json:"explicit_name_in_toml"` } `json:"deps"` Features map[string][]string `json:"features"` diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index f8e58c9d01994..28e7daf694713 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -130,9 +130,9 @@ func AddOrUpdatePackageIndex(ctx context.Context, doer, owner *user_model.User, type IndexVersionEntry struct { Name string `json:"name"` Version string `json:"vers"` - Dependencies []*cargo_module.Dependency `json:"deps,omitempty"` + Dependencies []*cargo_module.Dependency `json:"deps"` FileChecksum string `json:"cksum"` - Features map[string][]string `json:"features,omitempty"` + Features map[string][]string `json:"features"` Yanked bool `json:"yanked"` Links string `json:"links,omitempty"` } diff --git a/templates/package/content/cargo.tmpl b/templates/package/content/cargo.tmpl index 3f82a69c228fc..54c40a5b0dc0a 100644 --- a/templates/package/content/cargo.tmpl +++ b/templates/package/content/cargo.tmpl @@ -43,7 +43,7 @@ git-fetch-with-cli = true {{range .PackageDescriptor.Metadata.Dependencies}} {{.Name}} - {{.VersionReq}} + {{.Req}} {{end}} diff --git a/tests/integration/api_packages_cargo_test.go b/tests/integration/api_packages_cargo_test.go index 3d8f429f15d12..0c542eaf1e857 100644 --- a/tests/integration/api_packages_cargo_test.go +++ b/tests/integration/api_packages_cargo_test.go @@ -52,7 +52,10 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { "deps":[ { "name":"dep", - "version_req":"1.0" + "version_req":"1.0", + "registry": "https://gitea.io/user/_cargo-index", + "kind": "normal", + "default_features": true } ], "homepage":"` + packageHomepage + `", @@ -202,6 +205,18 @@ func testPackageCargo(t *testing.T, _ *neturl.URL) { assert.Equal(t, packageVersion, entry.Version) assert.Equal(t, pb.HashSHA256, entry.FileChecksum) assert.False(t, entry.Yanked) + assert.Len(t, entry.Dependencies, 1) + dep := entry.Dependencies[0] + assert.Equal(t, "dep", dep.Name) + assert.Equal(t, "1.0", dep.Req) + assert.Equal(t, "normal", dep.Kind) + assert.True(t, dep.DefaultFeatures) + assert.Empty(t, dep.Features) + assert.False(t, dep.Optional) + assert.Nil(t, dep.Target) + assert.NotNil(t, dep.Registry) + assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry) + assert.Nil(t, dep.Package) }) t.Run("Rebuild", func(t *testing.T) { From d0264263e9bc1bac8179b99986b17ede1356654c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 26 Jan 2023 20:01:37 +0000 Subject: [PATCH 10/11] Force non-nil values in json. --- services/packages/cargo/index.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index 28e7daf694713..ba116d5cc7062 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -134,7 +134,7 @@ type IndexVersionEntry struct { FileChecksum string `json:"cksum"` Features map[string][]string `json:"features"` Yanked bool `json:"yanked"` - Links string `json:"links,omitempty"` + Links *string `json:"links"` } func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { @@ -157,15 +157,31 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo var b bytes.Buffer for _, pd := range pds { metadata := pd.Metadata.(*cargo_module.Metadata) + + dependencies := metadata.Dependencies + if dependencies == nil { + dependencies = make([]*cargo_module.Dependency, 0) + } + + features := metadata.Features + if features == nil { + features = make(map[string][]string) + } + + var links *string + if metadata.Links != "" { + links = &metadata.Links + } + yanked, _ := strconv.ParseBool(pd.VersionProperties.GetByName(cargo_module.PropertyYanked)) entry, err := json.Marshal(&IndexVersionEntry{ Name: pd.Package.Name, Version: pd.Version.Version, - Dependencies: metadata.Dependencies, + Dependencies: dependencies, FileChecksum: pd.Files[0].Blob.HashSHA256, - Features: metadata.Features, + Features: features, Yanked: yanked, - Links: metadata.Links, + Links: links, }) if err != nil { return err From c43d84e69dbcd5dd664b4eaaf26cc00173b3e845 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 26 Jan 2023 20:06:17 +0000 Subject: [PATCH 11/11] Field is allowed to be missing. --- services/packages/cargo/index.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/services/packages/cargo/index.go b/services/packages/cargo/index.go index ba116d5cc7062..e58a47281628b 100644 --- a/services/packages/cargo/index.go +++ b/services/packages/cargo/index.go @@ -134,7 +134,7 @@ type IndexVersionEntry struct { FileChecksum string `json:"cksum"` Features map[string][]string `json:"features"` Yanked bool `json:"yanked"` - Links *string `json:"links"` + Links string `json:"links,omitempty"` } func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error { @@ -168,11 +168,6 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo features = make(map[string][]string) } - var links *string - if metadata.Links != "" { - links = &metadata.Links - } - yanked, _ := strconv.ParseBool(pd.VersionProperties.GetByName(cargo_module.PropertyYanked)) entry, err := json.Marshal(&IndexVersionEntry{ Name: pd.Package.Name, @@ -181,7 +176,7 @@ func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUplo FileChecksum: pd.Files[0].Blob.HashSHA256, Features: features, Yanked: yanked, - Links: links, + Links: metadata.Links, }) if err != nil { return err