diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index f11cf9b11f373..a283f91401839 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -112,11 +112,8 @@ func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) e func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error { return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error { - p, err := archiver.RelativePath() - if err != nil { - return err - } - _, err = storage.Copy(dstStorage, p, storage.RepoArchives, p) + p := archiver.RelativePath() + _, err := storage.Copy(dstStorage, p, storage.RepoArchives, p) return err }) } diff --git a/docs/content/doc/packages/overview.en-us.md b/docs/content/doc/packages/overview.en-us.md index 5e03b710170f1..0abb054b0f1ba 100644 --- a/docs/content/doc/packages/overview.en-us.md +++ b/docs/content/doc/packages/overview.en-us.md @@ -37,6 +37,7 @@ The following package managers are currently supported: | [Pub]({{< relref "doc/packages/pub.en-us.md" >}}) | Dart | `dart`, `flutter` | | [PyPI]({{< relref "doc/packages/pypi.en-us.md" >}}) | Python | `pip`, `twine` | | [RubyGems]({{< relref "doc/packages/rubygems.en-us.md" >}}) | Ruby | `gem`, `Bundler` | +| [Vagrant]({{< relref "doc/packages/vagrant.en-us.md" >}}) | - | `vagrant` | **The following paragraphs only apply if Packages are not globally disabled!** diff --git a/docs/content/doc/packages/vagrant.en-us.md b/docs/content/doc/packages/vagrant.en-us.md new file mode 100644 index 0000000000000..e846de1a2a707 --- /dev/null +++ b/docs/content/doc/packages/vagrant.en-us.md @@ -0,0 +1,78 @@ +--- +date: "2022-08-23T00:00:00+00:00" +title: "Vagrant Packages Repository" +slug: "packages/vagrant" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "vagrant" + weight: 120 + identifier: "vagrant" +--- + +# Vagrant Packages Repository + +Publish [Vagrant](https://www.vagrantup.com/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the Vagrant package registry, you need [Vagrant](https://www.vagrantup.com/downloads) and a tool to make HTTP requests like `curl`. + +## Publish a package + +Publish a Vagrant box by performing a HTTP PUT request to the registry: + +``` +PUT https://gitea.example.com/api/packages/{owner}/vagrant/{package_name}/{package_version}/{provider}.box +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the package. | +| `package_name` | The package name. | +| `package_version` | The package version, semver compatible. | +| `provider` | One of the [supported provider names](https://www.vagrantup.com/docs/providers). | + +Example for uploading a Hyper-V box: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/your/vagrant.box \ + https://gitea.example.com/api/packages/testuser/vagrant/test_system/1.0.0/hyperv.box +``` + +You cannot publish a box if a box of the same name, version and provider already exists. You must delete the existing package first. + +## Install a package + +To install a box from the package registry, execute the following command: + +```shell +vagrant box add "https://gitea.example.com/api/packages/{owner}/vagrant/{package_name}" +``` + +| Parameter | Description | +| -------------- | ----------- | +| `owner` | The owner of the package. | +| `package_name` | The package name. | + +For example: + +```shell +vagrant box add "https://gitea.example.com/api/packages/testuser/vagrant/test_system" +``` + +This will install the latest version of the package. To add a specific version, use the `--box-version` parameter. +If the registry is private you can pass your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) in the `VAGRANT_CLOUD_TOKEN` environment variable. + +## Supported commands + +``` +vagrant box add +``` diff --git a/docs/content/doc/usage/https-support.md b/docs/content/doc/usage/https-support.md index 783d6d803774f..7c9024743d965 100644 --- a/docs/content/doc/usage/https-support.md +++ b/docs/content/doc/usage/https-support.md @@ -72,7 +72,7 @@ ACME_DIRECTORY=https ACME_EMAIL=email@example.com ``` -Minimumg setup using a [smallstep CA](https://github.com/smallstep/certificates), refer to [their tutorial](https://smallstep.com/docs/tutorials/acme-challenge) for more information. +Minimum setup using a [smallstep CA](https://github.com/smallstep/certificates), refer to [their tutorial](https://smallstep.com/docs/tutorials/acme-challenge) for more information. ```ini [server] diff --git a/integrations/api_packages_vagrant_test.go b/integrations/api_packages_vagrant_test.go new file mode 100644 index 0000000000000..8ce4f24ef5cc4 --- /dev/null +++ b/integrations/api_packages_vagrant_test.go @@ -0,0 +1,170 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integrations + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "fmt" + "net/http" + "strings" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + vagrant_module "code.gitea.io/gitea/modules/packages/vagrant" + + "github.com/stretchr/testify/assert" +) + +func TestPackageVagrant(t *testing.T) { + defer prepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + + token := "Bearer " + getUserToken(t, user.Name) + + packageName := "test_package" + packageVersion := "1.0.1" + packageDescription := "Test Description" + packageProvider := "virtualbox" + + filename := fmt.Sprintf("%s.box", packageProvider) + + infoContent, _ := json.Marshal(map[string]string{ + "description": packageDescription, + }) + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + archive := tar.NewWriter(zw) + archive.WriteHeader(&tar.Header{ + Name: "info.json", + Mode: 0o600, + Size: int64(len(infoContent)), + }) + archive.Write(infoContent) + archive.Close() + zw.Close() + content := buf.Bytes() + + root := fmt.Sprintf("/api/packages/%s/vagrant", user.Name) + + t.Run("Authenticate", func(t *testing.T) { + defer PrintCurrentTest(t)() + + authenticateURL := fmt.Sprintf("%s/authenticate", root) + + req := NewRequest(t, "GET", authenticateURL) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequest(t, "GET", authenticateURL) + addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusOK) + }) + + boxURL := fmt.Sprintf("%s/%s", root, packageName) + + t.Run("Upload", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "HEAD", boxURL) + MakeRequest(t, req, http.StatusNotFound) + + uploadURL := fmt.Sprintf("%s/%s/%s", boxURL, packageVersion, filename) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + MakeRequest(t, req, http.StatusUnauthorized) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusCreated) + + req = NewRequest(t, "HEAD", boxURL) + resp := MakeRequest(t, req, http.StatusOK) + assert.True(t, strings.HasPrefix(resp.HeaderMap.Get("Content-Type"), "application/json")) + + pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeVagrant) + assert.NoError(t, err) + assert.Len(t, pvs, 1) + + pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) + assert.NoError(t, err) + assert.NotNil(t, pd.SemVer) + assert.IsType(t, &vagrant_module.Metadata{}, pd.Metadata) + assert.Equal(t, packageName, pd.Package.Name) + assert.Equal(t, packageVersion, pd.Version.Version) + + pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) + assert.NoError(t, err) + assert.Len(t, pfs, 1) + assert.Equal(t, filename, pfs[0].Name) + assert.True(t, pfs[0].IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + + req = NewRequestWithBody(t, "PUT", uploadURL, bytes.NewReader(content)) + addTokenAuthHeader(req, token) + MakeRequest(t, req, http.StatusConflict) + }) + + t.Run("Download", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", boxURL, packageVersion, filename)) + resp := MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, content, resp.Body.Bytes()) + }) + + t.Run("EnumeratePackageVersions", func(t *testing.T) { + defer PrintCurrentTest(t)() + + req := NewRequest(t, "GET", boxURL) + resp := MakeRequest(t, req, http.StatusOK) + + type providerData struct { + Name string `json:"name"` + URL string `json:"url"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` + } + + type versionMetadata struct { + Version string `json:"version"` + Status string `json:"status"` + DescriptionHTML string `json:"description_html,omitempty"` + DescriptionMarkdown string `json:"description_markdown,omitempty"` + Providers []*providerData `json:"providers"` + } + + type packageMetadata struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + ShortDescription string `json:"short_description,omitempty"` + Versions []*versionMetadata `json:"versions"` + } + + var result packageMetadata + DecodeJSON(t, resp, &result) + + assert.Equal(t, packageName, result.Name) + assert.Equal(t, packageDescription, result.Description) + assert.Len(t, result.Versions, 1) + version := result.Versions[0] + assert.Equal(t, packageVersion, version.Version) + assert.Equal(t, "active", version.Status) + assert.Len(t, version.Providers, 1) + provider := version.Providers[0] + assert.Equal(t, packageProvider, provider.Name) + assert.Equal(t, "sha512", provider.ChecksumType) + assert.Equal(t, "259bebd6160acad695016d22a45812e26f187aaf78e71a4c23ee3201528346293f991af3468a8c6c5d2a21d7d9e1bdc1bf79b87110b2fddfcc5a0d45963c7c30", provider.Checksum) + }) +} diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index dc753421d0210..357574a706c2d 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/packages/pub" "code.gitea.io/gitea/modules/packages/pypi" "code.gitea.io/gitea/modules/packages/rubygems" + "code.gitea.io/gitea/modules/packages/vagrant" "github.com/hashicorp/go-version" ) @@ -150,6 +151,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &pypi.Metadata{} case TypeRubyGems: metadata = &rubygems.Metadata{} + case TypeVagrant: + metadata = &vagrant.Metadata{} default: panic(fmt.Sprintf("unknown package type: %s", string(p.Type))) } diff --git a/models/packages/package.go b/models/packages/package.go index 39b1c83cfabf6..c203901a2d4c8 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -42,6 +42,7 @@ const ( TypePub Type = "pub" TypePyPI Type = "pypi" TypeRubyGems Type = "rubygems" + TypeVagrant Type = "vagrant" ) // Name gets the name of the package type @@ -69,6 +70,8 @@ func (pt Type) Name() string { return "PyPI" case TypeRubyGems: return "RubyGems" + case TypeVagrant: + return "Vagrant" } panic(fmt.Sprintf("unknown package type: %s", string(pt))) } @@ -98,6 +101,8 @@ func (pt Type) SVGName() string { return "gitea-python" case TypeRubyGems: return "gitea-rubygems" + case TypeVagrant: + return "gitea-vagrant" } panic(fmt.Sprintf("unknown package type: %s", string(pt))) } diff --git a/models/repo.go b/models/repo.go index 35af6c1ef46d7..f2676a5696148 100644 --- a/models/repo.go +++ b/models/repo.go @@ -218,8 +218,7 @@ func DeleteRepository(doer *user_model.User, uid, repoID int64) error { archivePaths := make([]string, 0, len(archives)) for _, v := range archives { - p, _ := v.RelativePath() - archivePaths = append(archivePaths, p) + archivePaths = append(archivePaths, v.RelativePath()) } if _, err := db.DeleteByBean(ctx, &repo_model.RepoArchiver{RepoID: repoID}); err != nil { diff --git a/models/repo/archiver.go b/models/repo/archiver.go index fd07d8554d361..003911943f52a 100644 --- a/models/repo/archiver.go +++ b/models/repo/archiver.go @@ -39,9 +39,9 @@ func init() { db.RegisterModel(new(RepoArchiver)) } -// RelativePath returns relative path -func (archiver *RepoArchiver) RelativePath() (string, error) { - return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()), nil +// RelativePath returns the archive path relative to the archive storage root. +func (archiver *RepoArchiver) RelativePath() string { + return fmt.Sprintf("%d/%s/%s.%s", archiver.RepoID, archiver.CommitID[:2], archiver.CommitID, archiver.Type.String()) } var delRepoArchiver = new(RepoArchiver) diff --git a/modules/nosql/manager_redis.go b/modules/nosql/manager_redis.go index 3b2ad75b41f96..f7d5a72ed2313 100644 --- a/modules/nosql/manager_redis.go +++ b/modules/nosql/manager_redis.go @@ -245,7 +245,7 @@ func getRedisTLSOptions(uri *url.URL) *tls.Config { if len(skipverify) > 0 { skipverify, err := strconv.ParseBool(skipverify) - if err != nil { + if err == nil { tlsConfig.InsecureSkipVerify = skipverify } } @@ -254,7 +254,7 @@ func getRedisTLSOptions(uri *url.URL) *tls.Config { if len(insecureskipverify) > 0 { insecureskipverify, err := strconv.ParseBool(insecureskipverify) - if err != nil { + if err == nil { tlsConfig.InsecureSkipVerify = insecureskipverify } } diff --git a/modules/nosql/manager_redis_test.go b/modules/nosql/manager_redis_test.go index 3d94532135162..99a8856f1e836 100644 --- a/modules/nosql/manager_redis_test.go +++ b/modules/nosql/manager_redis_test.go @@ -27,6 +27,24 @@ func TestRedisPasswordOpt(t *testing.T) { } } +func TestSkipVerifyOpt(t *testing.T) { + uri, _ := url.Parse("rediss://myredis/0?skipverify=true") + tlsConfig := getRedisTLSOptions(uri) + + if !tlsConfig.InsecureSkipVerify { + t.Fail() + } +} + +func TestInsecureSkipVerifyOpt(t *testing.T) { + uri, _ := url.Parse("rediss://myredis/0?insecureskipverify=true") + tlsConfig := getRedisTLSOptions(uri) + + if !tlsConfig.InsecureSkipVerify { + t.Fail() + } +} + func TestRedisSentinelUsernameOpt(t *testing.T) { uri, _ := url.Parse("redis+sentinel://redis:password@myredis/0?sentinelusername=suser&sentinelpassword=spass") opts := getRedisOptions(uri).Failover() diff --git a/modules/packages/vagrant/metadata.go b/modules/packages/vagrant/metadata.go new file mode 100644 index 0000000000000..278dfab32e813 --- /dev/null +++ b/modules/packages/vagrant/metadata.go @@ -0,0 +1,97 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vagrant + +import ( + "archive/tar" + "compress/gzip" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/validation" +) + +const ( + PropertyProvider = "vagrant.provider" +) + +// Metadata represents the metadata of a Vagrant package +type Metadata struct { + Author string `json:"author,omitempty"` + Description string `json:"description,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` +} + +// ParseMetadataFromBox parses the metdata of a box file +func ParseMetadataFromBox(r io.Reader) (*Metadata, error) { + gzr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.Typeflag != tar.TypeReg { + continue + } + + if hd.Name == "info.json" { + return ParseInfoFile(tr) + } + } + + return &Metadata{}, nil +} + +// ParseInfoFile parses a info.json file to retrieve the metadata of a Vagrant package +func ParseInfoFile(r io.Reader) (*Metadata, error) { + var values map[string]string + if err := json.NewDecoder(r).Decode(&values); err != nil { + return nil, err + } + + m := &Metadata{} + + // There is no defined format for this file, just try the common keys + for k, v := range values { + switch strings.ToLower(k) { + case "description": + fallthrough + case "short_description": + m.Description = v + case "website": + fallthrough + case "homepage": + fallthrough + case "url": + if validation.IsValidURL(v) { + m.ProjectURL = v + } + case "repository": + fallthrough + case "source": + if validation.IsValidURL(v) { + m.RepositoryURL = v + } + case "author": + fallthrough + case "authors": + m.Author = v + } + } + + return m, nil +} diff --git a/modules/packages/vagrant/metadata_test.go b/modules/packages/vagrant/metadata_test.go new file mode 100644 index 0000000000000..9720c945ae8ab --- /dev/null +++ b/modules/packages/vagrant/metadata_test.go @@ -0,0 +1,111 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vagrant + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/assert" +) + +const ( + author = "gitea" + description = "Package Description" + projectURL = "https://gitea.io" + repositoryURL = "https://gitea.io/gitea/gitea" +) + +func TestParseMetadataFromBox(t *testing.T) { + createArchive := func(files map[string][]byte) io.Reader { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + zw.Close() + return &buf + } + + t.Run("MissingInfoFile", func(t *testing.T) { + data := createArchive(map[string][]byte{"dummy.txt": {}}) + + metadata, err := ParseMetadataFromBox(data) + assert.NotNil(t, metadata) + assert.NoError(t, err) + }) + + t.Run("Valid", func(t *testing.T) { + content, err := json.Marshal(map[string]string{ + "description": description, + "author": author, + "website": projectURL, + "repository": repositoryURL, + }) + assert.NoError(t, err) + + data := createArchive(map[string][]byte{"info.json": content}) + + metadata, err := ParseMetadataFromBox(data) + assert.NotNil(t, metadata) + assert.NoError(t, err) + + assert.Equal(t, author, metadata.Author) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + }) +} + +func TestParseInfoFile(t *testing.T) { + t.Run("UnknownKeys", func(t *testing.T) { + content, err := json.Marshal(map[string]string{ + "package": "", + "dummy": "", + }) + assert.NoError(t, err) + + metadata, err := ParseInfoFile(bytes.NewReader(content)) + assert.NotNil(t, metadata) + assert.NoError(t, err) + + assert.Empty(t, metadata.Author) + assert.Empty(t, metadata.Description) + assert.Empty(t, metadata.ProjectURL) + assert.Empty(t, metadata.RepositoryURL) + }) + + t.Run("Valid", func(t *testing.T) { + content, err := json.Marshal(map[string]string{ + "description": description, + "author": author, + "website": projectURL, + "repository": repositoryURL, + }) + assert.NoError(t, err) + + metadata, err := ParseInfoFile(bytes.NewReader(content)) + assert.NotNil(t, metadata) + assert.NoError(t, err) + + assert.Equal(t, author, metadata.Author) + assert.Equal(t, description, metadata.Description) + assert.Equal(t, projectURL, metadata.ProjectURL) + assert.Equal(t, repositoryURL, metadata.RepositoryURL) + }) +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e87ee34441611..f900e90e7f851 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3131,6 +3131,8 @@ rubygems.dependencies.development = Development Dependencies rubygems.required.ruby = Requires Ruby version rubygems.required.rubygems = Requires RubyGem version rubygems.documentation = For more information on the RubyGems registry, see the documentation. +vagrant.install = To add a Vagrant box, run the following command: +vagrant.documentation = For more information on the Vagrant registry, see the documentation. settings.link = Link this package to a repository settings.link.description = If you link a package with a repository, the package is listed in the repository's package list. settings.link.select = Select Repository diff --git a/public/img/svg/gitea-vagrant.svg b/public/img/svg/gitea-vagrant.svg new file mode 100644 index 0000000000000..4c1b78cab5402 --- /dev/null +++ b/public/img/svg/gitea-vagrant.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 0761cacdf2e2b..93f9afca7b283 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/routers/api/packages/pub" "code.gitea.io/gitea/routers/api/packages/pypi" "code.gitea.io/gitea/routers/api/packages/rubygems" + "code.gitea.io/gitea/routers/api/packages/vagrant" "code.gitea.io/gitea/services/auth" context_service "code.gitea.io/gitea/services/context" ) @@ -266,6 +267,19 @@ func Routes(ctx gocontext.Context) *web.Route { r.Delete("/yank", rubygems.DeletePackage) }, reqPackageAccess(perm.AccessModeWrite)) }) + r.Group("/vagrant", func() { + r.Group("/authenticate", func() { + r.Get("", vagrant.CheckAuthenticate) + }) + r.Group("/{name}", func() { + r.Head("", vagrant.CheckBoxAvailable) + r.Get("", vagrant.EnumeratePackageVersions) + r.Group("/{version}/{provider}", func() { + r.Get("", vagrant.DownloadPackageFile) + r.Put("", reqPackageAccess(perm.AccessModeWrite), vagrant.UploadPackageFile) + }) + }) + }) }, context_service.UserAssignmentWeb(), context.PackageAssignment(), reqPackageAccess(perm.AccessModeRead)) return r diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go new file mode 100644 index 0000000000000..7750e5dc4b2ba --- /dev/null +++ b/routers/api/packages/vagrant/vagrant.go @@ -0,0 +1,239 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package vagrant + +import ( + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + + packages_model "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/modules/context" + packages_module "code.gitea.io/gitea/modules/packages" + vagrant_module "code.gitea.io/gitea/modules/packages/vagrant" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/routers/api/packages/helper" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/hashicorp/go-version" +) + +func apiError(ctx *context.Context, status int, obj interface{}) { + helper.LogAndProcessError(ctx, status, obj, func(message string) { + ctx.JSON(status, struct { + Errors []string `json:"errors"` + }{ + Errors: []string{ + message, + }, + }) + }) +} + +func CheckAuthenticate(ctx *context.Context) { + if ctx.Doer == nil { + apiError(ctx, http.StatusUnauthorized, "Invalid access token") + return + } + + ctx.Status(http.StatusOK) +} + +func CheckBoxAvailable(ctx *context.Context) { + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeVagrant, ctx.Params("name")) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + ctx.JSON(http.StatusOK, nil) // needs to be Content-Type: application/json +} + +type packageMetadata struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + ShortDescription string `json:"short_description,omitempty"` + Versions []*versionMetadata `json:"versions"` +} + +type versionMetadata struct { + Version string `json:"version"` + Status string `json:"status"` + DescriptionHTML string `json:"description_html,omitempty"` + DescriptionMarkdown string `json:"description_markdown,omitempty"` + Providers []*providerData `json:"providers"` +} + +type providerData struct { + Name string `json:"name"` + URL string `json:"url"` + Checksum string `json:"checksum"` + ChecksumType string `json:"checksum_type"` +} + +func packageDescriptorToMetadata(baseURL string, pd *packages_model.PackageDescriptor) *versionMetadata { + versionURL := baseURL + "/" + url.PathEscape(pd.Version.Version) + + providers := make([]*providerData, 0, len(pd.Files)) + + for _, f := range pd.Files { + providers = append(providers, &providerData{ + Name: f.Properties.GetByName(vagrant_module.PropertyProvider), + URL: versionURL + "/" + url.PathEscape(f.File.Name), + Checksum: f.Blob.HashSHA512, + ChecksumType: "sha512", + }) + } + + return &versionMetadata{ + Status: "active", + Version: pd.Version.Version, + Providers: providers, + } +} + +func EnumeratePackageVersions(ctx *context.Context) { + pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeVagrant, ctx.Params("name")) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if len(pvs) == 0 { + apiError(ctx, http.StatusNotFound, err) + return + } + + pds, err := packages_model.GetPackageDescriptors(ctx, pvs) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + sort.Slice(pds, func(i, j int) bool { + return pds[i].SemVer.LessThan(pds[j].SemVer) + }) + + baseURL := fmt.Sprintf("%sapi/packages/%s/vagrant/%s", setting.AppURL, url.PathEscape(ctx.Package.Owner.Name), url.PathEscape(pds[0].Package.Name)) + + versions := make([]*versionMetadata, 0, len(pds)) + for _, pd := range pds { + versions = append(versions, packageDescriptorToMetadata(baseURL, pd)) + } + + ctx.JSON(http.StatusOK, &packageMetadata{ + Name: pds[0].Package.Name, + Description: pds[len(pds)-1].Metadata.(*vagrant_module.Metadata).Description, + Versions: versions, + }) +} + +func UploadPackageFile(ctx *context.Context) { + boxName := ctx.Params("name") + boxVersion := ctx.Params("version") + _, err := version.NewSemver(boxVersion) + if err != nil { + apiError(ctx, http.StatusBadRequest, err) + return + } + boxProvider := ctx.Params("provider") + if !strings.HasSuffix(boxProvider, ".box") { + apiError(ctx, http.StatusBadRequest, err) + return + } + + upload, needsClose, err := ctx.UploadStream() + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + if needsClose { + defer upload.Close() + } + + buf, err := packages_module.CreateHashedBufferFromReader(upload, 32*1024*1024) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer buf.Close() + + metadata, err := vagrant_module.ParseMetadataFromBox(buf) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + + _, _, err = packages_service.CreatePackageOrAddFileToExisting( + &packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeVagrant, + Name: boxName, + Version: boxVersion, + }, + SemverCompatible: true, + Creator: ctx.Doer, + Metadata: metadata, + }, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(boxProvider), + }, + Data: buf, + IsLead: true, + Properties: map[string]string{ + vagrant_module.PropertyProvider: strings.TrimSuffix(boxProvider, ".box"), + }, + }, + ) + if err != nil { + if err == packages_model.ErrDuplicatePackageFile { + apiError(ctx, http.StatusConflict, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + + ctx.Status(http.StatusCreated) +} + +func DownloadPackageFile(ctx *context.Context) { + s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion( + ctx, + &packages_service.PackageInfo{ + Owner: ctx.Package.Owner, + PackageType: packages_model.TypeVagrant, + Name: ctx.Params("name"), + Version: ctx.Params("version"), + }, + &packages_service.PackageFileInfo{ + Filename: ctx.Params("provider"), + }, + ) + if err != nil { + if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist { + apiError(ctx, http.StatusNotFound, err) + return + } + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer s.Close() + + ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) +} diff --git a/routers/api/v1/packages/package.go b/routers/api/v1/packages/package.go index 2c023891022aa..29fa6c85a5b6f 100644 --- a/routers/api/v1/packages/package.go +++ b/routers/api/v1/packages/package.go @@ -41,7 +41,7 @@ func ListPackages(ctx *context.APIContext) { // in: query // description: package type filter // type: string - // enum: [composer, conan, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems] + // enum: [composer, conan, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant] // - name: q // in: query // description: name filter diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 8353a4e501cd7..6dead81e6d52c 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -8,6 +8,7 @@ package repo import ( "bytes" "encoding/base64" + "errors" "fmt" "io" "net/http" @@ -28,7 +29,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/common" - "code.gitea.io/gitea/routers/web/repo" + archiver_service "code.gitea.io/gitea/services/repository/archiver" files_service "code.gitea.io/gitea/services/repository/files" ) @@ -294,7 +295,53 @@ func GetArchive(ctx *context.APIContext) { defer gitRepo.Close() } - repo.Download(ctx.Context) + archiveDownload(ctx) +} + +func archiveDownload(ctx *context.APIContext) { + uri := ctx.Params("*") + aReq, err := archiver_service.NewRequest(ctx.Repo.Repository.ID, ctx.Repo.GitRepo, uri) + if err != nil { + if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { + ctx.Error(http.StatusBadRequest, "unknown archive format", err) + } else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) { + ctx.Error(http.StatusNotFound, "unrecognized reference", err) + } else { + ctx.ServerError("archiver_service.NewRequest", err) + } + return + } + + archiver, err := aReq.Await(ctx) + if err != nil { + ctx.ServerError("archiver.Await", err) + return + } + + download(ctx, aReq.GetArchiveName(), archiver) +} + +func download(ctx *context.APIContext, archiveName string, archiver *repo_model.RepoArchiver) { + downloadName := ctx.Repo.Repository.Name + "-" + archiveName + + rPath := archiver.RelativePath() + if setting.RepoArchive.ServeDirect { + // If we have a signed url (S3, object storage), redirect to this directly. + u, err := storage.RepoArchives.URL(rPath, downloadName) + if u != nil && err == nil { + ctx.Redirect(u.String()) + return + } + } + + // If we have matched and access to release or issue + fr, err := storage.RepoArchives.Open(rPath) + if err != nil { + ctx.ServerError("Open", err) + return + } + defer fr.Close() + ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) } // GetEditorconfig get editor config of a repository diff --git a/routers/web/admin/users.go b/routers/web/admin/users.go index aab633ec84b24..5cdfb8142ee89 100644 --- a/routers/web/admin/users.go +++ b/routers/web/admin/users.go @@ -209,7 +209,11 @@ func NewUserPost(ctx *context.Context) { func prepareUserInfo(ctx *context.Context) *user_model.User { u, err := user_model.GetUserByID(ctx.ParamsInt64(":userid")) if err != nil { - ctx.ServerError("GetUserByID", err) + if user_model.IsErrUserNotExist(err) { + ctx.Redirect(setting.AppSubURL + "/admin/users") + } else { + ctx.ServerError("GetUserByID", err) + } return nil } ctx.Data["User"] = u diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index ae0351a219bc5..974f03f951053 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -10,7 +10,6 @@ import ( "fmt" "net/http" "strings" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/db" @@ -22,7 +21,6 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/convert" - "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" @@ -390,68 +388,27 @@ func Download(ctx *context.Context) { if err != nil { if errors.Is(err, archiver_service.ErrUnknownArchiveFormat{}) { ctx.Error(http.StatusBadRequest, err.Error()) + } else if errors.Is(err, archiver_service.RepoRefNotFoundError{}) { + ctx.Error(http.StatusNotFound, err.Error()) } else { ctx.ServerError("archiver_service.NewRequest", err) } return } - if aReq == nil { - ctx.Error(http.StatusNotFound) - return - } - archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + archiver, err := aReq.Await(ctx) if err != nil { - ctx.ServerError("models.GetRepoArchiver", err) + ctx.ServerError("archiver.Await", err) return } - if archiver != nil && archiver.Status == repo_model.ArchiverReady { - download(ctx, aReq.GetArchiveName(), archiver) - return - } - - if err := archiver_service.StartArchive(aReq); err != nil { - ctx.ServerError("archiver_service.StartArchive", err) - return - } - - var times int - t := time.NewTicker(time.Second * 1) - defer t.Stop() - for { - select { - case <-graceful.GetManager().HammerContext().Done(): - log.Warn("exit archive download because system stop") - return - case <-t.C: - if times > 20 { - ctx.ServerError("wait download timeout", nil) - return - } - times++ - archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) - if err != nil { - ctx.ServerError("archiver_service.StartArchive", err) - return - } - if archiver != nil && archiver.Status == repo_model.ArchiverReady { - download(ctx, aReq.GetArchiveName(), archiver) - return - } - } - } + download(ctx, aReq.GetArchiveName(), archiver) } func download(ctx *context.Context, archiveName string, archiver *repo_model.RepoArchiver) { downloadName := ctx.Repo.Repository.Name + "-" + archiveName - rPath, err := archiver.RelativePath() - if err != nil { - ctx.ServerError("archiver.RelativePath", err) - return - } - + rPath := archiver.RelativePath() if setting.RepoArchive.ServeDirect { // If we have a signed url (S3, object storage), redirect to this directly. u, err := storage.RepoArchives.URL(rPath, downloadName) diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index ebd3eaf236ab9..ae43503bae125 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -57,6 +57,21 @@ func (ErrUnknownArchiveFormat) Is(err error) bool { return ok } +// RepoRefNotFoundError is returned when a requested reference (commit, tag) was not found. +type RepoRefNotFoundError struct { + RefName string +} + +// Error implements error. +func (e RepoRefNotFoundError) Error() string { + return fmt.Sprintf("unrecognized repository reference: %s", e.RefName) +} + +func (e RepoRefNotFoundError) Is(err error) bool { + _, ok := err.(RepoRefNotFoundError) + return ok +} + // NewRequest creates an archival request, based on the URI. The // resulting ArchiveRequest is suitable for being passed to ArchiveRepository() // if it's determined that the request still needs to be satisfied. @@ -103,7 +118,7 @@ func NewRequest(repoID int64, repo *git.Repository, uri string) (*ArchiveRequest } } } else { - return nil, fmt.Errorf("Unknow ref %s type", r.refName) + return nil, RepoRefNotFoundError{RefName: r.refName} } return r, nil @@ -115,6 +130,49 @@ func (aReq *ArchiveRequest) GetArchiveName() string { return strings.ReplaceAll(aReq.refName, "/", "-") + "." + aReq.Type.String() } +// Await awaits the completion of an ArchiveRequest. If the archive has +// already been prepared the method returns immediately. Otherwise an archiver +// process will be started and its completion awaited. On success the returned +// RepoArchiver may be used to download the archive. Note that even if the +// context is cancelled/times out a started archiver will still continue to run +// in the background. +func (aReq *ArchiveRequest) Await(ctx context.Context) (*repo_model.RepoArchiver, error) { + archiver, err := repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + if err != nil { + return nil, fmt.Errorf("models.GetRepoArchiver: %v", err) + } + + if archiver != nil && archiver.Status == repo_model.ArchiverReady { + // Archive already generated, we're done. + return archiver, nil + } + + if err := StartArchive(aReq); err != nil { + return nil, fmt.Errorf("archiver.StartArchive: %v", err) + } + + poll := time.NewTicker(time.Second * 1) + defer poll.Stop() + + for { + select { + case <-graceful.GetManager().HammerContext().Done(): + // System stopped. + return nil, graceful.GetManager().HammerContext().Err() + case <-ctx.Done(): + return nil, ctx.Err() + case <-poll.C: + archiver, err = repo_model.GetRepoArchiver(ctx, aReq.RepoID, aReq.Type, aReq.CommitID) + if err != nil { + return nil, fmt.Errorf("repo_model.GetRepoArchiver: %v", err) + } + if archiver != nil && archiver.Status == repo_model.ArchiverReady { + return archiver, nil + } + } + } +} + func doArchive(r *ArchiveRequest) (*repo_model.RepoArchiver, error) { txCtx, committer, err := db.TxContext() if err != nil { @@ -147,11 +205,7 @@ func doArchive(r *ArchiveRequest) (*repo_model.RepoArchiver, error) { } } - rPath, err := archiver.RelativePath() - if err != nil { - return nil, err - } - + rPath := archiver.RelativePath() _, err = storage.RepoArchives.Stat(rPath) if err == nil { if archiver.Status == repo_model.ArchiverGenerating { @@ -284,13 +338,10 @@ func StartArchive(request *ArchiveRequest) error { } func deleteOldRepoArchiver(ctx context.Context, archiver *repo_model.RepoArchiver) error { - p, err := archiver.RelativePath() - if err != nil { - return err - } if err := repo_model.DeleteRepoArchiver(ctx, archiver); err != nil { return err } + p := archiver.RelativePath() if err := storage.RepoArchives.Delete(p); err != nil { log.Error("delete repo archive file failed: %v", err) } diff --git a/templates/admin/packages/list.tmpl b/templates/admin/packages/list.tmpl index 61721532a441b..06d6163476a9f 100644 --- a/templates/admin/packages/list.tmpl +++ b/templates/admin/packages/list.tmpl @@ -24,6 +24,7 @@ + diff --git a/templates/package/content/vagrant.tmpl b/templates/package/content/vagrant.tmpl new file mode 100644 index 0000000000000..7bd686e6ab689 --- /dev/null +++ b/templates/package/content/vagrant.tmpl @@ -0,0 +1,18 @@ +{{if eq .PackageDescriptor.Package.Type "vagrant"}} +

{{.locale.Tr "packages.installation"}}

+
+
+
+ + 
vagrant box add --box-version {{.PackageDescriptor.Version.Version}} "{{AppUrl}}api/packages/{{.PackageDescriptor.Owner.Name}}/vagrant/{{.PackageDescriptor.Package.Name}}"
+
+
+ +
+
+
+ {{if .PackageDescriptor.Metadata.Description}} +

{{.locale.Tr "packages.about"}}

+
{{.PackageDescriptor.Metadata.Description}}
+ {{end}} +{{end}} diff --git a/templates/package/metadata/vagrant.tmpl b/templates/package/metadata/vagrant.tmpl new file mode 100644 index 0000000000000..344e417b77f99 --- /dev/null +++ b/templates/package/metadata/vagrant.tmpl @@ -0,0 +1,5 @@ +{{if eq .PackageDescriptor.Package.Type "vagrant"}} + {{if .PackageDescriptor.Metadata.Author}}
{{svg "octicon-person" 16 "mr-3"}} {{.PackageDescriptor.Metadata.Author}}
{{end}} + {{if .PackageDescriptor.Metadata.ProjectURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.details.project_site"}}
{{end}} + {{if .PackageDescriptor.Metadata.RepositoryURL}}
{{svg "octicon-link-external" 16 "mr-3"}} {{.locale.Tr "packages.conan.details.repository"}}
{{end}} +{{end}} diff --git a/templates/package/shared/list.tmpl b/templates/package/shared/list.tmpl index 020acba9d73a5..b48f772f39fd2 100644 --- a/templates/package/shared/list.tmpl +++ b/templates/package/shared/list.tmpl @@ -17,6 +17,7 @@ + diff --git a/templates/package/view.tmpl b/templates/package/view.tmpl index 254d1ff840ea1..a5b2a2ef68a39 100644 --- a/templates/package/view.tmpl +++ b/templates/package/view.tmpl @@ -30,6 +30,7 @@ {{template "package/content/pub" .}} {{template "package/content/pypi" .}} {{template "package/content/rubygems" .}} + {{template "package/content/vagrant" .}}
@@ -52,6 +53,7 @@ {{template "package/metadata/pub" .}} {{template "package/metadata/pypi" .}} {{template "package/metadata/rubygems" .}} + {{template "package/metadata/vagrant" .}}
{{svg "octicon-database" 16 "mr-3"}} {{FileSize .PackageDescriptor.CalculateBlobSize}}
{{if not (eq .PackageDescriptor.Package.Type "container")}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 8a8ca9a924fe6..5dcb4787e0b5f 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -1962,7 +1962,8 @@ "nuget", "pub", "pypi", - "rubygems" + "rubygems", + "vagrant" ], "type": "string", "description": "package type filter", diff --git a/web_src/svg/gitea-vagrant.svg b/web_src/svg/gitea-vagrant.svg new file mode 100644 index 0000000000000..37bc1673cb4a8 --- /dev/null +++ b/web_src/svg/gitea-vagrant.svg @@ -0,0 +1,6 @@ + + + + + +