-
-
Notifications
You must be signed in to change notification settings - Fork 1.6k
feat(drivers): add cnb_releases #1033
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
17 commits
Select commit
Hold shift + click to select a range
22b4036
feat(drivers): add cnb_releases
xrgzs b12bfc7
feat(cnb_release): implement reference
xrgzs 5ec2738
refactor(cnb_releases): get release info by ID instead of tag name
xrgzs 1e58e9c
feat(cnb_releases): add option to use tag name instead of release name
xrgzs 3250724
fix(cnb_releases): set default root and improve release info retrieval
xrgzs 7cd1b91
feat(cnb_releases): implement Put
xrgzs 36ca665
perf(cnb_release): use io.Pipe to stream file upload
xrgzs 87e0e82
perf(cnb_releases): add context timeout for file upload request
xrgzs b690269
feat(cnb_releases): implement Remove
xrgzs 13dfdde
feat(cnb_releases): implement MakeDir
xrgzs 2bc6d2c
feat(cnb_releases): implement Rename
xrgzs 7935ee7
feat(cnb_releases): require repo and token in Addition
xrgzs b78aefc
chore(cnb_releases): remove unused code
xrgzs 2855be6
Revert 'perf(cnb_release): use io.Pipe to stream file upload'
xrgzs 46eea3d
perf(cnb_releases): optimize upload with MultiReader
xrgzs 4eef98a
feat(cnb_releases): add DefaultBranch
xrgzs 392f146
Merge branch 'main' into cnb
ILoveScratch2 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,230 @@ | ||
| package cnb_releases | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "fmt" | ||
| "io" | ||
| "mime/multipart" | ||
| "net/http" | ||
| "time" | ||
|
|
||
| "github.com/OpenListTeam/OpenList/v4/drivers/base" | ||
| "github.com/OpenListTeam/OpenList/v4/internal/driver" | ||
| "github.com/OpenListTeam/OpenList/v4/internal/errs" | ||
| "github.com/OpenListTeam/OpenList/v4/internal/model" | ||
| "github.com/OpenListTeam/OpenList/v4/pkg/utils" | ||
| "github.com/go-resty/resty/v2" | ||
| ) | ||
|
|
||
| type CnbReleases struct { | ||
| model.Storage | ||
| Addition | ||
| ref *CnbReleases | ||
| } | ||
|
|
||
| func (d *CnbReleases) Config() driver.Config { | ||
| return config | ||
| } | ||
|
|
||
| func (d *CnbReleases) GetAddition() driver.Additional { | ||
| return &d.Addition | ||
| } | ||
|
|
||
| func (d *CnbReleases) Init(ctx context.Context) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (d *CnbReleases) InitReference(storage driver.Driver) error { | ||
| refStorage, ok := storage.(*CnbReleases) | ||
| if ok { | ||
| d.ref = refStorage | ||
| return nil | ||
| } | ||
| return fmt.Errorf("ref: storage is not CnbReleases") | ||
| } | ||
|
|
||
| func (d *CnbReleases) Drop(ctx context.Context) error { | ||
| d.ref = nil | ||
| return nil | ||
| } | ||
|
|
||
| func (d *CnbReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { | ||
| if dir.GetPath() == "/" { | ||
| // get all releases for root dir | ||
| var resp ReleaseList | ||
|
|
||
| err := d.Request(http.MethodGet, "/{repo}/-/releases", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| }, &resp) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return utils.SliceConvert(resp, func(src Release) (model.Obj, error) { | ||
| name := src.Name | ||
| if d.UseTagName { | ||
| name = src.TagName | ||
| } | ||
| return &model.Object{ | ||
| ID: src.ID, | ||
| Name: name, | ||
| Size: d.sumAssetsSize(src.Assets), | ||
| Ctime: src.CreatedAt, | ||
| Modified: src.UpdatedAt, | ||
| IsFolder: true, | ||
| }, nil | ||
| }) | ||
| } else { | ||
| // get release info by release id | ||
| releaseID := dir.GetID() | ||
| if releaseID == "" { | ||
| return nil, errs.ObjectNotFound | ||
| } | ||
| var resp Release | ||
| err := d.Request(http.MethodGet, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| req.SetPathParam("release_id", releaseID) | ||
| }, &resp) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return utils.SliceConvert(resp.Assets, func(src ReleaseAsset) (model.Obj, error) { | ||
| return &Object{ | ||
| Object: model.Object{ | ||
| ID: src.ID, | ||
| Path: src.Path, | ||
| Name: src.Name, | ||
| Size: src.Size, | ||
| Ctime: src.CreatedAt, | ||
| Modified: src.UpdatedAt, | ||
| IsFolder: false, | ||
| }, | ||
| ParentID: dir.GetID(), | ||
| }, nil | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func (d *CnbReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { | ||
| return &model.Link{ | ||
| URL: "https://cnb.cool" + file.GetPath(), | ||
| }, nil | ||
| } | ||
|
|
||
| func (d *CnbReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { | ||
| if parentDir.GetPath() == "/" { | ||
| // create a new release | ||
| branch := d.DefaultBranch | ||
| if branch == "" { | ||
| branch = "main" // fallback to "main" if not set | ||
| } | ||
| return d.Request(http.MethodPost, "/{repo}/-/releases", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| req.SetBody(base.Json{ | ||
| "name": dirName, | ||
| "tag_name": dirName, | ||
| "target_commitish": branch, | ||
| }) | ||
| }, nil) | ||
| } | ||
| return errs.NotImplement | ||
| } | ||
|
|
||
| func (d *CnbReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { | ||
| return nil, errs.NotImplement | ||
| } | ||
|
|
||
| func (d *CnbReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) error { | ||
| if srcObj.IsDir() && !d.UseTagName { | ||
| return d.Request(http.MethodPatch, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| req.SetPathParam("release_id", srcObj.GetID()) | ||
| req.SetFormData(map[string]string{ | ||
| "name": newName, | ||
| }) | ||
| }, nil) | ||
| } | ||
| return errs.NotImplement | ||
| } | ||
|
|
||
| func (d *CnbReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { | ||
| return nil, errs.NotImplement | ||
| } | ||
|
|
||
| func (d *CnbReleases) Remove(ctx context.Context, obj model.Obj) error { | ||
| if obj.IsDir() { | ||
| return d.Request(http.MethodDelete, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| req.SetPathParam("release_id", obj.GetID()) | ||
| }, nil) | ||
| } | ||
| if o, ok := obj.(*Object); ok { | ||
| return d.Request(http.MethodDelete, "/{repo}/-/releases/{release_id}/assets/{asset_id}", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| req.SetPathParam("release_id", o.ParentID) | ||
| req.SetPathParam("asset_id", obj.GetID()) | ||
| }, nil) | ||
| } else { | ||
| return fmt.Errorf("unable to get release ID") | ||
| } | ||
| } | ||
|
|
||
| func (d *CnbReleases) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { | ||
| // 1. get upload info | ||
| var resp ReleaseAssetUploadURL | ||
| err := d.Request(http.MethodPost, "/{repo}/-/releases/{release_id}/asset-upload-url", func(req *resty.Request) { | ||
| req.SetPathParam("repo", d.Repo) | ||
| req.SetPathParam("release_id", dstDir.GetID()) | ||
| req.SetBody(base.Json{ | ||
| "asset_name": file.GetName(), | ||
| "overwrite": true, | ||
| "size": file.GetSize(), | ||
| }) | ||
| }, &resp) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // 2. upload file | ||
| // use multipart to create form file | ||
| var b bytes.Buffer | ||
| w := multipart.NewWriter(&b) | ||
| _, err = w.CreateFormFile("file", file.GetName()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| headSize := b.Len() | ||
| err = w.Close() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| head := bytes.NewReader(b.Bytes()[:headSize]) | ||
| tail := bytes.NewReader(b.Bytes()[headSize:]) | ||
| rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) | ||
|
|
||
| // use net/http to upload file | ||
| ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(resp.ExpiresInSec+1)*time.Second) | ||
xrgzs marked this conversation as resolved.
Show resolved
Hide resolved
xrgzs marked this conversation as resolved.
Show resolved
Hide resolved
xrgzs marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| defer cancel() | ||
| req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodPost, resp.UploadURL, rateLimitedRd) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| req.Header.Set("Content-Type", w.FormDataContentType()) | ||
| req.Header.Set("User-Agent", base.UserAgent) | ||
| httpResp, err := base.HttpClient.Do(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer httpResp.Body.Close() | ||
| if httpResp.StatusCode != http.StatusNoContent { | ||
| return fmt.Errorf("upload file failed: %s", httpResp.Status) | ||
| } | ||
|
|
||
| // 3. verify upload | ||
| return d.Request(http.MethodPost, resp.VerifyURL, nil, nil) | ||
| } | ||
|
|
||
| var _ driver.Driver = (*CnbReleases)(nil) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| package cnb_releases | ||
|
|
||
| import ( | ||
| "github.com/OpenListTeam/OpenList/v4/internal/driver" | ||
| "github.com/OpenListTeam/OpenList/v4/internal/op" | ||
| ) | ||
|
|
||
| type Addition struct { | ||
| driver.RootPath | ||
| Repo string `json:"repo" type:"string" required:"true"` | ||
| Token string `json:"token" type:"string" required:"true"` | ||
| UseTagName bool `json:"use_tag_name" type:"bool" default:"false" help:"Use tag name instead of release name"` | ||
| DefaultBranch string `json:"default_branch" type:"string" default:"main" help:"Default branch for new releases"` | ||
| } | ||
|
|
||
| var config = driver.Config{ | ||
| Name: "CNB Releases", | ||
| LocalSort: true, | ||
| DefaultRoot: "/", | ||
| } | ||
|
|
||
| func init() { | ||
| op.RegisterDriver(func() driver.Driver { | ||
| return &CnbReleases{} | ||
| }) | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| package cnb_releases | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/OpenListTeam/OpenList/v4/internal/model" | ||
| ) | ||
|
|
||
| type Object struct { | ||
| model.Object | ||
| ParentID string | ||
| } | ||
|
|
||
| type TagList []Tag | ||
|
|
||
| type Tag struct { | ||
| Commit struct { | ||
| Author UserInfo `json:"author"` | ||
| Commit CommitObject `json:"commit"` | ||
| Committer UserInfo `json:"committer"` | ||
| Parents []CommitParent `json:"parents"` | ||
| Sha string `json:"sha"` | ||
| } `json:"commit"` | ||
| Name string `json:"name"` | ||
| Target string `json:"target"` | ||
| TargetType string `json:"target_type"` | ||
| Verification TagObjectVerification `json:"verification"` | ||
| } | ||
|
|
||
| type UserInfo struct { | ||
| Freeze bool `json:"freeze"` | ||
| Nickname string `json:"nickname"` | ||
| Username string `json:"username"` | ||
| } | ||
|
|
||
| type CommitObject struct { | ||
| Author Signature `json:"author"` | ||
| CommentCount int `json:"comment_count"` | ||
| Committer Signature `json:"committer"` | ||
| Message string `json:"message"` | ||
| Tree CommitObjectTree `json:"tree"` | ||
| Verification CommitObjectVerification `json:"verification"` | ||
| } | ||
|
|
||
| type Signature struct { | ||
| Date time.Time `json:"date"` | ||
| Email string `json:"email"` | ||
| Name string `json:"name"` | ||
| } | ||
|
|
||
| type CommitObjectTree struct { | ||
| Sha string `json:"sha"` | ||
| } | ||
|
|
||
| type CommitObjectVerification struct { | ||
| Payload string `json:"payload"` | ||
| Reason string `json:"reason"` | ||
| Signature string `json:"signature"` | ||
| Verified bool `json:"verified"` | ||
| VerifiedAt string `json:"verified_at"` | ||
| } | ||
|
|
||
| type CommitParent = CommitObjectTree | ||
|
|
||
| type TagObjectVerification = CommitObjectVerification | ||
|
|
||
| type ReleaseList []Release | ||
|
|
||
| type Release struct { | ||
| Assets []ReleaseAsset `json:"assets"` | ||
| Author UserInfo `json:"author"` | ||
| Body string `json:"body"` | ||
| CreatedAt time.Time `json:"created_at"` | ||
| Draft bool `json:"draft"` | ||
| ID string `json:"id"` | ||
| IsLatest bool `json:"is_latest"` | ||
| Name string `json:"name"` | ||
| Prerelease bool `json:"prerelease"` | ||
| PublishedAt time.Time `json:"published_at"` | ||
| TagCommitish string `json:"tag_commitish"` | ||
| TagName string `json:"tag_name"` | ||
| UpdatedAt time.Time `json:"updated_at"` | ||
| } | ||
|
|
||
| type ReleaseAsset struct { | ||
| ContentType string `json:"content_type"` | ||
| CreatedAt time.Time `json:"created_at"` | ||
| ID string `json:"id"` | ||
| Name string `json:"name"` | ||
| Path string `json:"path"` | ||
| Size int64 `json:"size"` | ||
| UpdatedAt time.Time `json:"updated_at"` | ||
| Uploader UserInfo `json:"uploader"` | ||
| } | ||
|
|
||
| type ReleaseAssetUploadURL struct { | ||
| UploadURL string `json:"upload_url"` | ||
| ExpiresInSec int `json:"expires_in_sec"` | ||
| VerifyURL string `json:"verify_url"` | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.