From 7527c759eb29cde08cf2ff26fd1e2c5cd0e0034a Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Mon, 15 Feb 2021 19:17:19 +0200 Subject: [PATCH 1/5] Add "get metarange", "get range" API endpoints Returns data (currently just the address of the object) by ID. --- api/controller.go | 50 +++++ catalog/cataloger.go | 6 + catalog/entry_catalog.go | 9 + catalog/fake_graveler_test.go | 8 + catalog/rocks_cataloger.go | 8 + docs/assets/js/swagger.yml | 231 +++++++++++------------ graveler/committed/manager.go | 10 + graveler/committed/meta_range.go | 8 + graveler/committed/meta_range_manager.go | 8 + graveler/committed/range.go | 1 + graveler/committed/range_manager.go | 4 + graveler/graveler.go | 49 ++++- graveler/sstable/range_manager.go | 4 + graveler/testutil/fakes.go | 12 ++ pyramid/pyramid.go | 4 + pyramid/tier_fs.go | 4 + swagger.yml | 54 ++++++ 17 files changed, 345 insertions(+), 125 deletions(-) diff --git a/api/controller.go b/api/controller.go index bb6b5630c27..d34822675f8 100644 --- a/api/controller.go +++ b/api/controller.go @@ -1281,6 +1281,56 @@ func writeSymlinkToS3(params metadata.CreateSymlinkParams, repo *catalog.Reposit return err } +func (c *Controller) MetadataGetMetarangeHandler() metadata.GetMetaRangeHandler { + return metadata.GetMetaRangeHandlerFunc(func(params metadata.GetMetaRangeParams, user *models.User) middleware.Responder { + deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ + { + // TODO(oz): May want tighter permissions here. + Action: permissions.ReadRepositoryAction, + Resource: permissions.RepoArn(params.Repository), + }, + }) + if err != nil { + return metadata.NewGetMetaRangeUnauthorized().WithPayload(responseErrorFrom(err)) + } + deps.LogAction("metadata_get_metarange") + cataloger := deps.Cataloger + + metarange, err := cataloger.GetMetaRange(deps.ctx, params.Repository, params.MetaRange) + if err != nil { + return metadata.NewGetMetaRangeDefault(http.StatusInternalServerError).WithPayload(responseErrorFrom(err)) + } + ret := metadata.NewGetMetaRangeOK() + ret.Location = metarange.Address + return ret + }) +} + +func (c *Controller) MetadataGetRangeHandler() metadata.GetRangeHandler { + return metadata.GetRangeHandlerFunc(func(params metadata.GetRangeParams, user *models.User) middleware.Responder { + deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ + { + // TODO(oz): May want tighter permissions here. + Action: permissions.ReadRepositoryAction, + Resource: permissions.RepoArn(params.Repository), + }, + }) + if err != nil { + return metadata.NewGetRangeUnauthorized().WithPayload(responseErrorFrom(err)) + } + deps.LogAction("metadata_get_range") + cataloger := deps.Cataloger + + rng, err := cataloger.GetRange(deps.ctx, params.Repository, params.Range) + if err != nil { + return metadata.NewGetRangeDefault(http.StatusInternalServerError).WithPayload(responseErrorFrom(err)) + } + ret := metadata.NewGetRangeOK() + ret.Location = rng.Address + return ret + }) +} + func (c *Controller) ObjectsListObjectsHandler() objects.ListObjectsHandler { return objects.ListObjectsHandlerFunc(func(params objects.ListObjectsParams, user *models.User) middleware.Responder { deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ diff --git a/catalog/cataloger.go b/catalog/cataloger.go index 0d1f3af595f..7b775c8a3fa 100644 --- a/catalog/cataloger.go +++ b/catalog/cataloger.go @@ -3,6 +3,8 @@ package catalog import ( "context" "io" + + "github.com/treeverse/lakefs/graveler" ) const ( @@ -108,5 +110,9 @@ type Cataloger interface { LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string) error LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error + // forward metadata for thick clients + GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeData, error) + GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeData, error) + io.Closer } diff --git a/catalog/entry_catalog.go b/catalog/entry_catalog.go index 9f67900edda..69518f23411 100644 --- a/catalog/entry_catalog.go +++ b/catalog/entry_catalog.go @@ -79,6 +79,7 @@ type Store interface { graveler.KeyValueStore graveler.VersionController graveler.Dumper + graveler.Plumbing graveler.Loader } @@ -613,3 +614,11 @@ func (e *EntryCatalog) PreMergeHook(ctx context.Context, eventID uuid.UUID, repo func (e *EntryCatalog) PostMergeHook(ctx context.Context, eventID uuid.UUID, repositoryRecord graveler.RepositoryRecord, destination graveler.BranchID, source graveler.Ref, commitRecord graveler.CommitRecord) error { return nil } + +func (e *EntryCatalog) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeData, error) { + return e.Store.GetMetaRange(ctx, repositoryID, metaRangeID) +} + +func (e *EntryCatalog) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeData, error) { + return e.Store.GetRange(ctx, repositoryID, rangeID) +} diff --git a/catalog/fake_graveler_test.go b/catalog/fake_graveler_test.go index be2d212a26d..29d6e1d1e10 100644 --- a/catalog/fake_graveler_test.go +++ b/catalog/fake_graveler_test.go @@ -47,6 +47,14 @@ func (g *FakeGraveler) DumpTags(ctx context.Context, repositoryID graveler.Repos panic("implement me") } +func (g *FakeGraveler) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeData, error) { + panic("implement me") +} + +func (g *FakeGraveler) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeData, error) { + panic("implement me") +} + func fakeGravelerBuildKey(repositoryID graveler.RepositoryID, ref graveler.Ref, key graveler.Key) string { return strings.Join([]string{repositoryID.String(), ref.String(), key.String()}, "/") } diff --git a/catalog/rocks_cataloger.go b/catalog/rocks_cataloger.go index f4487651ad7..a695ee80d67 100644 --- a/catalog/rocks_cataloger.go +++ b/catalog/rocks_cataloger.go @@ -681,6 +681,14 @@ func (c *cataloger) LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID return c.EntryCatalog.LoadTags(ctx, graveler.RepositoryID(repositoryID), graveler.MetaRangeID(tagsMetaRangeID)) } +func (c *cataloger) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeData, error) { + return c.EntryCatalog.GetMetaRange(ctx, graveler.RepositoryID(repositoryID), graveler.MetaRangeID(metaRangeID)) +} + +func (c *cataloger) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeData, error) { + return c.EntryCatalog.GetRange(ctx, graveler.RepositoryID(repositoryID), graveler.RangeID(rangeID)) +} + func (c *cataloger) Close() error { return nil } diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index 5230b1e5169..73b34fb4fc1 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -22,10 +22,9 @@ securityDefinitions: in: header name: X-JWT-Authorization - security: - - basic_auth: [ ] - - jwt_token: [ ] + - jwt_token: [] + - basic_auth: [] responses: Unauthorized: @@ -233,6 +232,16 @@ definitions: ref: type: string + refs_dump: + type: object + properties: + commits_meta_range_id: + type: string + tags_meta_range_id: + type: string + branches_meta_range_id: + type: string + error: type: object properties: @@ -357,38 +366,6 @@ definitions: - id - statement - continuous_export_configuration: - type: object - required: - - exportPath - properties: - exportPath: - type: string - format: uri - x-nullable: false # Override https://github.com/go-swagger/go-swagger/issues/1188 - # go-swagger totally not a bug. This causes the generated field - # *not* to be a pointer. Then the regular (incorrect, in this - # case) JSON parser parses it as an empty field, and validation - # verifies the value is non-empty. In *this particular case* it - # works because a URI cannot be empty (at least not an absolute - # URI, which is what we require). - description: export objects to this path - example: s3://company-bucket/path/to/export - exportStatusPath: - type: string - format: uri - description: write export status object to this path - example: s3://company-bucket/path/to/status - lastKeysInPrefixRegexp: - type: array - items: - type: string - description: "list of regexps of keys to exported last in each prefix (for signalling)" - example: [ "^SUCCESS$", ".*/_SUCCESS$" ] - isContinuous: - type: boolean - description: if true, export every commit or merge to branch - config: type: object properties: @@ -1216,6 +1193,11 @@ paths: name: repository schema: $ref: "#/definitions/repository_creation" + - in: query + name: bare + type: boolean + default: false + description: If true, create a bare repository with no initial commit and branch responses: 201: description: repository @@ -1277,6 +1259,70 @@ paths: schema: $ref: "#/definitions/error" + /repositories/{repository}/refs/dump: + parameters: + - in: path + name: repository + required: true + type: string + put: + tags: + - refs + operationId: dump + summary: Dump repository refs (tags, commits, branches) to object store + responses: + 201: + description: refs dump + schema: + type: object + $ref: "#/definitions/refs_dump" + 400: + description: validation error + schema: + $ref: "#/definitions/error" + 401: + $ref: "#/responses/Unauthorized" + 404: + description: repository not found + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + /repositories/{repository}/refs/restore: + parameters: + - in: path + name: repository + required: true + type: string + put: + tags: + - refs + operationId: restore + summary: Restore repository refs (tags, commits, branches) from object store + parameters: + - in: body + name: manifest + schema: + $ref: "#/definitions/refs_dump" + responses: + 200: + description: refs successfully loaded + 400: + description: validation error + schema: + $ref: "#/definitions/error" + 401: + $ref: "#/responses/Unauthorized" + 404: + description: repository not found + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" /repositories/{repository}/tags: parameters: - in: path @@ -1627,6 +1673,9 @@ paths: ref: type: string description: the commit to revert, given by a ref + parent_number: + type: integer + description: when reverting a merge commit, the parent number (starting from 1) relative to which to perform the revert. responses: 204: description: revert successful @@ -1641,7 +1690,7 @@ paths: schema: $ref: "#/definitions/error" - /repositories/{repository}/refs/{sourceRef}/merge/{destinationRef}: + /repositories/{repository}/refs/{sourceRef}/merge/{destinationBranch}: parameters: - in: path name: repository @@ -1651,9 +1700,9 @@ paths: name: sourceRef required: true type: string - description: source branch name + description: source ref - in: path - name: destinationRef + name: destinationBranch required: true type: string description: destination branch name @@ -2105,121 +2154,59 @@ paths: schema: $ref: "#/definitions/error" - /repositories/{repository}/branches/{branch}/continuous-export: + /repositories/{repository}/metadata/meta_range/{meta_range}: parameters: - in: path name: repository required: true type: string - in: path - name: branch + name: meta_range required: true type: string get: tags: - - export - - branches - operationId: getContinuousExport - summary: returns the current continuous export configuration of a branch + - metadata + operationId: getMetaRange + summary: return URI to a meta-range file + # TODO(ariels): Actually redirect to S3. responses: 200: - description: continuous export policy - schema: - $ref: "#/definitions/continuous_export_configuration" - 401: - $ref: "#/responses/Unauthorized" - 404: - description: no continuous export policy defined - schema: - $ref: "#/definitions/error" - default: - description: generic error response - schema: - $ref: "#/definitions/error" - put: - tags: - - export - - branches - operationId: setContinuousExport - summary: sets a new continuous export configuration of a branch - parameters: - - in: body - name: config - required: true - schema: - $ref: "#/definitions/continuous_export_configuration" - responses: - 201: - description: continuous export successfullyconfigured - 401: - $ref: "#/responses/Unauthorized" - 404: - description: no branch defined at that repo - schema: - $ref: "#/definitions/error" - default: - description: generic error response - schema: - $ref: "#/definitions/error" - - /repositories/{repository}/branches/{branch}/repair-export: - parameters: - - in: path - name: repository - required: true - type: string - - in: path - name: branch - required: true - type: string - post: - tags: - - export - - branches - operationId: repair - summary: set continuous export state as repaired - responses: - 201: - description: continuous export status successfully changed to repaired + description: meta-range URI + headers: + Location: + type: string 401: $ref: "#/responses/Unauthorized" - 404: - description: no branch defined at that repo - schema: - $ref: "#/definitions/error" default: description: generic error response schema: $ref: "#/definitions/error" - /repositories/{repository}/branches/{branch}/export-hook: + /repositories/{repository}/metadata/range/{range}: parameters: - in: path name: repository required: true type: string - in: path - name: branch + name: range required: true type: string - post: + get: tags: - - export - - branches - operationId: run - summary: hook to be called in order to execute continuous export on branch + - metadata + operationId: getRange + summary: return URI to a range file + # TODO(ariels): Actually redirect to S3. responses: - 201: - description: continuous export successfully started - schema: - description: "export ID" - type: string + 200: + description: range URI + headers: + Location: + type: string 401: $ref: "#/responses/Unauthorized" - 404: - description: no branch defined at that repo - schema: - $ref: "#/definitions/error" default: description: generic error response schema: diff --git a/graveler/committed/manager.go b/graveler/committed/manager.go index 9e8ec1951b8..f94e43a995c 100644 --- a/graveler/committed/manager.go +++ b/graveler/committed/manager.go @@ -146,3 +146,13 @@ func (c *committedManager) Compare(ctx context.Context, ns graveler.StorageNames } return NewCompareIterator(diffIt, baseIt), nil } + +func (c *committedManager) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (graveler.MetaRangeData, error) { + uri, err := c.metaRangeManager.GetMetaRangeURI(ctx, ns, id) + return graveler.MetaRangeData{Address: uri}, err +} + +func (c *committedManager) GetRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (graveler.RangeData, error) { + uri, err := c.metaRangeManager.GetRangeURI(ctx, ns, id) + return graveler.RangeData{Address: uri}, err +} diff --git a/graveler/committed/meta_range.go b/graveler/committed/meta_range.go index d234f6bc94c..57bf60412c5 100644 --- a/graveler/committed/meta_range.go +++ b/graveler/committed/meta_range.go @@ -37,6 +37,14 @@ type MetaRangeManager interface { // NewMetaRangeIterator returns an Iterator over the MetaRange with id. NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (Iterator, error) + + // GetMetaRangeURI returns a URI with an object representing metarange ID. It may + // return a URI that does not resolve (rather than an error) if ID does not exist. + GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (string, error) + + // GetRangeURI returns a URI with an object representing metarange ID. It may + // return a URI that does not resolve (rather than an error) if ID does not exist. + GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (string, error) } // MetaRangeWriter is an abstraction for creating new MetaRanges diff --git a/graveler/committed/meta_range_manager.go b/graveler/committed/meta_range_manager.go index a841334d509..06521af1abd 100644 --- a/graveler/committed/meta_range_manager.go +++ b/graveler/committed/meta_range_manager.go @@ -95,3 +95,11 @@ func (m *metaRangeManager) NewMetaRangeIterator(ctx context.Context, ns graveler } return NewIterator(ctx, m.rangeManager, Namespace(ns), rangesIt), nil } + +func (m *metaRangeManager) GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (string, error) { + return m.metaManager.GetURI(ctx, Namespace(ns), ID(id)) +} + +func (m *metaRangeManager) GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (string, error) { + return m.rangeManager.GetURI(ctx, Namespace(ns), ID(id)) +} diff --git a/graveler/committed/range.go b/graveler/committed/range.go index d2868378188..a606ebbf896 100644 --- a/graveler/committed/range.go +++ b/graveler/committed/range.go @@ -3,6 +3,7 @@ package committed import "google.golang.org/protobuf/proto" // Range represents a range of sorted Keys + type Range struct { ID ID MinKey Key diff --git a/graveler/committed/range_manager.go b/graveler/committed/range_manager.go index 97d6e19991d..070d0b7ce05 100644 --- a/graveler/committed/range_manager.go +++ b/graveler/committed/range_manager.go @@ -59,6 +59,10 @@ type RangeManager interface { // GetWriter returns a new Range writer instance GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error) + + // GetURI returns a URI from which to read the contents of id. If id does not exist + // it may return a URI that resolves nowhere rather than an error. + GetURI(ctx context.Context, ns Namespace, id ID) (string, error) } // WriteResult is the result of a completed write of a Range diff --git a/graveler/graveler.go b/graveler/graveler.go index 173214912c3..e7ab067f4d6 100644 --- a/graveler/graveler.go +++ b/graveler/graveler.go @@ -46,6 +46,16 @@ type Reference interface { CommitID() CommitID } +type MetaRangeData struct { + // URI of metarange file. + Address string +} + +type RangeData struct { + // URI of range file. + Address string +} + // function/methods receiving the following basic types could assume they passed validation // StorageNamespace is the URI to the storage location @@ -74,6 +84,9 @@ type CommitID string // MetaRangeID represents a snapshot of the MetaRange, referenced by a commit type MetaRangeID string +// RangeID represents a part of a MetaRange, useful only for plumbing. +type RangeID string + // StagingToken represents a namespace for writes to apply as uncommitted type StagingToken string @@ -321,6 +334,15 @@ type VersionController interface { SetHooksHandler(handler HooksHandler) } +// Plumbing includes commands for fiddling more directly with graveler implementation +// internals. +type Plumbing interface { + // GetMetarange returns information where metarangeID is stored. + GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeData, error) + // GetRange returns information where rangeID is stored. + GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeData, error) +} + type Dumper interface { // DumpCommits iterates through all commits and dumps them in Graveler format DumpCommits(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error) @@ -333,13 +355,13 @@ type Dumper interface { } type Loader interface { - // DumpCommits iterates through all commits in Graveler format and loads them into repositoryID + // LoadCommits iterates through all commits in Graveler format and loads them into repositoryID LoadCommits(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error - // DumpBranches iterates through all branches in Graveler format and loads them into repositoryID + // LoadBranches iterates through all branches in Graveler format and loads them into repositoryID LoadBranches(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error - // DumpTags iterates through all tags in Graveler format and loads them into repositoryID + // LoadTags iterates through all tags in Graveler format and loads them into repositoryID LoadTags(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error } @@ -505,6 +527,11 @@ type CommittedManager interface { // A change is either an entity to write/overwrite, or a tombstone to mark a deletion // it returns a new MetaRangeID that is expected to be immediately addressable Apply(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID, iterator ValueIterator) (MetaRangeID, DiffSummary, error) + + // GetMetarange returns information where metarangeID is stored. + GetMetaRange(ctx context.Context, ns StorageNamespace, metaRangeID MetaRangeID) (MetaRangeData, error) + // GetRange returns information where rangeID is stored. + GetRange(ctx context.Context, ns StorageNamespace, rangeID RangeID) (RangeData, error) } // StagingManager manages entries in a staging area, denoted by a staging token @@ -1513,6 +1540,22 @@ func (g *Graveler) LoadTags(ctx context.Context, repositoryID RepositoryID, meta return nil } +func (g *Graveler) GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeData, error) { + repo, err := g.RefManager.GetRepository(ctx, repositoryID) + if err != nil { + return MetaRangeData{}, nil + } + return g.CommittedManager.GetMetaRange(ctx, repo.StorageNamespace, metaRangeID) +} + +func (g *Graveler) GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeData, error) { + repo, err := g.RefManager.GetRepository(ctx, repositoryID) + if err != nil { + return RangeData{}, nil + } + return g.CommittedManager.GetRange(ctx, repo.StorageNamespace, rangeID) +} + func (g *Graveler) DumpCommits(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error) { repo, err := g.GetRepository(ctx, repositoryID) if err != nil { diff --git a/graveler/sstable/range_manager.go b/graveler/sstable/range_manager.go index 4171c3215e5..04f875fb7c5 100644 --- a/graveler/sstable/range_manager.go +++ b/graveler/sstable/range_manager.go @@ -170,6 +170,10 @@ func (m *RangeManager) GetWriter(ctx context.Context, ns committed.Namespace, me return NewDiskWriter(ctx, m.fs, ns, m.hash.New(), metadata) } +func (m *RangeManager) GetURI(ctx context.Context, ns committed.Namespace, id committed.ID) (string, error) { + return m.fs.GetRemoteURI(ctx, string(ns), string(id)) +} + func (m *RangeManager) execAndLog(ctx context.Context, f func() error, msg string) { if err := f(); err != nil { logging.FromContext(ctx).WithError(err).Error(msg) diff --git a/graveler/testutil/fakes.go b/graveler/testutil/fakes.go index 6254ba251f3..ed126b2c263 100644 --- a/graveler/testutil/fakes.go +++ b/graveler/testutil/fakes.go @@ -97,6 +97,18 @@ func (c *CommittedFake) WriteMetaRange(ctx context.Context, ns graveler.StorageN return &c.MetaRangeID, nil } +func (c *CommittedFake) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeData, error) { + return graveler.MetaRangeData{ + Address: fmt.Sprintf("fake://prefix/%s(metarange)", metaRangeID), + }, nil +} + +func (c *CommittedFake) GetRange(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (graveler.RangeData, error) { + return graveler.RangeData{ + Address: fmt.Sprintf("fake://prefix/%s(range)", rangeID), + }, nil +} + type StagingFake struct { Err error DropErr error // specific error for drop call diff --git a/pyramid/pyramid.go b/pyramid/pyramid.go index 7c8482c61e1..283d5e7d5a8 100644 --- a/pyramid/pyramid.go +++ b/pyramid/pyramid.go @@ -22,6 +22,10 @@ type FS interface { // Exists returns true if filename currently exists on block storage. Exists(ctx context.Context, namespace, filename string) (bool, error) + + // GetRemoteURI returns the URI for filename on block storage. That URI might not + // resolve if filename does not exist. + GetRemoteURI(ctx context.Context, namespace, filename string) (string, error) } // File is pyramid abstraction for an os.File diff --git a/pyramid/tier_fs.go b/pyramid/tier_fs.go index 87a16bd5c5b..57e3a6bbb2d 100644 --- a/pyramid/tier_fs.go +++ b/pyramid/tier_fs.go @@ -170,6 +170,10 @@ func (tfs *TierFS) store(ctx context.Context, namespace, originalPath, nsPath, f } } +func (tfs *TierFS) GetRemoteURI(_ context.Context, _, filename string) (string, error) { + return tfs.blockStoragePath(filename), nil +} + // Create creates a new file in TierFS. File isn't stored in TierFS until a successful close // operation. Open(namespace, filename) calls will return an error before the close was // called. Create only performs local operations so it ignores the context. diff --git a/swagger.yml b/swagger.yml index d4c4c6910e5..034b65a153b 100644 --- a/swagger.yml +++ b/swagger.yml @@ -2364,6 +2364,60 @@ paths: description: output not found schema: $ref: "#/definitions/error" + + /repositories/{repository}/metadata/meta_range/{meta_range}: + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: meta_range + required: true + type: string + get: + tags: + - metadata + operationId: getMetaRange + summary: return URI to a meta-range file + # TODO(ariels): Actually redirect to S3. + responses: + 200: + description: meta-range URI + headers: + Location: + type: string + 401: + $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + + /repositories/{repository}/metadata/range/{range}: + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: range + required: true + type: string + get: + tags: + - metadata + operationId: getRange + summary: return URI to a range file + # TODO(ariels): Actually redirect to S3. + responses: + 200: + description: range URI + headers: + Location: + type: string + 401: + $ref: "#/responses/Unauthorized" default: description: generic error response schema: From 392814590eecb13ac69769d2f7e43b027c52b4e8 Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Wed, 17 Feb 2021 11:46:35 +0200 Subject: [PATCH 2/5] [CR] Change "{Meta,}RangeData" to "{Meta,}RangeInfo" --- catalog/cataloger.go | 4 ++-- catalog/entry_catalog.go | 4 ++-- catalog/fake_graveler_test.go | 4 ++-- catalog/rocks_cataloger.go | 4 ++-- graveler/committed/manager.go | 8 ++++---- graveler/graveler.go | 20 ++++++++++---------- graveler/testutil/fakes.go | 8 ++++---- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/catalog/cataloger.go b/catalog/cataloger.go index 7b775c8a3fa..36d6b852a82 100644 --- a/catalog/cataloger.go +++ b/catalog/cataloger.go @@ -111,8 +111,8 @@ type Cataloger interface { LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error // forward metadata for thick clients - GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeData, error) - GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeData, error) + GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeInfo, error) + GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeInfo, error) io.Closer } diff --git a/catalog/entry_catalog.go b/catalog/entry_catalog.go index 69518f23411..012281cf8e4 100644 --- a/catalog/entry_catalog.go +++ b/catalog/entry_catalog.go @@ -615,10 +615,10 @@ func (e *EntryCatalog) PostMergeHook(ctx context.Context, eventID uuid.UUID, rep return nil } -func (e *EntryCatalog) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeData, error) { +func (e *EntryCatalog) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { return e.Store.GetMetaRange(ctx, repositoryID, metaRangeID) } -func (e *EntryCatalog) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeData, error) { +func (e *EntryCatalog) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeInfo, error) { return e.Store.GetRange(ctx, repositoryID, rangeID) } diff --git a/catalog/fake_graveler_test.go b/catalog/fake_graveler_test.go index 29d6e1d1e10..e4ab0b29f45 100644 --- a/catalog/fake_graveler_test.go +++ b/catalog/fake_graveler_test.go @@ -47,11 +47,11 @@ func (g *FakeGraveler) DumpTags(ctx context.Context, repositoryID graveler.Repos panic("implement me") } -func (g *FakeGraveler) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeData, error) { +func (g *FakeGraveler) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { panic("implement me") } -func (g *FakeGraveler) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeData, error) { +func (g *FakeGraveler) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeInfo, error) { panic("implement me") } diff --git a/catalog/rocks_cataloger.go b/catalog/rocks_cataloger.go index a695ee80d67..bf0f20f55e0 100644 --- a/catalog/rocks_cataloger.go +++ b/catalog/rocks_cataloger.go @@ -681,11 +681,11 @@ func (c *cataloger) LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID return c.EntryCatalog.LoadTags(ctx, graveler.RepositoryID(repositoryID), graveler.MetaRangeID(tagsMetaRangeID)) } -func (c *cataloger) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeData, error) { +func (c *cataloger) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeInfo, error) { return c.EntryCatalog.GetMetaRange(ctx, graveler.RepositoryID(repositoryID), graveler.MetaRangeID(metaRangeID)) } -func (c *cataloger) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeData, error) { +func (c *cataloger) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeInfo, error) { return c.EntryCatalog.GetRange(ctx, graveler.RepositoryID(repositoryID), graveler.RangeID(rangeID)) } diff --git a/graveler/committed/manager.go b/graveler/committed/manager.go index f94e43a995c..21b4cc24851 100644 --- a/graveler/committed/manager.go +++ b/graveler/committed/manager.go @@ -147,12 +147,12 @@ func (c *committedManager) Compare(ctx context.Context, ns graveler.StorageNames return NewCompareIterator(diffIt, baseIt), nil } -func (c *committedManager) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (graveler.MetaRangeData, error) { +func (c *committedManager) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { uri, err := c.metaRangeManager.GetMetaRangeURI(ctx, ns, id) - return graveler.MetaRangeData{Address: uri}, err + return graveler.MetaRangeInfo{Address: uri}, err } -func (c *committedManager) GetRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (graveler.RangeData, error) { +func (c *committedManager) GetRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (graveler.RangeInfo, error) { uri, err := c.metaRangeManager.GetRangeURI(ctx, ns, id) - return graveler.RangeData{Address: uri}, err + return graveler.RangeInfo{Address: uri}, err } diff --git a/graveler/graveler.go b/graveler/graveler.go index e7ab067f4d6..1f851139313 100644 --- a/graveler/graveler.go +++ b/graveler/graveler.go @@ -46,12 +46,12 @@ type Reference interface { CommitID() CommitID } -type MetaRangeData struct { +type MetaRangeInfo struct { // URI of metarange file. Address string } -type RangeData struct { +type RangeInfo struct { // URI of range file. Address string } @@ -338,9 +338,9 @@ type VersionController interface { // internals. type Plumbing interface { // GetMetarange returns information where metarangeID is stored. - GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeData, error) + GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeInfo, error) // GetRange returns information where rangeID is stored. - GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeData, error) + GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeInfo, error) } type Dumper interface { @@ -529,9 +529,9 @@ type CommittedManager interface { Apply(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID, iterator ValueIterator) (MetaRangeID, DiffSummary, error) // GetMetarange returns information where metarangeID is stored. - GetMetaRange(ctx context.Context, ns StorageNamespace, metaRangeID MetaRangeID) (MetaRangeData, error) + GetMetaRange(ctx context.Context, ns StorageNamespace, metaRangeID MetaRangeID) (MetaRangeInfo, error) // GetRange returns information where rangeID is stored. - GetRange(ctx context.Context, ns StorageNamespace, rangeID RangeID) (RangeData, error) + GetRange(ctx context.Context, ns StorageNamespace, rangeID RangeID) (RangeInfo, error) } // StagingManager manages entries in a staging area, denoted by a staging token @@ -1540,18 +1540,18 @@ func (g *Graveler) LoadTags(ctx context.Context, repositoryID RepositoryID, meta return nil } -func (g *Graveler) GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeData, error) { +func (g *Graveler) GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeInfo, error) { repo, err := g.RefManager.GetRepository(ctx, repositoryID) if err != nil { - return MetaRangeData{}, nil + return MetaRangeInfo{}, nil } return g.CommittedManager.GetMetaRange(ctx, repo.StorageNamespace, metaRangeID) } -func (g *Graveler) GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeData, error) { +func (g *Graveler) GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeInfo, error) { repo, err := g.RefManager.GetRepository(ctx, repositoryID) if err != nil { - return RangeData{}, nil + return RangeInfo{}, nil } return g.CommittedManager.GetRange(ctx, repo.StorageNamespace, rangeID) } diff --git a/graveler/testutil/fakes.go b/graveler/testutil/fakes.go index ed126b2c263..e7a12ec4c5f 100644 --- a/graveler/testutil/fakes.go +++ b/graveler/testutil/fakes.go @@ -97,14 +97,14 @@ func (c *CommittedFake) WriteMetaRange(ctx context.Context, ns graveler.StorageN return &c.MetaRangeID, nil } -func (c *CommittedFake) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeData, error) { - return graveler.MetaRangeData{ +func (c *CommittedFake) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { + return graveler.MetaRangeInfo{ Address: fmt.Sprintf("fake://prefix/%s(metarange)", metaRangeID), }, nil } -func (c *CommittedFake) GetRange(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (graveler.RangeData, error) { - return graveler.RangeData{ +func (c *CommittedFake) GetRange(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (graveler.RangeInfo, error) { + return graveler.RangeInfo{ Address: fmt.Sprintf("fake://prefix/%s(range)", rangeID), }, nil } From f581e49ff9c1b0cd1d7397fe8180c92c5da062da Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Wed, 17 Feb 2021 12:13:52 +0200 Subject: [PATCH 3/5] [CR] Add "list objects" permission to read {meta,}range file URLs It really is a cross between "read repo" (which lets you see namespace on underlying storage) and "list objects" (which you will get, once you read all the files). So require both. --- api/controller.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/controller.go b/api/controller.go index d34822675f8..b8669b6fcb9 100644 --- a/api/controller.go +++ b/api/controller.go @@ -1285,7 +1285,10 @@ func (c *Controller) MetadataGetMetarangeHandler() metadata.GetMetaRangeHandler return metadata.GetMetaRangeHandlerFunc(func(params metadata.GetMetaRangeParams, user *models.User) middleware.Responder { deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ { - // TODO(oz): May want tighter permissions here. + Action: permissions.ListObjectsAction, + Resource: permissions.RepoArn(params.Repository), + }, + { Action: permissions.ReadRepositoryAction, Resource: permissions.RepoArn(params.Repository), }, @@ -1310,7 +1313,10 @@ func (c *Controller) MetadataGetRangeHandler() metadata.GetRangeHandler { return metadata.GetRangeHandlerFunc(func(params metadata.GetRangeParams, user *models.User) middleware.Responder { deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ { - // TODO(oz): May want tighter permissions here. + Action: permissions.ListObjectsAction, + Resource: permissions.RepoArn(params.Repository), + }, + { Action: permissions.ReadRepositoryAction, Resource: permissions.RepoArn(params.Repository), }, From dcef40c69a1a52135ae1856531f4ee229fb239ca Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Wed, 17 Feb 2021 13:17:09 +0200 Subject: [PATCH 4/5] make gen-ui Update The Other swagger.yml. --- docs/assets/js/swagger.yml | 211 +++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index 73b34fb4fc1..034b65a153b 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -372,6 +372,54 @@ definitions: blockstore.type: type: string + action_run: + type: object + required: + - run_id + - branch + - commit_id + properties: + run_id: + type: string + branch: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + event_type: + type: string + enum: [ pre_commit, pre_merge ] + status: + type: string + enum: [ failed, completed ] + commit_id: + type: string + + hook_run: + type: object + required: + - hook_id + properties: + hook_id: + type: string + hook_type: + type: string + enum: [webhook] + action: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + status: + type: string + enum: [ failed, completed ] + paths: /setup_lakefs: @@ -2154,6 +2202,169 @@ paths: schema: $ref: "#/definitions/error" + /repositories/{repository}/actions/runs: + get: + tags: + - actions + operationId: listRuns + summary: list runs + parameters: + - in: path + name: repository + required: true + type: string + - in: query + name: before + type: string + format: date-time + - in: query + name: amount + type: integer + default: 100 + - in: query + name: branch + type: string + responses: + 200: + description: list action runs + schema: + type: object + properties: + pagination: + $ref: "#/definitions/pagination" + results: + type: array + items: + $ref: "#/definitions/action_run" + 401: + $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + 404: + description: not found + schema: + $ref: "#/definitions/error" + + /repositories/{repository}/actions/runs/{run_id}: + get: + tags: + - actions + operationId: getRun + summary: get a run + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: run_id + required: true + type: string + responses: + 200: + description: action run result + schema: + $ref: "#/definitions/action_run" + 401: + $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + 404: + description: not found + schema: + $ref: "#/definitions/error" + + /repositories/{repository}/actions/runs/{run_id}/hooks: + get: + tags: + - actions + operationId: listRunHooks + summary: list run hooks + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: run_id + required: true + type: string + - in: query + name: before + type: string + format: date-time + - in: query + name: amount + type: integer + default: 100 + responses: + 200: + description: list specific run hooks + schema: + type: object + properties: + pagination: + $ref: "#/definitions/pagination" + results: + type: array + items: + $ref: "#/definitions/hook_run" + 401: + $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + 404: + description: not found + schema: + $ref: "#/definitions/error" + + /repositories/{repository}/actions/runs/{run_id}/hooks/{hook_id}/output: + get: + tags: + - actions + operationId: getRunHookOutput + summary: get run hook output + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: run_id + required: true + type: string + - in: path + name: hook_id + required: true + type: string + produces: + - application/octet-stream + responses: + 200: + description: run hook output + schema: + type: file + headers: + Content-Length: + type: integer + format: int64 + Last-Modified: + type: string + Content-Disposition: + type: string + 401: + $ref: "#/responses/Unauthorized" + 404: + description: output not found + schema: + $ref: "#/definitions/error" + /repositories/{repository}/metadata/meta_range/{meta_range}: parameters: - in: path From d05d64d0775caed5ea3626780cefac4489d8975f Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Wed, 17 Feb 2021 13:29:50 +0200 Subject: [PATCH 5/5] [bugfix] restore "default" stanza dropped during merge --- swagger.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/swagger.yml b/swagger.yml index 034b65a153b..ae367c5e165 100644 --- a/swagger.yml +++ b/swagger.yml @@ -2364,6 +2364,10 @@ paths: description: output not found schema: $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" /repositories/{repository}/metadata/meta_range/{meta_range}: parameters: