diff --git a/api/controller.go b/api/controller.go index bb6b5630c27..b8669b6fcb9 100644 --- a/api/controller.go +++ b/api/controller.go @@ -1281,6 +1281,62 @@ func writeSymlinkToS3(params metadata.CreateSymlinkParams, repo *catalog.Reposit return err } +func (c *Controller) MetadataGetMetarangeHandler() metadata.GetMetaRangeHandler { + return metadata.GetMetaRangeHandlerFunc(func(params metadata.GetMetaRangeParams, user *models.User) middleware.Responder { + deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ + { + Action: permissions.ListObjectsAction, + Resource: permissions.RepoArn(params.Repository), + }, + { + Action: permissions.ReadRepositoryAction, + Resource: permissions.RepoArn(params.Repository), + }, + }) + if err != nil { + return metadata.NewGetMetaRangeUnauthorized().WithPayload(responseErrorFrom(err)) + } + deps.LogAction("metadata_get_metarange") + cataloger := deps.Cataloger + + metarange, err := cataloger.GetMetaRange(deps.ctx, params.Repository, params.MetaRange) + if err != nil { + return metadata.NewGetMetaRangeDefault(http.StatusInternalServerError).WithPayload(responseErrorFrom(err)) + } + ret := metadata.NewGetMetaRangeOK() + ret.Location = metarange.Address + return ret + }) +} + +func (c *Controller) MetadataGetRangeHandler() metadata.GetRangeHandler { + return metadata.GetRangeHandlerFunc(func(params metadata.GetRangeParams, user *models.User) middleware.Responder { + deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ + { + Action: permissions.ListObjectsAction, + Resource: permissions.RepoArn(params.Repository), + }, + { + Action: permissions.ReadRepositoryAction, + Resource: permissions.RepoArn(params.Repository), + }, + }) + if err != nil { + return metadata.NewGetRangeUnauthorized().WithPayload(responseErrorFrom(err)) + } + deps.LogAction("metadata_get_range") + cataloger := deps.Cataloger + + rng, err := cataloger.GetRange(deps.ctx, params.Repository, params.Range) + if err != nil { + return metadata.NewGetRangeDefault(http.StatusInternalServerError).WithPayload(responseErrorFrom(err)) + } + ret := metadata.NewGetRangeOK() + ret.Location = rng.Address + return ret + }) +} + func (c *Controller) ObjectsListObjectsHandler() objects.ListObjectsHandler { return objects.ListObjectsHandlerFunc(func(params objects.ListObjectsParams, user *models.User) middleware.Responder { deps, err := c.setupRequest(user, params.HTTPRequest, []permissions.Permission{ diff --git a/catalog/cataloger.go b/catalog/cataloger.go index 0d1f3af595f..36d6b852a82 100644 --- a/catalog/cataloger.go +++ b/catalog/cataloger.go @@ -3,6 +3,8 @@ package catalog import ( "context" "io" + + "github.com/treeverse/lakefs/graveler" ) const ( @@ -108,5 +110,9 @@ type Cataloger interface { LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string) error LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error + // forward metadata for thick clients + GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeInfo, error) + GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeInfo, error) + io.Closer } diff --git a/catalog/entry_catalog.go b/catalog/entry_catalog.go index 9f67900edda..012281cf8e4 100644 --- a/catalog/entry_catalog.go +++ b/catalog/entry_catalog.go @@ -79,6 +79,7 @@ type Store interface { graveler.KeyValueStore graveler.VersionController graveler.Dumper + graveler.Plumbing graveler.Loader } @@ -613,3 +614,11 @@ func (e *EntryCatalog) PreMergeHook(ctx context.Context, eventID uuid.UUID, repo func (e *EntryCatalog) PostMergeHook(ctx context.Context, eventID uuid.UUID, repositoryRecord graveler.RepositoryRecord, destination graveler.BranchID, source graveler.Ref, commitRecord graveler.CommitRecord) error { return nil } + +func (e *EntryCatalog) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { + return e.Store.GetMetaRange(ctx, repositoryID, metaRangeID) +} + +func (e *EntryCatalog) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeInfo, error) { + return e.Store.GetRange(ctx, repositoryID, rangeID) +} diff --git a/catalog/fake_graveler_test.go b/catalog/fake_graveler_test.go index be2d212a26d..e4ab0b29f45 100644 --- a/catalog/fake_graveler_test.go +++ b/catalog/fake_graveler_test.go @@ -47,6 +47,14 @@ func (g *FakeGraveler) DumpTags(ctx context.Context, repositoryID graveler.Repos panic("implement me") } +func (g *FakeGraveler) GetMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { + panic("implement me") +} + +func (g *FakeGraveler) GetRange(ctx context.Context, repositoryID graveler.RepositoryID, rangeID graveler.RangeID) (graveler.RangeInfo, error) { + panic("implement me") +} + func fakeGravelerBuildKey(repositoryID graveler.RepositoryID, ref graveler.Ref, key graveler.Key) string { return strings.Join([]string{repositoryID.String(), ref.String(), key.String()}, "/") } diff --git a/catalog/rocks_cataloger.go b/catalog/rocks_cataloger.go index f4487651ad7..bf0f20f55e0 100644 --- a/catalog/rocks_cataloger.go +++ b/catalog/rocks_cataloger.go @@ -681,6 +681,14 @@ func (c *cataloger) LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID return c.EntryCatalog.LoadTags(ctx, graveler.RepositoryID(repositoryID), graveler.MetaRangeID(tagsMetaRangeID)) } +func (c *cataloger) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeInfo, error) { + return c.EntryCatalog.GetMetaRange(ctx, graveler.RepositoryID(repositoryID), graveler.MetaRangeID(metaRangeID)) +} + +func (c *cataloger) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeInfo, error) { + return c.EntryCatalog.GetRange(ctx, graveler.RepositoryID(repositoryID), graveler.RangeID(rangeID)) +} + func (c *cataloger) Close() error { return nil } diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml index 5230b1e5169..034b65a153b 100644 --- a/docs/assets/js/swagger.yml +++ b/docs/assets/js/swagger.yml @@ -22,10 +22,9 @@ securityDefinitions: in: header name: X-JWT-Authorization - security: - - basic_auth: [ ] - - jwt_token: [ ] + - jwt_token: [] + - basic_auth: [] responses: Unauthorized: @@ -233,6 +232,16 @@ definitions: ref: type: string + refs_dump: + type: object + properties: + commits_meta_range_id: + type: string + tags_meta_range_id: + type: string + branches_meta_range_id: + type: string + error: type: object properties: @@ -357,43 +366,59 @@ definitions: - id - statement - continuous_export_configuration: + config: + type: object + properties: + blockstore.type: + type: string + + action_run: type: object required: - - exportPath + - run_id + - branch + - commit_id properties: - exportPath: - type: string - format: uri - x-nullable: false # Override https://github.com/go-swagger/go-swagger/issues/1188 - # go-swagger totally not a bug. This causes the generated field - # *not* to be a pointer. Then the regular (incorrect, in this - # case) JSON parser parses it as an empty field, and validation - # verifies the value is non-empty. In *this particular case* it - # works because a URI cannot be empty (at least not an absolute - # URI, which is what we require). - description: export objects to this path - example: s3://company-bucket/path/to/export - exportStatusPath: - type: string - format: uri - description: write export status object to this path - example: s3://company-bucket/path/to/status - lastKeysInPrefixRegexp: - type: array - items: - type: string - description: "list of regexps of keys to exported last in each prefix (for signalling)" - example: [ "^SUCCESS$", ".*/_SUCCESS$" ] - isContinuous: - type: boolean - description: if true, export every commit or merge to branch + run_id: + type: string + branch: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + event_type: + type: string + enum: [ pre_commit, pre_merge ] + status: + type: string + enum: [ failed, completed ] + commit_id: + type: string - config: + hook_run: type: object + required: + - hook_id properties: - blockstore.type: + hook_id: type: string + hook_type: + type: string + enum: [webhook] + action: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + status: + type: string + enum: [ failed, completed ] paths: @@ -1216,6 +1241,11 @@ paths: name: repository schema: $ref: "#/definitions/repository_creation" + - in: query + name: bare + type: boolean + default: false + description: If true, create a bare repository with no initial commit and branch responses: 201: description: repository @@ -1277,6 +1307,70 @@ paths: schema: $ref: "#/definitions/error" + /repositories/{repository}/refs/dump: + parameters: + - in: path + name: repository + required: true + type: string + put: + tags: + - refs + operationId: dump + summary: Dump repository refs (tags, commits, branches) to object store + responses: + 201: + description: refs dump + schema: + type: object + $ref: "#/definitions/refs_dump" + 400: + description: validation error + schema: + $ref: "#/definitions/error" + 401: + $ref: "#/responses/Unauthorized" + 404: + description: repository not found + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + /repositories/{repository}/refs/restore: + parameters: + - in: path + name: repository + required: true + type: string + put: + tags: + - refs + operationId: restore + summary: Restore repository refs (tags, commits, branches) from object store + parameters: + - in: body + name: manifest + schema: + $ref: "#/definitions/refs_dump" + responses: + 200: + description: refs successfully loaded + 400: + description: validation error + schema: + $ref: "#/definitions/error" + 401: + $ref: "#/responses/Unauthorized" + 404: + description: repository not found + schema: + $ref: "#/definitions/error" + default: + description: generic error response + schema: + $ref: "#/definitions/error" /repositories/{repository}/tags: parameters: - in: path @@ -1627,6 +1721,9 @@ paths: ref: type: string description: the commit to revert, given by a ref + parent_number: + type: integer + description: when reverting a merge commit, the parent number (starting from 1) relative to which to perform the revert. responses: 204: description: revert successful @@ -1641,7 +1738,7 @@ paths: schema: $ref: "#/definitions/error" - /repositories/{repository}/refs/{sourceRef}/merge/{destinationRef}: + /repositories/{repository}/refs/{sourceRef}/merge/{destinationBranch}: parameters: - in: path name: repository @@ -1651,9 +1748,9 @@ paths: name: sourceRef required: true type: string - description: source branch name + description: source ref - in: path - name: destinationRef + name: destinationBranch required: true type: string description: destination branch name @@ -2105,121 +2202,222 @@ paths: schema: $ref: "#/definitions/error" - /repositories/{repository}/branches/{branch}/continuous-export: - parameters: - - in: path - name: repository - required: true - type: string - - in: path - name: branch - required: true - type: string + /repositories/{repository}/actions/runs: get: tags: - - export - - branches - operationId: getContinuousExport - summary: returns the current continuous export configuration of a branch + - actions + operationId: listRuns + summary: list runs + parameters: + - in: path + name: repository + required: true + type: string + - in: query + name: before + type: string + format: date-time + - in: query + name: amount + type: integer + default: 100 + - in: query + name: branch + type: string responses: 200: - description: continuous export policy + description: list action runs schema: - $ref: "#/definitions/continuous_export_configuration" + type: object + properties: + pagination: + $ref: "#/definitions/pagination" + results: + type: array + items: + $ref: "#/definitions/action_run" 401: $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" 404: - description: no continuous export policy defined + description: not found schema: $ref: "#/definitions/error" + + /repositories/{repository}/actions/runs/{run_id}: + get: + tags: + - actions + operationId: getRun + summary: get a run + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: run_id + required: true + type: string + responses: + 200: + description: action run result + schema: + $ref: "#/definitions/action_run" + 401: + $ref: "#/responses/Unauthorized" default: description: generic error response schema: $ref: "#/definitions/error" - put: + 404: + description: not found + schema: + $ref: "#/definitions/error" + + /repositories/{repository}/actions/runs/{run_id}/hooks: + get: tags: - - export - - branches - operationId: setContinuousExport - summary: sets a new continuous export configuration of a branch + - actions + operationId: listRunHooks + summary: list run hooks parameters: - - in: body - name: config + - in: path + name: repository required: true - schema: - $ref: "#/definitions/continuous_export_configuration" + type: string + - in: path + name: run_id + required: true + type: string + - in: query + name: before + type: string + format: date-time + - in: query + name: amount + type: integer + default: 100 responses: - 201: - description: continuous export successfullyconfigured + 200: + description: list specific run hooks + schema: + type: object + properties: + pagination: + $ref: "#/definitions/pagination" + results: + type: array + items: + $ref: "#/definitions/hook_run" 401: $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" 404: - description: no branch defined at that repo + description: not found schema: $ref: "#/definitions/error" - default: - description: generic error response + + /repositories/{repository}/actions/runs/{run_id}/hooks/{hook_id}/output: + get: + tags: + - actions + operationId: getRunHookOutput + summary: get run hook output + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: run_id + required: true + type: string + - in: path + name: hook_id + required: true + type: string + produces: + - application/octet-stream + responses: + 200: + description: run hook output + schema: + type: file + headers: + Content-Length: + type: integer + format: int64 + Last-Modified: + type: string + Content-Disposition: + type: string + 401: + $ref: "#/responses/Unauthorized" + 404: + description: output not found schema: $ref: "#/definitions/error" - /repositories/{repository}/branches/{branch}/repair-export: + /repositories/{repository}/metadata/meta_range/{meta_range}: parameters: - in: path name: repository required: true type: string - in: path - name: branch + name: meta_range required: true type: string - post: + get: tags: - - export - - branches - operationId: repair - summary: set continuous export state as repaired + - metadata + operationId: getMetaRange + summary: return URI to a meta-range file + # TODO(ariels): Actually redirect to S3. responses: - 201: - description: continuous export status successfully changed to repaired + 200: + description: meta-range URI + headers: + Location: + type: string 401: $ref: "#/responses/Unauthorized" - 404: - description: no branch defined at that repo - schema: - $ref: "#/definitions/error" default: description: generic error response schema: $ref: "#/definitions/error" - /repositories/{repository}/branches/{branch}/export-hook: + /repositories/{repository}/metadata/range/{range}: parameters: - in: path name: repository required: true type: string - in: path - name: branch + name: range required: true type: string - post: + get: tags: - - export - - branches - operationId: run - summary: hook to be called in order to execute continuous export on branch + - metadata + operationId: getRange + summary: return URI to a range file + # TODO(ariels): Actually redirect to S3. responses: - 201: - description: continuous export successfully started - schema: - description: "export ID" - type: string + 200: + description: range URI + headers: + Location: + type: string 401: $ref: "#/responses/Unauthorized" - 404: - description: no branch defined at that repo - schema: - $ref: "#/definitions/error" default: description: generic error response schema: diff --git a/graveler/committed/manager.go b/graveler/committed/manager.go index 9e8ec1951b8..21b4cc24851 100644 --- a/graveler/committed/manager.go +++ b/graveler/committed/manager.go @@ -146,3 +146,13 @@ func (c *committedManager) Compare(ctx context.Context, ns graveler.StorageNames } return NewCompareIterator(diffIt, baseIt), nil } + +func (c *committedManager) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { + uri, err := c.metaRangeManager.GetMetaRangeURI(ctx, ns, id) + return graveler.MetaRangeInfo{Address: uri}, err +} + +func (c *committedManager) GetRange(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (graveler.RangeInfo, error) { + uri, err := c.metaRangeManager.GetRangeURI(ctx, ns, id) + return graveler.RangeInfo{Address: uri}, err +} diff --git a/graveler/committed/meta_range.go b/graveler/committed/meta_range.go index d234f6bc94c..57bf60412c5 100644 --- a/graveler/committed/meta_range.go +++ b/graveler/committed/meta_range.go @@ -37,6 +37,14 @@ type MetaRangeManager interface { // NewMetaRangeIterator returns an Iterator over the MetaRange with id. NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (Iterator, error) + + // GetMetaRangeURI returns a URI with an object representing metarange ID. It may + // return a URI that does not resolve (rather than an error) if ID does not exist. + GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (string, error) + + // GetRangeURI returns a URI with an object representing metarange ID. It may + // return a URI that does not resolve (rather than an error) if ID does not exist. + GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (string, error) } // MetaRangeWriter is an abstraction for creating new MetaRanges diff --git a/graveler/committed/meta_range_manager.go b/graveler/committed/meta_range_manager.go index a841334d509..06521af1abd 100644 --- a/graveler/committed/meta_range_manager.go +++ b/graveler/committed/meta_range_manager.go @@ -95,3 +95,11 @@ func (m *metaRangeManager) NewMetaRangeIterator(ctx context.Context, ns graveler } return NewIterator(ctx, m.rangeManager, Namespace(ns), rangesIt), nil } + +func (m *metaRangeManager) GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (string, error) { + return m.metaManager.GetURI(ctx, Namespace(ns), ID(id)) +} + +func (m *metaRangeManager) GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, id graveler.RangeID) (string, error) { + return m.rangeManager.GetURI(ctx, Namespace(ns), ID(id)) +} diff --git a/graveler/committed/range.go b/graveler/committed/range.go index d2868378188..a606ebbf896 100644 --- a/graveler/committed/range.go +++ b/graveler/committed/range.go @@ -3,6 +3,7 @@ package committed import "google.golang.org/protobuf/proto" // Range represents a range of sorted Keys + type Range struct { ID ID MinKey Key diff --git a/graveler/committed/range_manager.go b/graveler/committed/range_manager.go index 97d6e19991d..070d0b7ce05 100644 --- a/graveler/committed/range_manager.go +++ b/graveler/committed/range_manager.go @@ -59,6 +59,10 @@ type RangeManager interface { // GetWriter returns a new Range writer instance GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error) + + // GetURI returns a URI from which to read the contents of id. If id does not exist + // it may return a URI that resolves nowhere rather than an error. + GetURI(ctx context.Context, ns Namespace, id ID) (string, error) } // WriteResult is the result of a completed write of a Range diff --git a/graveler/graveler.go b/graveler/graveler.go index 173214912c3..1f851139313 100644 --- a/graveler/graveler.go +++ b/graveler/graveler.go @@ -46,6 +46,16 @@ type Reference interface { CommitID() CommitID } +type MetaRangeInfo struct { + // URI of metarange file. + Address string +} + +type RangeInfo struct { + // URI of range file. + Address string +} + // function/methods receiving the following basic types could assume they passed validation // StorageNamespace is the URI to the storage location @@ -74,6 +84,9 @@ type CommitID string // MetaRangeID represents a snapshot of the MetaRange, referenced by a commit type MetaRangeID string +// RangeID represents a part of a MetaRange, useful only for plumbing. +type RangeID string + // StagingToken represents a namespace for writes to apply as uncommitted type StagingToken string @@ -321,6 +334,15 @@ type VersionController interface { SetHooksHandler(handler HooksHandler) } +// Plumbing includes commands for fiddling more directly with graveler implementation +// internals. +type Plumbing interface { + // GetMetarange returns information where metarangeID is stored. + GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeInfo, error) + // GetRange returns information where rangeID is stored. + GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeInfo, error) +} + type Dumper interface { // DumpCommits iterates through all commits and dumps them in Graveler format DumpCommits(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error) @@ -333,13 +355,13 @@ type Dumper interface { } type Loader interface { - // DumpCommits iterates through all commits in Graveler format and loads them into repositoryID + // LoadCommits iterates through all commits in Graveler format and loads them into repositoryID LoadCommits(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error - // DumpBranches iterates through all branches in Graveler format and loads them into repositoryID + // LoadBranches iterates through all branches in Graveler format and loads them into repositoryID LoadBranches(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error - // DumpTags iterates through all tags in Graveler format and loads them into repositoryID + // LoadTags iterates through all tags in Graveler format and loads them into repositoryID LoadTags(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error } @@ -505,6 +527,11 @@ type CommittedManager interface { // A change is either an entity to write/overwrite, or a tombstone to mark a deletion // it returns a new MetaRangeID that is expected to be immediately addressable Apply(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID, iterator ValueIterator) (MetaRangeID, DiffSummary, error) + + // GetMetarange returns information where metarangeID is stored. + GetMetaRange(ctx context.Context, ns StorageNamespace, metaRangeID MetaRangeID) (MetaRangeInfo, error) + // GetRange returns information where rangeID is stored. + GetRange(ctx context.Context, ns StorageNamespace, rangeID RangeID) (RangeInfo, error) } // StagingManager manages entries in a staging area, denoted by a staging token @@ -1513,6 +1540,22 @@ func (g *Graveler) LoadTags(ctx context.Context, repositoryID RepositoryID, meta return nil } +func (g *Graveler) GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeInfo, error) { + repo, err := g.RefManager.GetRepository(ctx, repositoryID) + if err != nil { + return MetaRangeInfo{}, nil + } + return g.CommittedManager.GetMetaRange(ctx, repo.StorageNamespace, metaRangeID) +} + +func (g *Graveler) GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeInfo, error) { + repo, err := g.RefManager.GetRepository(ctx, repositoryID) + if err != nil { + return RangeInfo{}, nil + } + return g.CommittedManager.GetRange(ctx, repo.StorageNamespace, rangeID) +} + func (g *Graveler) DumpCommits(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error) { repo, err := g.GetRepository(ctx, repositoryID) if err != nil { diff --git a/graveler/sstable/range_manager.go b/graveler/sstable/range_manager.go index 4171c3215e5..04f875fb7c5 100644 --- a/graveler/sstable/range_manager.go +++ b/graveler/sstable/range_manager.go @@ -170,6 +170,10 @@ func (m *RangeManager) GetWriter(ctx context.Context, ns committed.Namespace, me return NewDiskWriter(ctx, m.fs, ns, m.hash.New(), metadata) } +func (m *RangeManager) GetURI(ctx context.Context, ns committed.Namespace, id committed.ID) (string, error) { + return m.fs.GetRemoteURI(ctx, string(ns), string(id)) +} + func (m *RangeManager) execAndLog(ctx context.Context, f func() error, msg string) { if err := f(); err != nil { logging.FromContext(ctx).WithError(err).Error(msg) diff --git a/graveler/testutil/fakes.go b/graveler/testutil/fakes.go index 6254ba251f3..e7a12ec4c5f 100644 --- a/graveler/testutil/fakes.go +++ b/graveler/testutil/fakes.go @@ -97,6 +97,18 @@ func (c *CommittedFake) WriteMetaRange(ctx context.Context, ns graveler.StorageN return &c.MetaRangeID, nil } +func (c *CommittedFake) GetMetaRange(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (graveler.MetaRangeInfo, error) { + return graveler.MetaRangeInfo{ + Address: fmt.Sprintf("fake://prefix/%s(metarange)", metaRangeID), + }, nil +} + +func (c *CommittedFake) GetRange(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (graveler.RangeInfo, error) { + return graveler.RangeInfo{ + Address: fmt.Sprintf("fake://prefix/%s(range)", rangeID), + }, nil +} + type StagingFake struct { Err error DropErr error // specific error for drop call diff --git a/pyramid/pyramid.go b/pyramid/pyramid.go index 7c8482c61e1..283d5e7d5a8 100644 --- a/pyramid/pyramid.go +++ b/pyramid/pyramid.go @@ -22,6 +22,10 @@ type FS interface { // Exists returns true if filename currently exists on block storage. Exists(ctx context.Context, namespace, filename string) (bool, error) + + // GetRemoteURI returns the URI for filename on block storage. That URI might not + // resolve if filename does not exist. + GetRemoteURI(ctx context.Context, namespace, filename string) (string, error) } // File is pyramid abstraction for an os.File diff --git a/pyramid/tier_fs.go b/pyramid/tier_fs.go index 87a16bd5c5b..57e3a6bbb2d 100644 --- a/pyramid/tier_fs.go +++ b/pyramid/tier_fs.go @@ -170,6 +170,10 @@ func (tfs *TierFS) store(ctx context.Context, namespace, originalPath, nsPath, f } } +func (tfs *TierFS) GetRemoteURI(_ context.Context, _, filename string) (string, error) { + return tfs.blockStoragePath(filename), nil +} + // Create creates a new file in TierFS. File isn't stored in TierFS until a successful close // operation. Open(namespace, filename) calls will return an error before the close was // called. Create only performs local operations so it ignores the context. diff --git a/swagger.yml b/swagger.yml index d4c4c6910e5..ae367c5e165 100644 --- a/swagger.yml +++ b/swagger.yml @@ -2369,6 +2369,64 @@ paths: schema: $ref: "#/definitions/error" + /repositories/{repository}/metadata/meta_range/{meta_range}: + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: meta_range + required: true + type: string + get: + tags: + - metadata + operationId: getMetaRange + summary: return URI to a meta-range file + # TODO(ariels): Actually redirect to S3. + responses: + 200: + description: meta-range URI + headers: + Location: + type: string + 401: + $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + + /repositories/{repository}/metadata/range/{range}: + parameters: + - in: path + name: repository + required: true + type: string + - in: path + name: range + required: true + type: string + get: + tags: + - metadata + operationId: getRange + summary: return URI to a range file + # TODO(ariels): Actually redirect to S3. + responses: + 200: + description: range URI + headers: + Location: + type: string + 401: + $ref: "#/responses/Unauthorized" + default: + description: generic error response + schema: + $ref: "#/definitions/error" + /healthcheck: get: operationId: healthCheck