Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve memory usage and execution time of listing objects with file system backend #556

Merged
merged 2 commits into from
Aug 14, 2021
Merged

Improve memory usage and execution time of listing objects with file system backend #556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3eb4422
Do not store all objects in memory on list commands
ironsmile Aug 13, 2021
f2340d3
List objects: filter files by prefix before reading them
ironsmile Aug 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions fakestorage/bucket_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ import (

func TestServerClientBucketAttrs(t *testing.T) {
objs := []Object{
{BucketName: "some-bucket", Name: "img/hi-res/party-01.jpg"},
{BucketName: "some-bucket", Name: "img/hi-res/party-02.jpg"},
{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"},
{BucketName: "other_bucket", Name: "static/css/website.css"},
{BucketName: "dot.bucket", Name: "static/js/app.js"},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-01.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-02.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "other_bucket", Name: "static/css/website.css"}},
{ObjectAttrs: ObjectAttrs{BucketName: "dot.bucket", Name: "static/js/app.js"}},
}
startTime := time.Now()
runServersTest(t, objs, func(t *testing.T, server *Server) {
Expand Down Expand Up @@ -86,7 +86,7 @@ func TestServerClientDeleteBucket(t *testing.T) {

t.Run("it returns an error for non-empty buckets", func(t *testing.T) {
const bucketName = "non-empty-bucket"
objs := []Object{{BucketName: bucketName, Name: "static/js/app.js"}}
objs := []Object{{ObjectAttrs: ObjectAttrs{BucketName: bucketName, Name: "static/js/app.js"}}}
runServersTest(t, objs, func(t *testing.T, server *Server) {
client := server.Client()
err := client.Bucket(bucketName).Delete(context.Background())
Expand Down Expand Up @@ -179,11 +179,11 @@ func TestServerClientBucketAttrsNotFound(t *testing.T) {

func TestServerClientListBuckets(t *testing.T) {
objs := []Object{
{BucketName: "some-bucket", Name: "img/hi-res/party-01.jpg"},
{BucketName: "some-bucket", Name: "img/hi-res/party-02.jpg"},
{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"},
{BucketName: "other_bucket", Name: "static/css/website.css"},
{BucketName: "dot.bucket", Name: "static/js/app.js"},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-01.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-02.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "other_bucket", Name: "static/css/website.css"}},
{ObjectAttrs: ObjectAttrs{BucketName: "dot.bucket", Name: "static/js/app.js"}},
}

runServersTest(t, objs, func(t *testing.T, server *Server) {
Expand Down Expand Up @@ -224,9 +224,9 @@ func TestServerClientListBuckets(t *testing.T) {

func TestServerClientListObjects(t *testing.T) {
objects := []Object{
{BucketName: "some-bucket", Name: "img/hi-res/party-01.jpg"},
{BucketName: "some-bucket", Name: "img/hi-res/party-02.jpg"},
{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-01.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-02.jpg"}},
{ObjectAttrs: ObjectAttrs{BucketName: "some-bucket", Name: "img/hi-res/party-03.jpg"}},
}
dir, err := ioutil.TempDir("", "fakestorage-test-root-")
if err != nil {
Expand Down
16 changes: 10 additions & 6 deletions fakestorage/example_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@ import (
func ExampleServer_Client() {
server := fakestorage.NewServer([]fakestorage.Object{
{
BucketName: "some-bucket",
Name: "some/object/file.txt",
Content: []byte("inside the file"),
ObjectAttrs: fakestorage.ObjectAttrs{
BucketName: "some-bucket",
Name: "some/object/file.txt",
},
Content: []byte("inside the file"),
},
})
defer server.Stop()
Expand All @@ -40,9 +42,11 @@ func ExampleServer_with_host_port() {
server, err := fakestorage.NewServerWithOptions(fakestorage.Options{
InitialObjects: []fakestorage.Object{
{
BucketName: "some-bucket",
Name: "some/object/file.txt",
Content: []byte("inside the file"),
ObjectAttrs: fakestorage.ObjectAttrs{
BucketName: "some-bucket",
Name: "some/object/file.txt",
},
Content: []byte("inside the file"),
},
},
Host: "127.0.0.1",
Expand Down
155 changes: 98 additions & 57 deletions fakestorage/object.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@ import (

var errInvalidGeneration = errors.New("invalid generation ID")

// Object represents the object that is stored within the fake server.
type Object struct {
// ObjectAttrs returns only the meta-data about an object without its contents.
type ObjectAttrs struct {
BucketName string
Name string
Size int64
ContentType string
ContentEncoding string
Content []byte
// Crc32c checksum of Content. calculated by server when it's upload methods are used.
Crc32c string
Md5Hash string
Expand All @@ -43,11 +43,22 @@ type Object struct {
Metadata map[string]string
}

func (o *ObjectAttrs) id() string {
return o.BucketName + "/" + o.Name
}

// Object represents the object that is stored within the fake server.
type Object struct {
ObjectAttrs
Content []byte
}

// MarshalJSON for Object to use ACLRule instead of storage.ACLRule
func (o Object) MarshalJSON() ([]byte, error) {
temp := struct {
BucketName string `json:"bucket"`
Name string `json:"name"`
Size int64 `json:"-"`
ContentType string `json:"contentType"`
ContentEncoding string `json:"contentEncoding"`
Content []byte `json:"-"`
Expand Down Expand Up @@ -85,6 +96,7 @@ func (o *Object) UnmarshalJSON(data []byte) error {
temp := struct {
BucketName string `json:"bucket"`
Name string `json:"name"`
Size int64 `json:"-"`
ContentType string `json:"contentType"`
ContentEncoding string `json:"contentEncoding"`
Content []byte `json:"-"`
Expand Down Expand Up @@ -193,21 +205,18 @@ func (team *projectTeam) UnmarshalJSON(data []byte) error {
team.Team = temp.Team
return nil
}
func (o *Object) id() string {
return o.BucketName + "/" + o.Name
}

type objectList []Object
type objectAttrsList []ObjectAttrs

func (o objectList) Len() int {
func (o objectAttrsList) Len() int {
return len(o)
}

func (o objectList) Less(i int, j int) bool {
func (o objectAttrsList) Less(i int, j int) bool {
return o[i].Name < o[j].Name
}

func (o *objectList) Swap(i int, j int) {
func (o *objectAttrsList) Swap(i int, j int) {
d := *o
d[i], d[j] = d[j], d[i]
}
Expand Down Expand Up @@ -244,37 +253,38 @@ type ListOptions struct {
// or an error if the bucket doesn't exist.
//
// Deprecated: use ListObjectsWithOptions.
func (s *Server) ListObjects(bucketName, prefix, delimiter string, versions bool) ([]Object, []string, error) {
func (s *Server) ListObjects(bucketName, prefix, delimiter string, versions bool) ([]ObjectAttrs, []string, error) {
return s.ListObjectsWithOptions(bucketName, ListOptions{
Prefix: prefix,
Delimiter: delimiter,
Versions: versions,
})
}

func (s *Server) ListObjectsWithOptions(bucketName string, options ListOptions) ([]Object, []string, error) {
backendObjects, err := s.backend.ListObjects(bucketName, options.Versions)
func (s *Server) ListObjectsWithOptions(bucketName string, options ListOptions) ([]ObjectAttrs, []string, error) {
backendObjects, err := s.backend.ListObjects(bucketName, options.Prefix, options.Versions)
if err != nil {
return nil, nil, err
}
objects := fromBackendObjects(backendObjects)
olist := objectList(objects)
objects := fromBackendObjectsAttrs(backendObjects)
olist := objectAttrsList(objects)
sort.Sort(&olist)
var respObjects []Object
var respObjects []ObjectAttrs
prefixes := make(map[string]bool)
for _, obj := range olist {
if strings.HasPrefix(obj.Name, options.Prefix) {
objName := strings.Replace(obj.Name, options.Prefix, "", 1)
delimPos := strings.Index(objName, options.Delimiter)
if options.Delimiter != "" && delimPos > -1 {
prefix := obj.Name[:len(options.Prefix)+delimPos+1]
if isInOffset(prefix, options.StartOffset, options.EndOffset) {
prefixes[prefix] = true
}
} else {
if isInOffset(obj.Name, options.StartOffset, options.EndOffset) {
respObjects = append(respObjects, obj)
}
if !strings.HasPrefix(obj.Name, options.Prefix) {
continue
}
objName := strings.Replace(obj.Name, options.Prefix, "", 1)
delimPos := strings.Index(objName, options.Delimiter)
if options.Delimiter != "" && delimPos > -1 {
prefix := obj.Name[:len(options.Prefix)+delimPos+1]
if isInOffset(prefix, options.StartOffset, options.EndOffset) {
prefixes[prefix] = true
}
} else {
if isInOffset(obj.Name, options.StartOffset, options.EndOffset) {
respObjects = append(respObjects, obj)
}
}
}
Expand Down Expand Up @@ -309,19 +319,22 @@ func toBackendObjects(objects []Object) []backend.Object {
backendObjects := []backend.Object{}
for _, o := range objects {
backendObjects = append(backendObjects, backend.Object{
BucketName: o.BucketName,
Name: o.Name,
Content: o.Content,
ContentType: o.ContentType,
ContentEncoding: o.ContentEncoding,
Crc32c: o.Crc32c,
Md5Hash: o.Md5Hash,
ACL: o.ACL,
Created: getCurrentIfZero(o.Created).Format(timestampFormat),
Deleted: o.Deleted.Format(timestampFormat),
Updated: getCurrentIfZero(o.Updated).Format(timestampFormat),
Generation: o.Generation,
Metadata: o.Metadata,
ObjectAttrs: backend.ObjectAttrs{
BucketName: o.BucketName,
Name: o.Name,
Size: int64(len(o.Content)),
ContentType: o.ContentType,
ContentEncoding: o.ContentEncoding,
Crc32c: o.Crc32c,
Md5Hash: o.Md5Hash,
ACL: o.ACL,
Created: getCurrentIfZero(o.Created).Format(timestampFormat),
Deleted: o.Deleted.Format(timestampFormat),
Updated: getCurrentIfZero(o.Updated).Format(timestampFormat),
Generation: o.Generation,
Metadata: o.Metadata,
},
Content: o.Content,
})
}
return backendObjects
Expand All @@ -331,9 +344,34 @@ func fromBackendObjects(objects []backend.Object) []Object {
backendObjects := []Object{}
for _, o := range objects {
backendObjects = append(backendObjects, Object{
ObjectAttrs: ObjectAttrs{
BucketName: o.BucketName,
Name: o.Name,
Size: int64(len(o.Content)),
ContentType: o.ContentType,
ContentEncoding: o.ContentEncoding,
Crc32c: o.Crc32c,
Md5Hash: o.Md5Hash,
ACL: o.ACL,
Created: convertTimeWithoutError(o.Created),
Deleted: convertTimeWithoutError(o.Deleted),
Updated: convertTimeWithoutError(o.Updated),
Generation: o.Generation,
Metadata: o.Metadata,
},
Content: o.Content,
})
}
return backendObjects
}

func fromBackendObjectsAttrs(objectAttrs []backend.ObjectAttrs) []ObjectAttrs {
oattrs := []ObjectAttrs{}
for _, o := range objectAttrs {
oattrs = append(oattrs, ObjectAttrs{
BucketName: o.BucketName,
Name: o.Name,
Content: o.Content,
Size: o.Size,
ContentType: o.ContentType,
ContentEncoding: o.ContentEncoding,
Crc32c: o.Crc32c,
Expand All @@ -346,7 +384,7 @@ func fromBackendObjects(objects []backend.Object) []Object {
Metadata: o.Metadata,
})
}
return backendObjects
return oattrs
}

func convertTimeWithoutError(t string) time.Time {
Expand Down Expand Up @@ -430,7 +468,7 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) {
header.Set("Accept-Ranges", "bytes")
return jsonResponse{
header: header,
data: newObjectResponse(obj),
data: newObjectResponse(obj.ObjectAttrs),
}
})

Expand All @@ -455,7 +493,7 @@ func (s *Server) listObjectACL(r *http.Request) jsonResponse {
return jsonResponse{status: http.StatusNotFound}
}

return jsonResponse{data: newACLListResponse(obj)}
return jsonResponse{data: newACLListResponse(obj.ObjectAttrs)}
}

func (s *Server) setObjectACL(r *http.Request) jsonResponse {
Expand Down Expand Up @@ -488,7 +526,7 @@ func (s *Server) setObjectACL(r *http.Request) jsonResponse {

s.CreateObject(obj)

return jsonResponse{data: newACLListResponse(obj)}
return jsonResponse{data: newACLListResponse(obj.ObjectAttrs)}
}

func (s *Server) rewriteObject(r *http.Request) jsonResponse {
Expand Down Expand Up @@ -523,19 +561,22 @@ func (s *Server) rewriteObject(r *http.Request) jsonResponse {

dstBucket := vars["destinationBucket"]
newObject := Object{
BucketName: dstBucket,
Name: vars["destinationObject"],
Content: append([]byte(nil), obj.Content...),
Crc32c: obj.Crc32c,
Md5Hash: obj.Md5Hash,
ACL: obj.ACL,
ContentType: metadata.ContentType,
ContentEncoding: metadata.ContentEncoding,
Metadata: metadata.Metadata,
ObjectAttrs: ObjectAttrs{
BucketName: dstBucket,
Name: vars["destinationObject"],
Size: int64(len(obj.Content)),
Crc32c: obj.Crc32c,
Md5Hash: obj.Md5Hash,
ACL: obj.ACL,
ContentType: metadata.ContentType,
ContentEncoding: metadata.ContentEncoding,
Metadata: metadata.Metadata,
},
Content: append([]byte(nil), obj.Content...),
}

s.CreateObject(newObject)
return jsonResponse{data: newObjectRewriteResponse(newObject)}
return jsonResponse{data: newObjectRewriteResponse(newObject.ObjectAttrs)}
}

func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) {
Expand Down Expand Up @@ -662,5 +703,5 @@ func (s *Server) composeObject(r *http.Request) jsonResponse {

obj := fromBackendObjects([]backend.Object{backendObj})[0]

return jsonResponse{data: newObjectResponse(obj)}
return jsonResponse{data: newObjectResponse(obj.ObjectAttrs)}
}
Loading