From 0a1124ffdce5d0aab031736c6c94964fcf8f3585 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 12 May 2023 14:37:11 +0100 Subject: [PATCH 1/4] Support XML API (#331) --- fakestorage/object.go | 68 ++++++++++++++++++++++++++++++++ fakestorage/server.go | 9 ++--- fakestorage/upload.go | 2 + fakestorage/upload_test.go | 79 ++++++++++++++++++++++++++++---------- 4 files changed, 131 insertions(+), 27 deletions(-) diff --git a/fakestorage/object.go b/fakestorage/object.go index ddeb299a11..a002d819aa 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "net/http" + "net/url" "sort" "strconv" "strings" @@ -626,6 +627,72 @@ func (s *Server) xmlListObjects(r *http.Request) xmlResponse { } } +func (s *Server) xmlPutObject(r *http.Request) xmlResponse { + // https://cloud.google.com/storage/docs/xml-api/put-object-upload + vars := unescapeMuxVars(mux.Vars(r)) + defer r.Body.Close() + + metaData := make(map[string]string) + for key := range r.Header { + lowerKey := strings.ToLower(key) + if metaDataKey := strings.TrimPrefix(lowerKey, "x-goog-meta-"); metaDataKey != lowerKey { + metaData[metaDataKey] = r.Header.Get(key) + } + } + + obj := StreamingObject{ + ObjectAttrs: ObjectAttrs{ + BucketName: vars["bucketName"], + Name: vars["objectName"], + ContentType: r.Header.Get(contentTypeHeader), + ContentEncoding: r.Header.Get(contentEncodingHeader), + Metadata: metaData, + }, + } + if source := r.Header.Get("x-goog-copy-source"); source != "" { + escaped, err := url.PathUnescape(source) + if err != nil { + return xmlResponse{status: http.StatusBadRequest} + } + + split := strings.SplitN(escaped, "/", 2) + if len(split) != 2 { + return xmlResponse{status: http.StatusBadRequest} + } + + sourceObject, err := s.GetObjectStreaming(split[0], split[1]) + if err != nil { + return xmlResponse{status: http.StatusNotFound} + } + obj.Content = sourceObject.Content + } else { + obj.Content = notImplementedSeeker{r.Body} + } + + obj, err := s.createObject(obj, backend.NoConditions{}) + + if err != nil { + return xmlResponse{ + status: http.StatusInternalServerError, + errorMessage: err.Error(), + } + } + + obj.Close() + return xmlResponse{ + status: http.StatusOK, + } +} + +func (s *Server) xmlDeleteObject(r *http.Request) xmlResponse { + resp := s.deleteObject(r) + return xmlResponse{ + status: resp.status, + errorMessage: resp.errorMessage, + header: resp.header, + } +} + func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { if alt := r.URL.Query().Get("alt"); alt == "media" || r.Method == http.MethodHead { s.downloadObject(w, r) @@ -867,6 +934,7 @@ func (s *Server) downloadObject(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Goog-Generation", strconv.FormatInt(obj.Generation, 10)) w.Header().Set("X-Goog-Hash", fmt.Sprintf("crc32c=%s,md5=%s", obj.Crc32c, obj.Md5Hash)) w.Header().Set("Last-Modified", obj.Updated.Format(http.TimeFormat)) + w.Header().Set("ETag", obj.Etag) for name, value := range obj.Metadata { w.Header().Set("X-Goog-Meta-"+name, value) } diff --git a/fakestorage/server.go b/fakestorage/server.go index 9a230012f3..09f6bb5a74 100644 --- a/fakestorage/server.go +++ b/fakestorage/server.go @@ -276,6 +276,9 @@ func (s *Server) buildMuxer() { for _, r := range xmlApiRouters { r.Path("/").Methods(http.MethodGet).HandlerFunc(xmlToHTTPHandler(s.xmlListObjects)) r.Path("").Methods(http.MethodGet).HandlerFunc(xmlToHTTPHandler(s.xmlListObjects)) + r.Path("/{objectName:.+}").Methods(http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.xmlPutObject)) + r.Path("/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.downloadObject) + r.Path("/{objectName:.+}").Methods(http.MethodDelete).HandlerFunc(xmlToHTTPHandler(s.xmlDeleteObject)) } bucketHost := fmt.Sprintf("{bucketName}.%s", s.publicHost) @@ -296,12 +299,6 @@ func (s *Server) buildMuxer() { handler.Host(s.publicHost).Path("/{bucketName}").MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject)) handler.Host(bucketHost).MatcherFunc(matchFormData).Methods(http.MethodPost, http.MethodPut).HandlerFunc(xmlToHTTPHandler(s.insertFormObject)) - // Signed URLs (upload and download) - handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject)) - handler.MatcherFunc(s.publicHostMatcher).Path("/{bucketName}/{objectName:.+}").Methods(http.MethodGet, http.MethodHead).HandlerFunc(s.getObject) - handler.Host(bucketHost).Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject)) - handler.Host("{bucketName:.+}").Path("/{objectName:.+}").Methods(http.MethodPost, http.MethodPut).HandlerFunc(jsonToHTTPHandler(s.insertObject)) - s.handler = handler } diff --git a/fakestorage/upload.go b/fakestorage/upload.go index 4b3d8593f9..254bee3bf6 100644 --- a/fakestorage/upload.go +++ b/fakestorage/upload.go @@ -27,6 +27,8 @@ import ( const contentTypeHeader = "Content-Type" +const contentEncodingHeader = "Content-Encoding" + const ( uploadTypeMedia = "media" uploadTypeMultipart = "multipart" diff --git a/fakestorage/upload_test.go b/fakestorage/upload_test.go index 2efbae34e6..c17ffb5409 100644 --- a/fakestorage/upload_test.go +++ b/fakestorage/upload_test.go @@ -10,7 +10,6 @@ import ( "context" "crypto/tls" "encoding/binary" - "encoding/json" "fmt" "io" "mime/multipart" @@ -570,9 +569,6 @@ func TestServerClientSignedUpload(t *testing.T) { func TestServerClientSignedUploadBucketCNAME(t *testing.T) { url := "https://mybucket.mydomain.com:4443/files/txt/text-02.txt?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=fake-gcs&X-Goog-Expires=3600&X-Goog-SignedHeaders=host&X-Goog-Signature=fake-gc" - expectedName := "files/txt/text-02.txt" - expectedContentType := "text/plain" - expectedHash := "bHupxaFBQh4cA8uYB8l8dA==" opts := Options{ InitialObjects: []Object{ {ObjectAttrs: ObjectAttrs{BucketName: "mybucket.mydomain.com", Name: "files/txt/text-01.txt"}, Content: []byte("something")}, @@ -596,23 +592,6 @@ func TestServerClientSignedUploadBucketCNAME(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Errorf("wrong status returned\nwant %d\ngot %d", http.StatusOK, resp.StatusCode) } - data, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - var obj Object - if err := json.Unmarshal(data, &obj); err != nil { - t.Fatal(err) - } - if obj.Name != expectedName { - t.Errorf("wrong filename\nwant %q\ngot %q", expectedName, obj.Name) - } - if obj.ContentType != expectedContentType { - t.Errorf("wrong content type\nwant %q\ngot %q", expectedContentType, obj.ContentType) - } - if obj.Md5Hash != expectedHash { - t.Errorf("wrong md5 hash\nwant %q\ngot %q", expectedHash, obj.Md5Hash) - } } func TestServerClientUploadWithPredefinedAclPublicRead(t *testing.T) { @@ -700,6 +679,64 @@ func TestServerClientSimpleUploadNoName(t *testing.T) { } } +func TestServerXMLPut(t *testing.T) { + server, err := NewServerWithOptions(Options{ + PublicHost: "test", + }) + if err != nil { + t.Fatal(err) + } + defer server.Stop() + server.CreateBucketWithOpts(CreateBucketOpts{Name: "bucket1"}) + server.CreateBucketWithOpts(CreateBucketOpts{Name: "bucket2"}) + + const data = "some nice content" + req, err := http.NewRequest("PUT", server.URL()+"/bucket1/path", strings.NewReader(data)) + req.Host = "test" + if err != nil { + t.Fatal(err) + } + client := http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("got %d expected %d", resp.StatusCode, http.StatusOK) + } + + req, err = http.NewRequest("PUT", server.URL()+"/bucket2/path", nil) + req.Host = "test" + req.Header.Set("x-goog-copy-source", "bucket1/path") + + resp, err = client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("got %d expected %d", resp.StatusCode, http.StatusOK) + } + + req, err = http.NewRequest("PUT", server.URL()+"/bucket2/path2", nil) + req.Host = "test" + req.Header.Set("x-goog-copy-source", "bucket1/nonexistent") + + resp, err = client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("got %d expected %d", resp.StatusCode, http.StatusNotFound) + } +} + func TestServerInvalidUploadType(t *testing.T) { server := NewServer(nil) defer server.Stop() From fb93ee203158f2ce9430722810593110400ca46e Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 12 May 2023 16:53:43 +0100 Subject: [PATCH 2/4] Check bucket exists --- fakestorage/object.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fakestorage/object.go b/fakestorage/object.go index a002d819aa..08c3f1383f 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -632,6 +632,10 @@ func (s *Server) xmlPutObject(r *http.Request) xmlResponse { vars := unescapeMuxVars(mux.Vars(r)) defer r.Body.Close() + if _, err := s.backend.GetBucket(vars["bucketName"]); err != nil { + return xmlResponse{status: http.StatusNotFound} + } + metaData := make(map[string]string) for key := range r.Header { lowerKey := strings.ToLower(key) From 02e0ce0db7d9a31210e0954c5f003a7df28eb65e Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Tue, 17 Oct 2023 14:24:07 +0100 Subject: [PATCH 3/4] Return ETag --- fakestorage/object.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fakestorage/object.go b/fakestorage/object.go index 08c3f1383f..95003d4158 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -683,8 +683,13 @@ func (s *Server) xmlPutObject(r *http.Request) xmlResponse { } obj.Close() + + header := make(http.Header) + header.Set("ETag", obj.Etag) + return xmlResponse{ status: http.StatusOK, + header: header, } } From 46f2f7232d7250f960df496ca60da81326196038 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 27 Oct 2023 13:31:09 +0100 Subject: [PATCH 4/4] Support list start-after --- fakestorage/object.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fakestorage/object.go b/fakestorage/object.go index 95003d4158..cb9c8e1d3a 100644 --- a/fakestorage/object.go +++ b/fakestorage/object.go @@ -325,6 +325,7 @@ type ListOptions struct { StartOffset string EndOffset string IncludeTrailingDelimiter bool + StartExclusive bool } // ListObjects returns a sorted list of objects that match the given criteria, @@ -353,6 +354,9 @@ func (s *Server) ListObjectsWithOptions(bucketName string, options ListOptions) if !strings.HasPrefix(obj.Name, options.Prefix) { continue } + if options.StartExclusive && obj.Name == options.StartOffset { + continue + } objName := strings.Replace(obj.Name, options.Prefix, "", 1) delimPos := strings.Index(objName, options.Delimiter) if options.Delimiter != "" && delimPos > -1 { @@ -577,9 +581,11 @@ func (s *Server) xmlListObjects(r *http.Request) xmlResponse { bucketName := unescapeMuxVars(mux.Vars(r))["bucketName"] opts := ListOptions{ - Prefix: r.URL.Query().Get("prefix"), - Delimiter: r.URL.Query().Get("delimiter"), - Versions: r.URL.Query().Get("versions") == "true", + Prefix: r.URL.Query().Get("prefix"), + Delimiter: r.URL.Query().Get("delimiter"), + Versions: r.URL.Query().Get("versions") == "true", + StartOffset: r.URL.Query().Get("start-after"), + StartExclusive: true, } objs, prefixes, err := s.ListObjectsWithOptions(bucketName, opts)