From d3f56cc0d0be4d8daf0790a14b0cd09647cb7f56 Mon Sep 17 00:00:00 2001 From: Robert van Gent Date: Wed, 31 Oct 2018 14:28:46 -0700 Subject: [PATCH 1/3] ckpt --- blob/gcsblob/gcsblob.go | 71 ++++++++++++++++++++++++++++++++++++-- blob/s3blob/s3blob.go | 41 +++++++++++++++++++--- blob/s3blob/s3blob_test.go | 2 +- 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/blob/gcsblob/gcsblob.go b/blob/gcsblob/gcsblob.go index 322b7f396f..f93663776d 100644 --- a/blob/gcsblob/gcsblob.go +++ b/blob/gcsblob/gcsblob.go @@ -14,6 +14,17 @@ // Package gcsblob provides an implementation of blob that uses GCS. // +// For blob.Open URLs, gcsblob registers for the "gs" protocol. +// The URL's Host is used as the bucket name. +// The following query options are supported: +// - cred_path: Sets path to the Google credentials file. If unset, default +// credentials are loaded. +// See https://cloud.google.com/docs/authentication/production. +// - access_id: Sets Options.GoogleAccessID. +// - private_key_path: Sets path to a private key, which is read and used +// to set Options.PrivateKey. +// Example URL: blob.Open("gs://mybucket") +// // It exposes the following types for As: // Bucket: *storage.Client // ListObject: storage.ObjectAttrs @@ -28,6 +39,7 @@ import ( "fmt" "io" "io/ioutil" + "net/url" "sort" "strings" "time" @@ -37,6 +49,7 @@ import ( "github.com/google/go-cloud/gcp" "cloud.google.com/go/storage" + "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" "google.golang.org/api/option" @@ -44,6 +57,49 @@ import ( const defaultPageSize = 1000 +func init() { + blob.Register("gs", func(ctx context.Context, u *url.URL) (driver.Bucket, error) { + q := u.Query() + opts := &Options{} + + if accessID := q["access_id"]; len(accessID) > 0 { + opts.GoogleAccessID = accessID[0] + } + + if keyPath := q["private_key_path"]; len(keyPath) > 0 { + pk, err := ioutil.ReadFile(keyPath[0]) + if err != nil { + return nil, fmt.Errorf("failed to read private key: %v", err) + } + opts.PrivateKey = pk + } + + var creds *google.Credentials + if credPath := q["cred_path"]; len(credPath) == 0 { + var err error + creds, err = gcp.DefaultCredentials(ctx) + if err != nil { + return nil, err + } + } else { + jsonCreds, err := ioutil.ReadFile(credPath[0]) + if err != nil { + return nil, fmt.Errorf("failed to read credentials: %v", err) + } + creds, err = google.CredentialsFromJSON(ctx, jsonCreds) + if err != nil { + return nil, fmt.Errorf("failed to load credentials: %v", err) + } + } + + client, err := gcp.NewHTTPClient(gcp.DefaultTransport(), gcp.CredentialsTokenSource(creds)) + if err != nil { + return nil, err + } + return openBucket(ctx, u.Host, client, opts) + }) +} + // Options sets options for constructing a *blob.Bucket backed by GCS. type Options struct { // GoogleAccessID represents the authorizer for SignedURL. @@ -62,8 +118,8 @@ type Options struct { SignBytes func([]byte) ([]byte, error) } -// OpenBucket returns a GCS Bucket that communicates using the given HTTP client. -func OpenBucket(ctx context.Context, bucketName string, client *gcp.HTTPClient, opts *Options) (*blob.Bucket, error) { +// openBucket returns a GCS Bucket that communicates using the given HTTP client. +func openBucket(ctx context.Context, bucketName string, client *gcp.HTTPClient, opts *Options) (driver.Bucket, error) { if client == nil { return nil, fmt.Errorf("OpenBucket requires an HTTP client") } @@ -74,7 +130,16 @@ func OpenBucket(ctx context.Context, bucketName string, client *gcp.HTTPClient, if opts == nil { opts = &Options{} } - return blob.NewBucket(&bucket{name: bucketName, client: c, opts: opts}), nil + return &bucket{name: bucketName, client: c, opts: opts}, nil +} + +// OpenBucket returns a GCS Bucket that communicates using the given HTTP client. +func OpenBucket(ctx context.Context, bucketName string, client *gcp.HTTPClient, opts *Options) (*blob.Bucket, error) { + drv, err := openBucket(ctx, bucketName, client, opts) + if err != nil { + return nil, err + } + return blob.NewBucket(drv), nil } // bucket represents a GCS bucket, which handles read, write and delete operations diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index a23acc3f25..2c45ea7b65 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -14,6 +14,12 @@ // Package s3blob provides an implementation of blob using S3. // +// For blob.Open URLs, s3blob registers for the "s3" protocol. +// The URL's Host is used as the bucket name. +// The following query options are supported: +// - region: The AWS region for requests. +// Example URL: blob.Open("s3://mybucket?region=us-east-1") +// // It exposes the following types for As: // Bucket: *s3.S3 // ListObject: s3.Object for objects, s3.CommonPrefix for "directories". @@ -30,6 +36,7 @@ import ( "fmt" "io" "io/ioutil" + "net/url" "sort" "strconv" "strings" @@ -40,22 +47,48 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" ) const defaultPageSize = 1000 -// OpenBucket returns an S3 Bucket. -func OpenBucket(ctx context.Context, sess client.ConfigProvider, bucketName string) (*blob.Bucket, error) { +func init() { + blob.Register("s3", func(ctx context.Context, u *url.URL) (driver.Bucket, error) { + q := u.Query() + cfg := &aws.Config{} + + if region := q["region"]; len(region) > 0 { + cfg.Region = aws.String(region[0]) + } + sess, err := session.NewSession(cfg) + if err != nil { + return nil, err + } + return openBucket(ctx, sess, u.Host) + }) +} + +// openBucket returns an S3 Bucket. +func openBucket(ctx context.Context, sess client.ConfigProvider, bucketName string) (driver.Bucket, error) { if sess == nil { return nil, errors.New("sess must be provided to get bucket") } - return blob.NewBucket(&bucket{ + return &bucket{ name: bucketName, sess: sess, client: s3.New(sess), - }), nil + }, nil +} + +// OpenBucket returns an S3 Bucket. +func OpenBucket(ctx context.Context, sess client.ConfigProvider, bucketName string) (*blob.Bucket, error) { + drv, err := openBucket(ctx, sess, bucketName) + if err != nil { + return nil, err + } + return blob.NewBucket(drv), nil } var emptyBody = ioutil.NopCloser(strings.NewReader("")) diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go index f44bdb6cad..f41826d8f9 100644 --- a/blob/s3blob/s3blob_test.go +++ b/blob/s3blob/s3blob_test.go @@ -59,7 +59,7 @@ func (h *harness) HTTPClient() *http.Client { } func (h *harness) MakeDriver(ctx context.Context) (driver.Bucket, error) { - return &bucket{name: bucketName, sess: h.session, client: s3.New(h.session)}, nil + return openBucket(ctx, h.session, bucketName) } func (h *harness) Close() { From f06e6e79d8e8e6f72bf1a4afdf8da91f7325823d Mon Sep 17 00:00:00 2001 From: Robert van Gent Date: Fri, 2 Nov 2018 09:30:53 -0700 Subject: [PATCH 2/3] clean up error messages, clarify s3 URL docstrings --- blob/gcsblob/gcsblob.go | 6 +++--- blob/s3blob/s3blob.go | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/blob/gcsblob/gcsblob.go b/blob/gcsblob/gcsblob.go index f93663776d..20f7a24bb0 100644 --- a/blob/gcsblob/gcsblob.go +++ b/blob/gcsblob/gcsblob.go @@ -69,7 +69,7 @@ func init() { if keyPath := q["private_key_path"]; len(keyPath) > 0 { pk, err := ioutil.ReadFile(keyPath[0]) if err != nil { - return nil, fmt.Errorf("failed to read private key: %v", err) + return nil, fmt.Errorf("reading private key: %v", err) } opts.PrivateKey = pk } @@ -84,11 +84,11 @@ func init() { } else { jsonCreds, err := ioutil.ReadFile(credPath[0]) if err != nil { - return nil, fmt.Errorf("failed to read credentials: %v", err) + return nil, fmt.Errorf("reading credentials: %v", err) } creds, err = google.CredentialsFromJSON(ctx, jsonCreds) if err != nil { - return nil, fmt.Errorf("failed to load credentials: %v", err) + return nil, fmt.Errorf("loading credentials: %v", err) } } diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index c3d706c57c..ccb5593b35 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -16,8 +16,10 @@ // // For blob.Open URLs, s3blob registers for the "s3" protocol. // The URL's Host is used as the bucket name. +// The AWS session is created as described in +// https://docs.aws.amazon.com/sdk-for-go/api/aws/session/. // The following query options are supported: -// - region: The AWS region for requests. +// - region: The AWS region for requests; sets aws.Config.Region. // Example URL: blob.Open("s3://mybucket?region=us-east-1") // // It exposes the following types for As: From a397d70c4dcc41ab30fec179752ec3edd46a9556 Mon Sep 17 00:00:00 2001 From: Robert van Gent Date: Fri, 2 Nov 2018 10:15:20 -0700 Subject: [PATCH 3/3] minor comment --- blob/s3blob/s3blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index ccb5593b35..b182b7c0e3 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -22,7 +22,7 @@ // - region: The AWS region for requests; sets aws.Config.Region. // Example URL: blob.Open("s3://mybucket?region=us-east-1") // -// It exposes the following types for As: +// s3blob exposes the following types for As: // Bucket: *s3.S3 // ListObject: s3.Object for objects, s3.CommonPrefix for "directories". // ListOptions.BeforeList: *s3.ListObjectsV2Input