From 0e467a4047a2010858eb201e4531e2848c77893a Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Thu, 15 Jun 2023 13:02:29 +0300 Subject: [PATCH 1/8] Add scheme option to bind to both HTTP and HTTPS --- README.md | 35 +++++++++++++++++++---------- internal/config/config.go | 23 +++++++++++++------ internal/config/config_test.go | 28 ++++++++++++++++++++--- main.go | 41 +++++++++++++++++++++------------- 4 files changed, 89 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 3b9e203543..b50d575dc5 100644 --- a/README.md +++ b/README.md @@ -7,18 +7,18 @@ fake-gcs-server provides an emulator for Google Cloud Storage API. It can be used as a library in Go projects and/or as a standalone binary/Docker image. The library is available inside the package -[``github.com/fsouza/fake-gcs-server/fakestorage``](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc) +[`github.com/fsouza/fake-gcs-server/fakestorage`](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc) and can be used from within test suites in Go package. The emulator is available as a binary that can be built manually, downloaded from the [releases page](https://github.com/fsouza/fake-gcs-server/releases) or pulled from Docker -Hub ([``docker pull -fsouza/fake-gcs-server``](https://hub.docker.com/r/fsouza/fake-gcs-server)). +Hub ([`docker pull +fsouza/fake-gcs-server`](https://hub.docker.com/r/fsouza/fake-gcs-server)). ## Using the emulator in Docker You can stub/mock Google Cloud Storage as a standalone server (like the datastore/pubsub emulators) which is ideal for integration tests and/or tests in other languages you may want to run the -``fake-gcs-server`` inside a Docker container: +`fake-gcs-server` inside a Docker container: ```shell docker run -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server @@ -26,14 +26,14 @@ docker run -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server ### Preload data -In case you want to preload some data in ``fake-gcs-server`` just mount a -folder in the container at ``/data``: +In case you want to preload some data in `fake-gcs-server` just mount a +folder in the container at `/data`: ```shell docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server ``` -Where the content of ``${PWD}/examples/data`` is something like: +Where the content of `${PWD}/examples/data` is something like: ``` . @@ -51,13 +51,14 @@ curl --insecure https://0.0.0.0:4443/storage/v1/b/sample-bucket/o {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]} ``` -This will result in one bucket called ``sample-bucket`` containing one object called ``some_file.txt``. +This will result in one bucket called `sample-bucket` containing one object called `some_file.txt`. ### Running with HTTP fake-gcs-server defaults to HTTPS, but it can also be used with HTTP. The flag -`-scheme` can be used to specify the protocol. For example, the previous -example could be changed to pass `-scheme http`: +`-scheme` can be used to specify the protocol. +The binding port willl be `-port` (defaults to `4443`). +For example, the previous example could be changed to pass `-scheme http`: ```shell docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme http @@ -74,6 +75,16 @@ curl http://0.0.0.0:4443/storage/v1/b/sample-bucket/o {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]} ``` +### Running with both HTTPS and HTTP + +To start both HTTPS and HTTP server, pass `-scheme both`. +HTTPS will bind to `-port` (defaults to `4443`) and HTTP will bind to `-port-http` (defaults to `8000`). +For example, the previous example could be changed to pass `-scheme both`: + +```shell +docker run -d --name fake-gcs-server -p 4443:4443 -p 8000:8000 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme both +``` + ### Using with signed URLs It is possible to use fake-gcs-server with signed URLs, although with a few caveats: @@ -97,11 +108,11 @@ docker run --rm fsouza/fake-gcs-server -help ## Client library examples For examples using SDK from multiple languages, check out the -[``examples``](/examples/) directory. +[`examples`](/examples/) directory. ### Building the image locally -You may use ``docker build`` to build the image locally instead of pulling it +You may use `docker build` to build the image locally instead of pulling it from Docker Hub: ```shell diff --git a/internal/config/config.go b/internal/config/config.go index 3302b1778e..62ec0dc862 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -31,6 +31,7 @@ type Config struct { Seed string Host string Port uint + PortHTTP uint CertificateLocation string PrivateKeyLocation string @@ -65,11 +66,12 @@ func Load(args []string) (Config, error) { fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist") fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host") fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running") - fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https") + fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https or both") fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to") fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)") fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist") - fs.UintVar(&cfg.Port, "port", 4443, "port to bind to") + fs.UintVar(&cfg.Port, "port", 4443, "port to bind https or http to, according to scheme, and for https if scheme is 'both'") + fs.UintVar(&cfg.PortHTTP, "port-http", 8000, "used only when scheme is 'both' as port to bind http to") fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic") fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on") fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events") @@ -111,12 +113,15 @@ func (c *Config) validate() error { if c.backend == filesystemBackend && c.fsRoot == "" { return fmt.Errorf("backend %q requires the filesystem-root to be defined", c.backend) } - if c.Scheme != "http" && c.Scheme != "https" { - return fmt.Errorf(`invalid scheme %s, must be either "http"" or "https"`, c.Scheme) + if c.Scheme != "http" && c.Scheme != "https" && c.Scheme != "both" { + return fmt.Errorf(`invalid scheme %s, must be either "http"", "https" or "both"`, c.Scheme) } if c.Port > math.MaxUint16 { return fmt.Errorf("port %d is too high, maximum value is %d", c.Port, math.MaxUint16) } + if c.PortHTTP > math.MaxUint16 { + return fmt.Errorf("port-http %d is too high, maximum value is %d", c.PortHTTP, math.MaxUint16) + } return c.event.validate() } @@ -148,7 +153,7 @@ func (c *EventConfig) validate() error { return nil } -func (c *Config) ToFakeGcsOptions() fakestorage.Options { +func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options { storageRoot := c.fsRoot if c.backend == memoryBackend { storageRoot = "" @@ -173,14 +178,18 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options { } } } + port := c.Port + if c.Scheme == "both" && scheme == "http" { + port = c.PortHTTP + } logger := logrus.New() logger.SetLevel(c.LogLevel) opts := fakestorage.Options{ StorageRoot: storageRoot, Seed: c.Seed, - Scheme: c.Scheme, + Scheme: scheme, Host: c.Host, - Port: uint16(c.Port), + Port: uint16(port), PublicHost: c.publicHost, ExternalURL: c.externalURL, AllowedCORSHeaders: c.allowedCORSHeaders, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9e33018ff4..8a3caad6f2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -32,8 +32,9 @@ func TestLoadConfig(t *testing.T) { "-cors-headers", "X-Goog-Meta-Uploader", "-host", "127.0.0.1", "-port", "443", + "-port-http", "80", "-data", "/var/gcs", - "-scheme", "http", + "-scheme", "both", "-event.pubsub-project-id", "test-project", "-event.pubsub-topic", "gcs-events", "-event.object-prefix", "uploads/", @@ -50,7 +51,8 @@ func TestLoadConfig(t *testing.T) { allowedCORSHeaders: []string{"X-Goog-Meta-Uploader"}, Host: "127.0.0.1", Port: 443, - Scheme: "http", + PortHTTP: 80, + Scheme: "both", event: EventConfig{ pubsubProjectID: "test-project", pubsubTopic: "gcs-events", @@ -72,6 +74,7 @@ func TestLoadConfig(t *testing.T) { allowedCORSHeaders: nil, Host: "0.0.0.0", Port: 4443, + PortHTTP: 8000, Scheme: "https", event: EventConfig{ list: []string{"finalize"}, @@ -85,11 +88,26 @@ func TestLoadConfig(t *testing.T) { args: []string{"-port", "not-a-number"}, expectErr: true, }, + { + name: "invalid port-http value type", + args: []string{"-port-http", "not-a-number"}, + expectErr: true, + }, { name: "invalid port value", args: []string{"-port", "65536"}, expectErr: true, }, + { + name: "invalid port-http value", + args: []string{"-port-http", "65536"}, + expectErr: true, + }, + { + name: "invalid scheme value", + args: []string{"-scheme", "wrong-scheme-value"}, + expectErr: true, + }, { name: "invalid backend", args: []string{"-backend", "in-memory"}, @@ -154,6 +172,7 @@ func TestToFakeGcsOptions(t *testing.T) { externalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", event: EventConfig{ pubsubProjectID: "test-project", pubsubTopic: "gcs-events", @@ -169,6 +188,7 @@ func TestToFakeGcsOptions(t *testing.T) { ExternalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", EventOptions: notification.EventManagerOptions{ ProjectID: "test-project", TopicName: "gcs-events", @@ -193,6 +213,7 @@ func TestToFakeGcsOptions(t *testing.T) { externalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", }, fakestorage.Options{ StorageRoot: "", @@ -200,6 +221,7 @@ func TestToFakeGcsOptions(t *testing.T) { ExternalURL: "https://myhost.example.com:8443", Host: "0.0.0.0", Port: 443, + Scheme: "https", NoListener: true, }, }, @@ -209,7 +231,7 @@ func TestToFakeGcsOptions(t *testing.T) { test := test t.Run(test.name, func(t *testing.T) { t.Parallel() - opts := test.config.ToFakeGcsOptions() + opts := test.config.ToFakeGcsOptions(test.config.Scheme) ignWriter := cmpopts.IgnoreFields(fakestorage.Options{}, "Writer") if diff := cmp.Diff(opts, test.expected, ignWriter); diff != "" { t.Errorf("wrong set of options returned\nwant %#v\ngot %#v\ndiff: %v", test.expected, opts, diff) diff --git a/main.go b/main.go index 83c30bee16..44165bc4e7 100644 --- a/main.go +++ b/main.go @@ -24,27 +24,16 @@ import ( "github.com/sirupsen/logrus" ) -func main() { - cfg, err := config.Load(os.Args[1:]) - if err == flag.ErrHelp { - return - } - if err != nil { - log.Fatal(err) - } - - logger := logrus.New() - logger.SetLevel(cfg.LogLevel) +func startServer(logger *logrus.Logger, cfg *config.Config, scheme string) { + opts := cfg.ToFakeGcsOptions(scheme) - opts := cfg.ToFakeGcsOptions() - - addr := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) + addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port) listener, err := net.Listen("tcp", addr) if err != nil { log.Fatal(err) } - if cfg.Scheme == "https" { + if opts.Scheme == "https" { var tlsConfig *tls.Config if opts.CertificateLocation != "" && opts.PrivateKeyLocation != "" { cert, err := tls.LoadX509KeyPair(opts.CertificateLocation, opts.PrivateKeyLocation) @@ -81,7 +70,27 @@ func main() { })) }() - logger.Infof("server started at %s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port) + logger.Infof("server started at %s://%s:%d", opts.Scheme, opts.Host, opts.Port) +} + +func main() { + cfg, err := config.Load(os.Args[1:]) + if err == flag.ErrHelp { + return + } + if err != nil { + log.Fatal(err) + } + + logger := logrus.New() + logger.SetLevel(cfg.LogLevel) + + if cfg.Scheme != "both" { + startServer(logger, &cfg, cfg.Scheme) + } else { + go startServer(logger, &cfg, "http") + go startServer(logger, &cfg, "https") + } ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) From ea0ba662c5381c1317c13151f231e361cb7e51e9 Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Thu, 15 Jun 2023 13:15:20 +0300 Subject: [PATCH 2/8] changes --- README.md | 6 +++--- main.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b50d575dc5..28bda0d328 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ This will result in one bucket called `sample-bucket` containing one object call fake-gcs-server defaults to HTTPS, but it can also be used with HTTP. The flag `-scheme` can be used to specify the protocol. -The binding port willl be `-port` (defaults to `4443`). +The binding port will be `-port` (defaults to `4443`). For example, the previous example could be changed to pass `-scheme http`: ```shell @@ -77,8 +77,8 @@ curl http://0.0.0.0:4443/storage/v1/b/sample-bucket/o ### Running with both HTTPS and HTTP -To start both HTTPS and HTTP server, pass `-scheme both`. -HTTPS will bind to `-port` (defaults to `4443`) and HTTP will bind to `-port-http` (defaults to `8000`). +To start both HTTPS and HTTP servers, pass `-scheme both`. +HTTPS will bind to `-port` (defaults to `4443`) and HTTP will bind to `-port-http` (defaults to `8000`). For example, the previous example could be changed to pass `-scheme both`: ```shell diff --git a/main.go b/main.go index 44165bc4e7..cacf26cc87 100644 --- a/main.go +++ b/main.go @@ -88,8 +88,8 @@ func main() { if cfg.Scheme != "both" { startServer(logger, &cfg, cfg.Scheme) } else { - go startServer(logger, &cfg, "http") - go startServer(logger, &cfg, "https") + startServer(logger, &cfg, "http") + startServer(logger, &cfg, "https") } ch := make(chan os.Signal, 1) From b053e10e2a8ebc7ee589a82598f0188936cce125 Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Mon, 26 Jun 2023 14:42:16 +0300 Subject: [PATCH 3/8] Change to 2 listeners and a single server --- main.go | 58 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index cacf26cc87..b7bb040c14 100644 --- a/main.go +++ b/main.go @@ -24,7 +24,7 @@ import ( "github.com/sirupsen/logrus" ) -func startServer(logger *logrus.Logger, cfg *config.Config, scheme string) { +func createListener(logger *logrus.Logger, cfg *config.Config, scheme string) (net.Listener, *fakestorage.Options) { opts := cfg.ToFakeGcsOptions(scheme) addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port) @@ -51,26 +51,51 @@ func startServer(logger *logrus.Logger, cfg *config.Config, scheme string) { listener = tls.NewListener(listener, tlsConfig) } + return listener, &opts +} + +func startServer(logger *logrus.Logger, cfg *config.Config) { + type listenerAndOpts struct { + listener net.Listener + opts *fakestorage.Options + } + + listenersAndOpts := make([]listenerAndOpts, 2) + + if cfg.Scheme != "both" { + listener, opts := createListener(logger, cfg, cfg.Scheme) + listenersAndOpts[0] = listenerAndOpts{listener, opts} + } else { + listener, opts := createListener(logger, cfg, "http") + listenersAndOpts[0] = listenerAndOpts{listener, opts} + listener, opts = createListener(logger, cfg, "https") + listenersAndOpts[1] = listenerAndOpts{listener, opts} + } + addMimeTypes() - httpServer, err := fakestorage.NewServerWithOptions(opts) + httpServer, err := fakestorage.NewServerWithOptions(*listenersAndOpts[1].opts) if err != nil { logger.WithError(err).Fatal("couldn't start the server") } grpcServer := grpc.NewServerWithBackend(httpServer.Backend()) - go func() { - http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.ProtoMajor == 2 && strings.HasPrefix( - r.Header.Get("Content-Type"), "application/grpc") { - grpcServer.ServeHTTP(w, r) - } else { - httpServer.HTTPHandler().ServeHTTP(w, r) - } - })) - }() - logger.Infof("server started at %s://%s:%d", opts.Scheme, opts.Host, opts.Port) + for _, listenerAndOpts := range listenersAndOpts { + go func(listener net.Listener) { + http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ProtoMajor == 2 && strings.HasPrefix( + r.Header.Get("Content-Type"), "application/grpc") { + grpcServer.ServeHTTP(w, r) + } else { + httpServer.HTTPHandler().ServeHTTP(w, r) + } + })) + }(listenerAndOpts.listener) + + logger.Infof("server started at %s://%s:%d", + listenerAndOpts.opts.Scheme, listenerAndOpts.opts.Host, listenerAndOpts.opts.Port) + } } func main() { @@ -85,12 +110,7 @@ func main() { logger := logrus.New() logger.SetLevel(cfg.LogLevel) - if cfg.Scheme != "both" { - startServer(logger, &cfg, cfg.Scheme) - } else { - startServer(logger, &cfg, "http") - startServer(logger, &cfg, "https") - } + startServer(logger, &cfg) ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt, syscall.SIGTERM) From d708fee0c0141f385fa1bc2d5f12b3a419b86f12 Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Sun, 30 Jul 2023 10:18:17 +0300 Subject: [PATCH 4/8] fix --- internal/config/config.go | 2 +- main.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 62ec0dc862..daf1c5463a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -179,7 +179,7 @@ func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options { } } port := c.Port - if c.Scheme == "both" && scheme == "http" { + if scheme == "http" { port = c.PortHTTP } logger := logrus.New() diff --git a/main.go b/main.go index b7bb040c14..58d26964e6 100644 --- a/main.go +++ b/main.go @@ -60,12 +60,14 @@ func startServer(logger *logrus.Logger, cfg *config.Config) { opts *fakestorage.Options } - listenersAndOpts := make([]listenerAndOpts, 2) + var listenersAndOpts []listenerAndOpts if cfg.Scheme != "both" { + listenersAndOpts = make([]listenerAndOpts, 1) listener, opts := createListener(logger, cfg, cfg.Scheme) listenersAndOpts[0] = listenerAndOpts{listener, opts} } else { + listenersAndOpts = make([]listenerAndOpts, 2) listener, opts := createListener(logger, cfg, "http") listenersAndOpts[0] = listenerAndOpts{listener, opts} listener, opts = createListener(logger, cfg, "https") @@ -74,7 +76,7 @@ func startServer(logger *logrus.Logger, cfg *config.Config) { addMimeTypes() - httpServer, err := fakestorage.NewServerWithOptions(*listenersAndOpts[1].opts) + httpServer, err := fakestorage.NewServerWithOptions(*listenersAndOpts[0].opts) if err != nil { logger.WithError(err).Fatal("couldn't start the server") } From 46a6906dcb550f72fd5d8fc3bec13ca96662eb7b Mon Sep 17 00:00:00 2001 From: Raz Amir <88726761+ramir-savvy@users.noreply.github.com> Date: Mon, 31 Jul 2023 09:26:15 +0300 Subject: [PATCH 5/8] Apply suggestions from code review Co-authored-by: fsouza <108725+fsouza@users.noreply.github.com> --- main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 58d26964e6..437444290d 100644 --- a/main.go +++ b/main.go @@ -30,7 +30,7 @@ func createListener(logger *logrus.Logger, cfg *config.Config, scheme string) (n addr := fmt.Sprintf("%s:%d", opts.Host, opts.Port) listener, err := net.Listen("tcp", addr) if err != nil { - log.Fatal(err) + logger.Fatal(err) } if opts.Scheme == "https" { @@ -63,15 +63,15 @@ func startServer(logger *logrus.Logger, cfg *config.Config) { var listenersAndOpts []listenerAndOpts if cfg.Scheme != "both" { - listenersAndOpts = make([]listenerAndOpts, 1) listener, opts := createListener(logger, cfg, cfg.Scheme) - listenersAndOpts[0] = listenerAndOpts{listener, opts} + listenersAndOpts = []listenerAndOpts{{listener, opts}} } else { - listenersAndOpts = make([]listenerAndOpts, 2) - listener, opts := createListener(logger, cfg, "http") - listenersAndOpts[0] = listenerAndOpts{listener, opts} - listener, opts = createListener(logger, cfg, "https") - listenersAndOpts[1] = listenerAndOpts{listener, opts} + httpListener, httpOpts := createListener(logger, cfg, "http") + httpsListener, httpsOpts := createListener(logger, cfg, "https") + listenersAndOpts = []listenerAndOpts{ + {httpListener, httpOpts}, + {httpsListener, httpsOpts}, + } } addMimeTypes() From 9cffba94198f71508d8b194f50e08de6c217df92 Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Mon, 31 Jul 2023 09:57:33 +0300 Subject: [PATCH 6/8] fix for default ports handling --- internal/config/config.go | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index daf1c5463a..c5516acc7b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,6 +24,8 @@ const ( eventDelete = "delete" eventMetadataUpdate = "metadataUpdate" eventArchive = "archive" + defaultHTTPSPort = 4443 + defaultHTTPPort = 8000 ) type Config struct { @@ -66,12 +68,12 @@ func Load(args []string) (Config, error) { fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist") fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host") fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running") - fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https or both") + fs.StringVar(&cfg.Scheme, "scheme", "https", "using 'http' or 'https' or 'both'") fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to") fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)") fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist") - fs.UintVar(&cfg.Port, "port", 4443, "port to bind https or http to, according to scheme, and for https if scheme is 'both'") - fs.UintVar(&cfg.PortHTTP, "port-http", 8000, "used only when scheme is 'both' as port to bind http to") + fs.UintVar(&cfg.Port, "port", 0, "port to which https (default 4443) or http (default 8000) will be bound, based on the specified scheme. If the scheme is 'both', then bind to https") + fs.UintVar(&cfg.PortHTTP, "port-http", 0, "used only when scheme is 'both' as the port to bind http to (default 8000)") fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic") fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on") fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events") @@ -87,6 +89,19 @@ func Load(args []string) (Config, error) { return cfg, err } + // setting default values, if not provided, for port and http ports based on scheme value + if cfg.Port == 0 { + if cfg.Scheme == "https" || cfg.Scheme == "both" { + cfg.Port = defaultHTTPSPort + } else if cfg.Scheme == "http" { + cfg.Port = defaultHTTPPort + } + } + + if cfg.PortHTTP == 0 && cfg.Scheme == "both" { + cfg.PortHTTP = defaultHTTPPort + } + cfg.LogLevel, err = logrus.ParseLevel(logLevel) if err != nil { return cfg, err @@ -179,8 +194,8 @@ func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options { } } port := c.Port - if scheme == "http" { - port = c.PortHTTP + if c.Scheme == "both" && scheme == "http" { + port = c.PortHTTP // this cli flag, for port http, is relevant only when scheme is both } logger := logrus.New() logger.SetLevel(c.LogLevel) From 8614c048c3871573dd9474a9769745579f54293a Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Mon, 31 Jul 2023 11:55:45 +0300 Subject: [PATCH 7/8] check if flags default value using visit --- internal/config/config.go | 33 +++++++++++++++++++++++---------- internal/config/config_test.go | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index c5516acc7b..1850903a76 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -26,6 +26,11 @@ const ( eventArchive = "archive" defaultHTTPSPort = 4443 defaultHTTPPort = 8000 + schemeHTTPS = "https" + schemeHTTP = "http" + schemeBoth = "both" + flagPort = "port" + flagPortHTTP = "port-http" ) type Config struct { @@ -68,12 +73,12 @@ func Load(args []string) (Config, error) { fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist") fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host") fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running") - fs.StringVar(&cfg.Scheme, "scheme", "https", "using 'http' or 'https' or 'both'") + fs.StringVar(&cfg.Scheme, "scheme", schemeHTTPS, "using 'http' or 'https' or 'both'") fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to") fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)") fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist") - fs.UintVar(&cfg.Port, "port", 0, "port to which https (default 4443) or http (default 8000) will be bound, based on the specified scheme. If the scheme is 'both', then bind to https") - fs.UintVar(&cfg.PortHTTP, "port-http", 0, "used only when scheme is 'both' as the port to bind http to (default 8000)") + fs.UintVar(&cfg.Port, flagPort, 0, "port to which https (default 4443) or http (default 8000) will be bound, based on the specified scheme. If the scheme is 'both', then bind to https") + fs.UintVar(&cfg.PortHTTP, flagPortHTTP, 0, "used only when scheme is 'both' as the port to bind http to (default 8000)") fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic") fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on") fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events") @@ -89,16 +94,24 @@ func Load(args []string) (Config, error) { return cfg, err } + // Create a map to store the flags and their values + setFlags := make(map[string]interface{}) + + // Check if a flag was used using Visit + fs.Visit(func(f *flag.Flag) { + setFlags[f.Name] = f.Value + }) + // setting default values, if not provided, for port and http ports based on scheme value - if cfg.Port == 0 { - if cfg.Scheme == "https" || cfg.Scheme == "both" { + if _, ok := setFlags[flagPort]; !ok { + if cfg.Scheme == schemeHTTPS || cfg.Scheme == schemeBoth { cfg.Port = defaultHTTPSPort - } else if cfg.Scheme == "http" { + } else if cfg.Scheme == schemeHTTP { cfg.Port = defaultHTTPPort } } - if cfg.PortHTTP == 0 && cfg.Scheme == "both" { + if _, ok := setFlags[flagPortHTTP]; !ok && cfg.Scheme == schemeBoth { cfg.PortHTTP = defaultHTTPPort } @@ -128,8 +141,8 @@ func (c *Config) validate() error { if c.backend == filesystemBackend && c.fsRoot == "" { return fmt.Errorf("backend %q requires the filesystem-root to be defined", c.backend) } - if c.Scheme != "http" && c.Scheme != "https" && c.Scheme != "both" { - return fmt.Errorf(`invalid scheme %s, must be either "http"", "https" or "both"`, c.Scheme) + if c.Scheme != schemeHTTP && c.Scheme != schemeHTTPS && c.Scheme != schemeBoth { + return fmt.Errorf(`invalid scheme %s, must be either "%s", "%s" or "%s"`, c.Scheme, schemeHTTP, schemeHTTPS, schemeBoth) } if c.Port > math.MaxUint16 { return fmt.Errorf("port %d is too high, maximum value is %d", c.Port, math.MaxUint16) @@ -194,7 +207,7 @@ func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options { } } port := c.Port - if c.Scheme == "both" && scheme == "http" { + if c.Scheme == schemeBoth && scheme == schemeHTTP { port = c.PortHTTP // this cli flag, for port http, is relevant only when scheme is both } logger := logrus.New() diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 8a3caad6f2..ba86cd484d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -74,7 +74,7 @@ func TestLoadConfig(t *testing.T) { allowedCORSHeaders: nil, Host: "0.0.0.0", Port: 4443, - PortHTTP: 8000, + PortHTTP: 0, Scheme: "https", event: EventConfig{ list: []string{"finalize"}, From da920f7c3467d42fa59a60f09c6e754da88b7676 Mon Sep 17 00:00:00 2001 From: Raz Amir Date: Mon, 31 Jul 2023 13:14:40 +0300 Subject: [PATCH 8/8] add example test runs and unit tests --- .github/workflows/main.yml | 6 + ...-curl-both-scheme-default-ports-example.sh | 11 + ...l-both-scheme-non-default-ports-example.sh | 11 + internal/config/config.go | 7 +- internal/config/config_test.go | 190 ++++++++++++++++++ 5 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 ci/run-curl-both-scheme-default-ports-example.sh create mode 100644 ci/run-curl-both-scheme-non-default-ports-example.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09cabd7efb..5c0fd0893a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -146,6 +146,12 @@ jobs: - lang: curl docker-image: alpine entrypoint: /bin/sh + - lang: curl-both-scheme-default-ports + docker-image: alpine + entrypoint: /bin/sh + - lang: curl-both-scheme-non-default-ports + docker-image: alpine + entrypoint: /bin/sh name: test-${{ matrix.lang }}-example runs-on: ubuntu-latest diff --git a/ci/run-curl-both-scheme-default-ports-example.sh b/ci/run-curl-both-scheme-default-ports-example.sh new file mode 100644 index 0000000000..b9ac4d038a --- /dev/null +++ b/ci/run-curl-both-scheme-default-ports-example.sh @@ -0,0 +1,11 @@ +# Copyright 2023 Francisco Souza. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +./fake-gcs-server -backend memory -scheme both -data ${PWD}/examples/data & + +apk add --update curl +curl --silent --fail --insecure https://0.0.0.0:4443/storage/v1/b +curl --silent --fail --insecure http://0.0.0.0:8000/storage/v1/b diff --git a/ci/run-curl-both-scheme-non-default-ports-example.sh b/ci/run-curl-both-scheme-non-default-ports-example.sh new file mode 100644 index 0000000000..4b53bafffe --- /dev/null +++ b/ci/run-curl-both-scheme-non-default-ports-example.sh @@ -0,0 +1,11 @@ +# Copyright 2023 Francisco Souza. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -e + +./fake-gcs-server -backend memory -port 5553 -port-http 9000 -scheme both -data ${PWD}/examples/data & + +apk add --update curl +curl --silent --fail --insecure https://0.0.0.0:5553/storage/v1/b +curl --silent --fail --insecure http://0.0.0.0:9000/storage/v1/b diff --git a/internal/config/config.go b/internal/config/config.go index 1850903a76..24b2c41336 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -128,7 +128,12 @@ func Load(args []string) (Config, error) { } if cfg.externalURL == "" { - cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port) + if cfg.Scheme != "both" { + cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port) + } else { + // for scheme 'both' taking externalURL as HTTPs by default + cfg.externalURL = fmt.Sprintf("%s://%s:%d", schemeHTTPS, cfg.Host, cfg.Port) + } } return cfg, cfg.validate() diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ba86cd484d..ca391d5570 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -83,6 +83,196 @@ func TestLoadConfig(t *testing.T) { LogLevel: logrus.InfoLevel, }, }, + { + name: "https scheme and default port", + args: []string{ + "-scheme", "https", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 4443, + PortHTTP: 0, + Scheme: "https", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "http scheme and default port", + args: []string{ + "-scheme", "http", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "http://0.0.0.0:8000", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 8000, + PortHTTP: 0, + Scheme: "http", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "https scheme and non-default port", + args: []string{ + "-port", "5553", + "-scheme", "https", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:5553", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 5553, + PortHTTP: 0, + Scheme: "https", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "http scheme and non-default port", + args: []string{ + "-port", "9000", + "-scheme", "http", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "http://0.0.0.0:9000", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 9000, + PortHTTP: 0, + Scheme: "http", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme and default ports", + args: []string{ + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 4443, + PortHTTP: 8000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme with non-default https port and default http", + args: []string{ + "-port", "5553", + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:5553", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 5553, + PortHTTP: 8000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme with default https port and non-default http", + args: []string{ + "-port-http", "9000", + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:4443", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 4443, + PortHTTP: 9000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, + { + name: "'both' scheme with non-default ports", + args: []string{ + "-port", "5553", + "-port-http", "9000", + "-scheme", "both", + }, + expectedConfig: Config{ + Seed: "", + backend: "filesystem", + fsRoot: "/storage", + publicHost: "storage.googleapis.com", + externalURL: "https://0.0.0.0:5553", + allowedCORSHeaders: nil, + Host: "0.0.0.0", + Port: 5553, + PortHTTP: 9000, + Scheme: "both", + event: EventConfig{ + list: []string{"finalize"}, + }, + bucketLocation: "US-CENTRAL1", + LogLevel: logrus.InfoLevel, + }, + }, { name: "invalid port value type", args: []string{"-port", "not-a-number"},