Add scheme option to bind to both HTTP and HTTPS

.github/workflows/main.yml

@@ -146,6 +146,12 @@ jobs:
  - lang: curl
  docker-image: alpine
  entrypoint: /bin/sh
+ - lang: curl-both-scheme-default-ports
+ docker-image: alpine
+ entrypoint: /bin/sh
+ - lang: curl-both-scheme-non-default-ports
+ docker-image: alpine
+ entrypoint: /bin/sh
 
  name: test-${{ matrix.lang }}-example
  runs-on: ubuntu-latest

README.md

@@ -7,33 +7,33 @@ fake-gcs-server provides an emulator for Google Cloud Storage API. It can be
 used as a library in Go projects and/or as a standalone binary/Docker image.
 
 The library is available inside the package
-[``github.com/fsouza/fake-gcs-server/fakestorage``](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc)
+[`github.com/fsouza/fake-gcs-server/fakestorage`](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc)
 and can be used from within test suites in Go package. The emulator is
 available as a binary that can be built manually, downloaded from the [releases
 page](https://github.com/fsouza/fake-gcs-server/releases) or pulled from Docker
-Hub ([``docker pull
-fsouza/fake-gcs-server``](https://hub.docker.com/r/fsouza/fake-gcs-server)).
+Hub ([`docker pull
+fsouza/fake-gcs-server`](https://hub.docker.com/r/fsouza/fake-gcs-server)).
 
 ## Using the emulator in Docker
 
 You can stub/mock Google Cloud Storage as a standalone server (like the datastore/pubsub emulators)
 which is ideal for integration tests and/or tests in other languages you may want to run the
-``fake-gcs-server`` inside a Docker container:
+`fake-gcs-server` inside a Docker container:
 
 ```shell
 docker run -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server
 ```
 
 ### Preload data
 
-In case you want to preload some data in ``fake-gcs-server`` just mount a
-folder in the container at ``/data``:
+In case you want to preload some data in `fake-gcs-server` just mount a
+folder in the container at `/data`:
 
 ```shell
 docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server
 ```
 
-Where the content of ``${PWD}/examples/data`` is something like:
+Where the content of `${PWD}/examples/data` is something like:
 
 ```
 .
@@ -51,13 +51,14 @@ curl --insecure https://0.0.0.0:4443/storage/v1/b/sample-bucket/o
 {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]}
 ```
 
-This will result in one bucket called ``sample-bucket`` containing one object called ``some_file.txt``.
+This will result in one bucket called `sample-bucket` containing one object called `some_file.txt`.
 
 ### Running with HTTP
 
 fake-gcs-server defaults to HTTPS, but it can also be used with HTTP. The flag
-`-scheme` can be used to specify the protocol. For example, the previous
-example could be changed to pass `-scheme http`:
+`-scheme` can be used to specify the protocol. 
+The binding port will be `-port` (defaults to `4443`). 
+For example, the previous example could be changed to pass `-scheme http`:
 
 ```shell
 docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme http
@@ -74,6 +75,16 @@ curl http://0.0.0.0:4443/storage/v1/b/sample-bucket/o
 {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]}
 ```
 
+### Running with both HTTPS and HTTP
+
+To start both HTTPS and HTTP servers, pass `-scheme both`. 
+HTTPS will bind to `-port` (defaults to `4443`) and HTTP will bind to `-port-http` (defaults to `8000`). 
+For example, the previous example could be changed to pass `-scheme both`:
+
+```shell
+docker run -d --name fake-gcs-server -p 4443:4443 -p 8000:8000 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme both
+```
+
 ### Using with signed URLs
 
 It is possible to use fake-gcs-server with signed URLs, although with a few caveats:
@@ -97,11 +108,11 @@ docker run --rm fsouza/fake-gcs-server -help
 ## Client library examples
 
 For examples using SDK from multiple languages, check out the
-[``examples``](/examples/) directory.
+[`examples`](/examples/) directory.
 
 ### Building the image locally
 
-You may use ``docker build`` to build the image locally instead of pulling it
+You may use `docker build` to build the image locally instead of pulling it
 from Docker Hub:
 
 ```shell

ci/run-curl-both-scheme-default-ports-example.sh

@@ -0,0 +1,11 @@
+# Copyright 2023 Francisco Souza. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -e
+
+./fake-gcs-server -backend memory -scheme both -data ${PWD}/examples/data &
+
+apk add --update curl
+curl --silent --fail --insecure https://0.0.0.0:4443/storage/v1/b
+curl --silent --fail --insecure http://0.0.0.0:8000/storage/v1/b

ci/run-curl-both-scheme-non-default-ports-example.sh

@@ -0,0 +1,11 @@
+# Copyright 2023 Francisco Souza. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -e
+
+./fake-gcs-server -backend memory -port 5553 -port-http 9000 -scheme both -data ${PWD}/examples/data &
+
+apk add --update curl
+curl --silent --fail --insecure https://0.0.0.0:5553/storage/v1/b
+curl --silent --fail --insecure http://0.0.0.0:9000/storage/v1/b

internal/config/config.go

@@ -24,13 +24,21 @@ const (
  eventDelete = "delete"
  eventMetadataUpdate = "metadataUpdate"
  eventArchive = "archive"
+ defaultHTTPSPort = 4443
+ defaultHTTPPort = 8000
+ schemeHTTPS = "https"
+ schemeHTTP = "http"
+ schemeBoth = "both"
+ flagPort = "port"
+ flagPortHTTP = "port-http"
 )
 
 type Config struct {
  Scheme string
  Seed string
  Host string
  Port uint
+ PortHTTP uint
  CertificateLocation string
  PrivateKeyLocation string
 
@@ -65,11 +73,12 @@ func Load(args []string) (Config, error) {
  fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist")
  fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host")
  fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running")
- fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https")
+ fs.StringVar(&cfg.Scheme, "scheme", schemeHTTPS, "using 'http' or 'https' or 'both'")
  fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to")
  fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)")
  fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist")
- fs.UintVar(&cfg.Port, "port", 4443, "port to bind to")
+ fs.UintVar(&cfg.Port, flagPort, 0, "port to which https (default 4443) or http (default 8000) will be bound, based on the specified scheme. If the scheme is 'both', then bind to https")
+ fs.UintVar(&cfg.PortHTTP, flagPortHTTP, 0, "used only when scheme is 'both' as the port to bind http to (default 8000)")
  fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic")
  fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on")
  fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events")
@@ -85,6 +94,27 @@ func Load(args []string) (Config, error) {
  return cfg, err
  }
 
+ // Create a map to store the flags and their values
+ setFlags := make(map[string]interface{})
+
+ // Check if a flag was used using Visit
+ fs.Visit(func(f *flag.Flag) {
+ setFlags[f.Name] = f.Value
+ })
+
+ // setting default values, if not provided, for port and http ports based on scheme value
+ if _, ok := setFlags[flagPort]; !ok {
+ if cfg.Scheme == schemeHTTPS || cfg.Scheme == schemeBoth {
+ cfg.Port = defaultHTTPSPort
+ } else if cfg.Scheme == schemeHTTP {
+ cfg.Port = defaultHTTPPort
+ }
+ }
+
+ if _, ok := setFlags[flagPortHTTP]; !ok && cfg.Scheme == schemeBoth {
+ cfg.PortHTTP = defaultHTTPPort
+ }
+
  cfg.LogLevel, err = logrus.ParseLevel(logLevel)
  if err != nil {
  return cfg, err
@@ -98,7 +128,12 @@ func Load(args []string) (Config, error) {
  }
 
  if cfg.externalURL == "" {
- cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port)
+ if cfg.Scheme != "both" {
+ cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port)
+ } else {
+ // for scheme 'both' taking externalURL as HTTPs by default
+ cfg.externalURL = fmt.Sprintf("%s://%s:%d", schemeHTTPS, cfg.Host, cfg.Port)
+ }
  }
 
  return cfg, cfg.validate()
@@ -111,12 +146,15 @@ func (c *Config) validate() error {
  if c.backend == filesystemBackend && c.fsRoot == "" {
  return fmt.Errorf("backend %q requires the filesystem-root to be defined", c.backend)
  }
- if c.Scheme != "http" && c.Scheme != "https" {
- return fmt.Errorf(`invalid scheme %s, must be either "http"" or "https"`, c.Scheme)
+ if c.Scheme != schemeHTTP && c.Scheme != schemeHTTPS && c.Scheme != schemeBoth {
+ return fmt.Errorf(`invalid scheme %s, must be either "%s", "%s" or "%s"`, c.Scheme, schemeHTTP, schemeHTTPS, schemeBoth)
  }
  if c.Port > math.MaxUint16 {
  return fmt.Errorf("port %d is too high, maximum value is %d", c.Port, math.MaxUint16)
  }
+ if c.PortHTTP > math.MaxUint16 {
+ return fmt.Errorf("port-http %d is too high, maximum value is %d", c.PortHTTP, math.MaxUint16)
+ }
 
  return c.event.validate()
 }
@@ -148,7 +186,7 @@ func (c *EventConfig) validate() error {
  return nil
 }
 
-func (c *Config) ToFakeGcsOptions() fakestorage.Options {
+func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options {
  storageRoot := c.fsRoot
  if c.backend == memoryBackend {
  storageRoot = ""
@@ -173,14 +211,18 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options {
  }
  }
  }
+ port := c.Port
+ if c.Scheme == schemeBoth && scheme == schemeHTTP {
+ port = c.PortHTTP // this cli flag, for port http, is relevant only when scheme is both
+ }
  logger := logrus.New()
  logger.SetLevel(c.LogLevel)
  opts := fakestorage.Options{
  StorageRoot: storageRoot,
  Seed: c.Seed,
- Scheme: c.Scheme,
+ Scheme: scheme,
  Host: c.Host,
- Port: uint16(c.Port),
+ Port: uint16(port),
  PublicHost: c.publicHost,
  ExternalURL: c.externalURL,
  AllowedCORSHeaders: c.allowedCORSHeaders,