support the s3 endpoint config option

Dockerfile

@@ -36,7 +36,12 @@ RUN apt-get purge -y --auto-remove $BUILD_DEPS && \
 COPY rootfs /
 ENV WALE_ENVDIR=/etc/wal-e.d/env
 RUN mkdir -p $WALE_ENVDIR
-RUN python3 /patcher-script.py
+
+ARG PATCH_CMD="python3 /patcher-script.py"
+RUN $PATCH_CMD file /bin/create_bucket /patcher-script.d/patch_boto_s3.py
+RUN $PATCH_CMD file /usr/local/bin/wal-e /patcher-script.d/patch_boto_s3.py
+RUN $PATCH_CMD module wal_e.worker.worker_util /patcher-script.d/patch_wal_e_s3.py
+
 
 CMD ["/docker-entrypoint.sh", "postgres"]
 EXPOSE 5432

rootfs/bin/create_bucket

@@ -12,6 +12,7 @@ from oauth2client.service_account import ServiceAccountCredentials
 from gcloud.storage.client import Client
 from gcloud import exceptions
 from azure.storage.blob import BlobService
+from urllib.parse import urlparse
 
 def bucket_exists(conn, name):
     bucket = conn.lookup(name)
@@ -20,28 +21,25 @@ def bucket_exists(conn, name):
     return True
 
 bucket_name = os.getenv('BUCKET_NAME')
-region = os.getenv('AWS_REGION')
+region = os.getenv('S3_REGION')
 
 if os.getenv('DATABASE_STORAGE') == "s3":
-    conn = boto.s3.connect_to_region(region)
+    if os.getenv('S3_ENDPOINT'):
+        endpoint = urlparse(os.getenv('S3_ENDPOINT'))
+        conn = boto.s3.connect_to_region(region,
+            host=endpoint.hostname,
+            port=endpoint.port,
+            path=endpoint.path,
+            calling_format=boto.s3.connection.OrdinaryCallingFormat())
+    else:
+        conn = boto.s3.connect_to_region(region)
+
     if not bucket_exists(conn, bucket_name):
-        try:
-            if region == "us-east-1":
-                # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125
-                conn.create_bucket(bucket_name)
-            else:
-                conn.create_bucket(bucket_name, location=region)
-        # NOTE(bacongobbler): for versions prior to v2.9.0, the bucket is created in the default region.
-        # if we got here, we need to propagate "us-east-1" into WALE_S3_ENDPOINT because the bucket
-        # exists in a different region and we cannot find it.
-        # TODO(bacongobbler): deprecate this once we drop support for v2.8.0 and lower
-        except S3CreateError as err:
-            if region != 'us-east-1':
-                print('Failed to create bucket in {}. We are now assuming that the bucket was created in us-east-1.'.format(region))
-                with open(os.path.join(os.environ['WALE_ENVDIR'], "WALE_S3_ENDPOINT"), "w+") as file:
-                    file.write('https+path://s3.amazonaws.com:443')
-            else:
-                raise
+        if region == "us-east-1":
+            # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125
+            conn.create_bucket(bucket_name)
+        else:
+            conn.create_bucket(bucket_name, location=region)
 
 elif os.getenv('DATABASE_STORAGE') == "gcs":
     scopes = ['https://www.googleapis.com/auth/devstorage.full_control']
@@ -76,15 +74,17 @@ elif os.getenv('DATABASE_STORAGE') == "swift":
     conn.put_container(os.getenv('BUCKET_NAME'))
 
 else:
-    botoconfig.add_section('s3')
+    if not botoconfig.has_section("s3"):
+        botoconfig.add_section('s3')
     botoconfig.set('s3', 'use-sigv4', 'True')
-    botoconfig.add_section('Boto')
+    if not botoconfig.has_section("Boto"):
+        botoconfig.add_section('Boto')
     botoconfig.set('Boto', 'is_secure', 'False')
     conn = S3Connection(
         host=os.getenv('S3_HOST'),
         port=int(os.getenv('S3_PORT')),
         calling_format=OrdinaryCallingFormat())
     # HACK(bacongobbler): allow boto to connect to minio by changing the region name for s3v4 auth
-    conn.auth_region_name = os.getenv('AWS_REGION')
+    conn.auth_region_name = os.getenv('S3_REGION')
     if not bucket_exists(conn, bucket_name):
         conn.create_bucket(bucket_name)

rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh

@@ -7,15 +7,21 @@ if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
   AWS_SECRET_ACCESS_KEY=$(cat /var/run/secrets/deis/objectstore/creds/secretkey)
   if [[ "$DATABASE_STORAGE" == "s3" ]]; then
     AWS_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
+    S3_ENDPOINT=$(cat /var/run/secrets/deis/objectstore/creds/endpoint)
     BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
-    # Convert $AWS_REGION into $WALE_S3_ENDPOINT to avoid "Connection reset by peer" from
-    # regions other than us-standard.
-    # See https://github.com/wal-e/wal-e/issues/167
-    # See https://github.com/boto/boto/issues/2207
-    if [[ "$AWS_REGION" == "us-east-1" ]]; then
-      echo "https+path://s3.amazonaws.com:443" > WALE_S3_ENDPOINT
+    if [[ "$S3_ENDPOINT" == "" ]]; then
+        # Convert $AWS_REGION into $WALE_S3_ENDPOINT to avoid "Connection reset by peer" from
+        # regions other than us-standard.
+        # See https://github.com/wal-e/wal-e/issues/167
+        # See https://github.com/boto/boto/issues/2207
+        if [[ "$AWS_REGION" == "us-east-1" ]]; then
+          echo "https+path://s3.amazonaws.com:443" > WALE_S3_ENDPOINT
+        else
+          echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
+        fi
     else
-      echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
+        echo "$S3_ENDPOINT" > S3_ENDPOINT
+        echo "$S3_ENDPOINT" | sed -E -e 's!http(s?)://!http\1+path://!' -e 's!/$!!' > WALE_S3_ENDPOINT
     fi
   else
     AWS_REGION="us-east-1"
@@ -36,7 +42,7 @@ if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
   else
     echo "1" > AWS_INSTANCE_PROFILE
   fi
-  echo $AWS_REGION > AWS_REGION
+  echo $AWS_REGION > S3_REGION
   echo $BUCKET_NAME > BUCKET_NAME
 elif [ "$DATABASE_STORAGE" == "gcs" ]; then
   GOOGLE_APPLICATION_CREDENTIALS="/var/run/secrets/deis/objectstore/creds/key.json"

rootfs/patcher-script.d/patch_boto_s3.py

@@ -0,0 +1,9 @@
+def patch_boto_s3_hmac_auth_v4_handler():
+    import os
+    from boto.auth import HmacAuthV4Handler
+    _init = HmacAuthV4Handler.__init__
+    def wrap_init(self, *args, **kwargs):
+        _init(self, *args, **kwargs)
+        self.region_name = os.getenv('S3_REGION', self.region_name)
+    HmacAuthV4Handler.__init__ = wrap_init
+patch_boto_s3_hmac_auth_v4_handler()

rootfs/patcher-script.d/patch_wal_e_s3.py

@@ -1,12 +1,18 @@
-def patch_wal_e_hmac_auth_v4_handler():
+def patch_uri_put_file():
     import os
-    from boto.auth import HmacAuthV4Handler
-    _init = HmacAuthV4Handler.__init__
-    def wrap_init(self, *args, **kwargs):
-        _init(self, *args, **kwargs)
-        self.region_name = os.getenv('AWS_REGION', self.region_name)
-    HmacAuthV4Handler.__init__ = wrap_init
-
-
-if __name__ == '__main__':
-    patch_wal_e_hmac_auth_v4_handler()
+    from wal_e.blobstore import s3
+    from wal_e.blobstore.s3 import s3_util
+    def wrap_uri_put_file(creds, uri, fp, content_type=None, conn=None):
+        assert fp.tell() == 0
+        k = s3_util._uri_to_key(creds, uri, conn=conn)
+        if content_type is not None:
+            k.content_type = content_type
+        if os.getenv('DATABASE_STORAGE') == 's3':
+            encrypt_key=True
+        else:
+            encrypt_key=False
+        k.set_contents_from_file(fp, encrypt_key=encrypt_key)
+        return k
+    s3.uri_put_file = wrap_uri_put_file
+    s3_util.uri_put_file = wrap_uri_put_file
+patch_uri_put_file()

rootfs/patcher-script.py

@@ -1,32 +1,41 @@
-patch_script = """
+import sys
+
+patch_script_tmp = """
 
 def run_patch_scripts(patch_script_path):
-    import os
-    for patch in os.listdir(patch_script_path):
-        full_patch_file = os.path.join(patch_script_path, patch)
-        if full_patch_file.endswith('.py') and os.path.isfile(full_patch_file):
-            with open(full_patch_file, 'r') as f:
-                try:
-                    exec(f.read())
-                except:
-                    pass
-run_patch_scripts('/patcher-script.d')
+    with open(patch_script_path, 'r') as f:
+        try:
+            exec(f.read())
+        except:
+            pass
+run_patch_scripts("%s")
 
 """
 
 
-def main():
+def main(patch_file, patch_script_file):
     result_list = []
-    with open("/usr/local/bin/wal-e", "r") as f:
+    patch_script = patch_script_tmp % patch_script_file
+    with open(patch_file, "r") as f:
         has_patched = False
         for line in f:
-            if not has_patched and line.startswith('import'):
+            if (line.startswith('import') or line.startswith('from')) \
+                    and not has_patched:
                 result_list.append(patch_script)
                 has_patched = True
             result_list.append(line)
-    with open("/usr/local/bin/wal-e", "w") as f:
+        if not has_patched: result_list.append(patch_script)
+    with open(patch_file, "w") as f:
         for line in result_list:
             f.write(line)
 
 if __name__ == '__main__':
-    main()
+    patch_type = sys.argv[1]
+    if patch_type == 'file':
+        patch_file = sys.argv[2]
+    elif patch_type == 'module':
+        module = __import__(sys.argv[2], fromlist=True)
+        patch_file = module.__file__
+    patch_script_file = sys.argv[3]
+    main(patch_file, patch_script_file)
+