feat(backup): honor s3 endpoint config value

teamhephy · Nov 15, 2018 · c614d1f · c614d1f
1 parent 2e39f6e
commit c614d1f
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 25 deletions.
diff --git a/rootfs/bin/create_bucket b/rootfs/bin/create_bucket
@@ -12,6 +12,7 @@ from oauth2client.service_account import ServiceAccountCredentials
 from gcloud.storage.client import Client
 from gcloud import exceptions
 from azure.storage.blob import BlobService
+from urllib.parse import urlparse
 
 def bucket_exists(conn, name):
     bucket = conn.lookup(name)
@@ -23,25 +24,23 @@ bucket_name = os.getenv('BUCKET_NAME')
 region = os.getenv('S3_REGION')
 
 if os.getenv('DATABASE_STORAGE') == "s3":
-    conn = boto.s3.connect_to_region(region)
+    conn = False
+    if os.getenv('S3_ENDPOINT'):
+        endpoint = urlparse(os.getenv('S3_ENDPOINT'))
+        conn = boto.s3.connect_to_region(region,
+            host=endpoint.hostname,
+            port=endpoint.port,
+            path=endpoint.path,
+            calling_format=boto.s3.connection.OrdinaryCallingFormat())
+    else:
+        conn = boto.s3.connect_to_region(region)
+
     if not bucket_exists(conn, bucket_name):
-        try:
-            if region == "us-east-1":
-                # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125
-                conn.create_bucket(bucket_name)
-            else:
-                conn.create_bucket(bucket_name, location=region)
-        # NOTE(bacongobbler): for versions prior to v2.9.0, the bucket is created in the default region.
-        # if we got here, we need to propagate "us-east-1" into WALE_S3_ENDPOINT because the bucket
-        # exists in a different region and we cannot find it.
-        # TODO(bacongobbler): deprecate this once we drop support for v2.8.0 and lower
-        except S3CreateError as err:
-            if region != 'us-east-1':
-                print('Failed to create bucket in {}. We are now assuming that the bucket was created in us-east-1.'.format(region))
-                with open(os.path.join(os.environ['WALE_ENVDIR'], "WALE_S3_ENDPOINT"), "w+") as file:
-                    file.write('https+path://s3.amazonaws.com:443')
-            else:
-                raise
+        if region == "us-east-1":
+            # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125
+            conn.create_bucket(bucket_name)
+        else:
+            conn.create_bucket(bucket_name, location=region)
 
 elif os.getenv('DATABASE_STORAGE') == "gcs":
     scopes = ['https://www.googleapis.com/auth/devstorage.full_control']

diff --git a/rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh b/rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh
@@ -7,15 +7,21 @@ if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
   AWS_SECRET_ACCESS_KEY=$(cat /var/run/secrets/deis/objectstore/creds/secretkey)
   if [[ "$DATABASE_STORAGE" == "s3" ]]; then
     AWS_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
+    S3_ENDPOINT=$(cat /var/run/secrets/deis/objectstore/creds/endpoint)
     BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
-    # Convert $AWS_REGION into $WALE_S3_ENDPOINT to avoid "Connection reset by peer" from
-    # regions other than us-standard.
-    # See https://github.com/wal-e/wal-e/issues/167
-    # See https://github.com/boto/boto/issues/2207
-    if [[ "$AWS_REGION" == "us-east-1" ]]; then
-      echo "https+path://s3.amazonaws.com:443" > WALE_S3_ENDPOINT
+    if [[ "$S3_ENDPOINT" == "" ]]; then
+        # Convert $AWS_REGION into $WALE_S3_ENDPOINT to avoid "Connection reset by peer" from
+        # regions other than us-standard.
+        # See https://github.com/wal-e/wal-e/issues/167
+        # See https://github.com/boto/boto/issues/2207
+        if [[ "$AWS_REGION" == "us-east-1" ]]; then
+          echo "https+path://s3.amazonaws.com:443" > WALE_S3_ENDPOINT
+        else
+          echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
+        fi
     else
-      echo "https+path://s3-${AWS_REGION}.amazonaws.com:443" > WALE_S3_ENDPOINT
+        echo "$S3_ENDPOINT" > S3_ENDPOINT
+        echo "$S3_ENDPOINT" | sed -E -e 's!http(s?)://!http\1+path://!' -e 's!/$!!' > WALE_S3_ENDPOINT
     fi
   else
     AWS_REGION="us-east-1"