bug-1906959: Allow GCS buckets to be behind a CDN.

docker-compose.yml

@@ -168,6 +168,16 @@ services:
       interval: 1s
       timeout: 3s
       retries: 5
+    depends_on:
+      - gcs-cdn
+
+  # nginx as a reverse proxy simulating a CDN in front of the GCS emulator.
+  gcs-cdn:
+    build:
+      context: docker/images/gcs-cdn
+    image: local/tecken_gcs_cdn
+    ports:
+      - "${EXPOSE_CDN_PORT:-8002}:8002"
 
   # https://hub.docker.com/r/localstack/localstack/
   # localstack running a fake AWS S3

docker/config/local_dev.env

@@ -33,6 +33,7 @@ DEBUG=true
 LOCAL_DEV_ENV=true
 CLOUD_SERVICE_PROVIDER=GCS
 UPLOAD_GCS_BUCKET=publicbucket
+UPLOAD_GCS_PUBLIC_URL=http://gcs-cdn:8002/publicbucket
 UPLOAD_S3_BUCKET=publicbucket
 
 # Default to the test oidcprovider container for Open ID Connect

docker/images/gcs-cdn/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx:1.27-alpine
+COPY default.conf /etc/nginx/conf.d/default.conf

docker/images/gcs-cdn/default.conf

@@ -0,0 +1,8 @@
+server {
+    listen 8002;
+    server_name cdn;
+
+    location / {
+        proxy_pass http://gcs-emulator:8001;
+    }
+}

tecken/download/views.py

@@ -156,13 +156,14 @@ def download_symbol(request, debugfilename, debugid, filename, try_symbols=False
     )
     if metadata:
         url = metadata.download_url
-        if "http://localstack:4566" in url and request.get_host() == "localhost:8000":
+        if request.get_host() == "localhost:8000":
             # If doing local development, with Docker, you're most likely running
             # localstack as a fake S3. It runs on its own hostname that is only
             # available from other Docker containers. But to make it really convenient,
             # for testing symbol download we'll rewrite the URL to one that is possible
             # to reach from the host.
-            url = url.replace("localstack:4566", "localhost:4566")
+            url = url.replace("http://gcs-cdn:8002/", "http://localhost:8002/")
+            url = url.replace("http://localstack:4566/", "http://localhost:4566/")
         response = http.HttpResponseRedirect(url)
         if request._request_debug:
             response["Debug-Time"] = elapsed_time

tecken/ext/gcs/storage.py

@@ -27,11 +27,16 @@ def __init__(
         prefix: str,
         try_symbols: bool = False,
         endpoint_url: Optional[str] = None,
+        public_url: Optional[str] = None,
     ):
         self.bucket = bucket
         self.prefix = prefix
         self.try_symbols = try_symbols
         self.endpoint_url = endpoint_url
+        if public_url:
+            self.public_url = public_url.removesuffix("/")
+        else:
+            self.public_url = None
         self.clients = threading.local()
         # The Cloud Storage client doesn't support setting global timeouts for all requests, so we
         # need to pass the timeout for every single request. the default timeout is 60 seconds for
@@ -106,8 +111,12 @@ def get_object_metadata(self, key: str) -> Optional[ObjectMetadata]:
                 original_content_length = int(original_content_length)
             except ValueError:
                 original_content_length = None
+        if self.public_url:
+            download_url = f"{self.public_url}/{quote(gcs_key)}"
+        else:
+            download_url = blob.public_url
         metadata = ObjectMetadata(
-            download_url=blob.public_url,
+            download_url=download_url,
             content_type=blob.content_type,
             content_length=blob.size,
             content_encoding=blob.content_encoding,

tecken/settings.py

@@ -577,6 +577,10 @@ def filter(self, record):
         "UPLOAD_GCS_BUCKET",
         doc="The GCS bucket name for uploads and downloads.",
     )
+    UPLOAD_GCS_PUBLIC_URL = _config(
+        "UPLOAD_GCS_PUBLIC_URL",
+        doc="The base URL for downloading files from the upload bucket.",
+    )
     DOWNLOAD_S3_BUCKET = _config(
         "DOWNLOAD_S3_BUCKET",
         raise_error=False,
@@ -596,6 +600,7 @@ def filter(self, record):
             "bucket": UPLOAD_GCS_BUCKET,
             "prefix": "v1",
             "try_symbols": False,
+            "public_url": UPLOAD_GCS_PUBLIC_URL,
         },
     }
     TRY_UPLOAD_BACKEND = {
@@ -604,6 +609,7 @@ def filter(self, record):
             "bucket": UPLOAD_GCS_BUCKET,
             "prefix": "try/v1",
             "try_symbols": True,
+            "public_url": UPLOAD_GCS_PUBLIC_URL,
         },
     }
     DOWNLOAD_BACKENDS = []

tecken/tests/conftest.py

@@ -151,19 +151,24 @@ def get_storage_backend(bucket_name):
     """Return a function to create a unique storage backend for the current test."""
 
     def _get_storage_backend(
-        kind: Literal["gcs", "s3"], try_symbols: bool = False
+        kind: Literal["gcs", "gcs-cdn", "s3"], try_symbols: bool = False
     ) -> StorageBackend:
         prefix = "try/" * try_symbols + "v1"
         match kind:
             case "gcs":
                 return GCSStorage(bucket_name, prefix, try_symbols)
+            case "gcs-cdn":
+                public_url = f"http://gcs-cdn:8002/{bucket_name}"
+                return GCSStorage(
+                    bucket_name, prefix, try_symbols, public_url=public_url
+                )
             case "s3":
                 return S3Storage(bucket_name, prefix, try_symbols)
 
     return _get_storage_backend
 
 
-@pytest.fixture(params=["gcs", "s3"])
+@pytest.fixture(params=["gcs", "gcs-cdn", "s3"])
 def symbol_storage_no_create(request, get_storage_backend):
     """Replace the global SymbolStorage instance with a new instance.
 

tecken/tests/test_storage_backends.py

@@ -12,10 +12,13 @@
 from tecken.tests.utils import Upload, UPLOADS
 
 
+@pytest.mark.parametrize("try_storage", [False, True])
 @pytest.mark.parametrize("upload", UPLOADS.values(), ids=UPLOADS.keys())
-@pytest.mark.parametrize("storage_kind", ["gcs", "s3"])
-def test_upload_and_download(get_storage_backend, storage_kind: str, upload: Upload):
-    backend = get_storage_backend(storage_kind)
+@pytest.mark.parametrize("storage_kind", ["gcs", "gcs-cdn", "s3"])
+def test_upload_and_download(
+    get_storage_backend, storage_kind: str, upload: Upload, try_storage: bool
+):
+    backend = get_storage_backend(storage_kind, try_storage)
     backend.clear()
     assert backend.exists()
 
@@ -35,20 +38,20 @@ def test_upload_and_download(get_storage_backend, storage_kind: str, upload: Upl
     assert metadata.original_md5_sum == upload.metadata.original_md5_sum
 
 
-@pytest.mark.parametrize("storage_kind", ["gcs", "s3"])
+@pytest.mark.parametrize("storage_kind", ["gcs", "gcs-cdn", "s3"])
 def test_non_exsiting_bucket(get_storage_backend, storage_kind: str):
     backend = get_storage_backend(storage_kind)
     assert not backend.exists()
 
 
-@pytest.mark.parametrize("storage_kind", ["gcs", "s3"])
+@pytest.mark.parametrize("storage_kind", ["gcs", "gcs-cdn", "s3"])
 def test_storageerror_msg(get_storage_backend, storage_kind: str):
     backend = get_storage_backend(storage_kind)
     error = StorageError("storage error message", backend=backend)
     assert repr(backend) in str(error)
 
 
-@pytest.mark.parametrize("storage_kind", ["gcs", "s3"])
+@pytest.mark.parametrize("storage_kind", ["gcs", "gcs-cdn", "s3"])
 def test_s3_download_url(bucket_name: str, get_storage_backend, storage_kind: str):
     backend = get_storage_backend(storage_kind)
     backend.clear()

tecken/tests/utils.py

@@ -35,7 +35,7 @@ class Upload:
     backend: Optional[StorageBackend] = None
 
     @property
-    def key(self):
+    def key(self) -> str:
         return SymbolStorage.make_key(self.debug_file, self.debug_id, self.sym_file)
 
     @classmethod