Support optional AWS requester-pays S3 transfer costs configuration

.github/workflows/pr.yml

@@ -11,7 +11,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: '3.8'
-    
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
@@ -24,17 +24,24 @@ jobs:
 
       - name: Run pre-commit
         run: pre-commit run --all-files
-    
+
   test:
     needs: [lint]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
+      - name: Launch services
+        run: |
+          AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} \
+          docker-compose up --build -d stac raster
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.8'
-    
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
@@ -43,9 +50,7 @@ jobs:
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .[dev,deploy,test] 
-      - name: Launch services
-        run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker-compose up --build -d stac raster 
+          python -m pip install -e .[dev,deploy,test]
 
       - name: Ingest Stac Items/Collection
         run: |
@@ -62,23 +67,23 @@ jobs:
 
       - name: Stop services
         run: docker-compose stop
-  predeploy: 
+  predeploy:
     needs: [test]
 
     runs-on: ubuntu-latest
     steps:
-    
+
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.8'
-      
+
       - name: Setup Node
         uses: actions/setup-node@v1
-        with: 
+        with:
           node-version: 17
-      
+
       - name: Configure awscli
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -90,24 +95,24 @@ jobs:
         with:
           path: ~/.npm
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-      
+
       - name: Install CDK
         run: npm install -g aws-cdk@2
-      
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
           key:  ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
-      
+
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .[dev,deploy,test] 
+          python -m pip install -e .[dev,deploy,test]
 
       - name: Get dev environment configuration for develop branch
         run: ./scripts/get-env.sh "veda-backend-uah-dev-env"
 
-      - name: Pre deployment CDK diff 
+      - name: Pre deployment CDK diff
         run: |
           echo $STAGE
-          cdk diff veda-backend-uah-dev
+          cdk diff veda-backend-uah-dev

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
         language_version: python
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.4.2
+    rev: 5.12.0
     hooks:
       - id: isort
         language_version: python

docker-compose.yml

@@ -35,7 +35,7 @@ services:
     depends_on:
       - database
       - raster
-    command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 45 -h database -p 5432 && /start.sh"
+    command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && /start.sh"
     volumes:
       - ./scripts:/tmp/scripts
 
@@ -87,22 +87,21 @@ services:
 
     depends_on:
       - database
-    command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 25 -h database -p 5432 && /start.sh"
+    command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && /start.sh"
     volumes:
       - ./scripts:/tmp/scripts
       - $HOME/.aws/credentials:/root/.aws/credentials
 
   database:
     container_name: veda.db
     platform: linux/amd64
-    image: ghcr.io/stac-utils/pgstac:v0.6.6
+    image: ghcr.io/stac-utils/pgstac:v0.7.1
     environment:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password
       - POSTGRES_DB=postgis
       - PGUSER=username
       - PGPASSWORD=password
-      - PGHOST=localhost
       - PGDATABASE=postgis
     ports:
       - "5432:5432"

docs/advanced_configuration.md

@@ -21,4 +21,5 @@ Environment variables for specific VEDA backend components are prefixed, for exa
 | `VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME` | Optional second custom domain name, i.e. alt-veda-backend.xyz |
 | `VEDA_DOMAIN_API_PREFIX` | Optional domain prefix override supports using a custom prefix instead of the STAGE variabe (an alternate version of the stack can be deployed with a unique STAGE=altprod and after testing prod API traffic can be cut over to the alternate version of the stack by setting the prefix to prod) |
 | `VEDA_RASTER_ENABLE_MOSAIC_SEARCH` | Optional deploy the raster API with the mosaic/list endpoint TRUE/FALSE |
-| `VEDA_RASTER_DATA_ACCESS_ROLE_ARN` | Optional arn of IAM Role to be assumed by raster-api for S3 bucket data access, if not provided default role for the lambda construct is used |
+| `VEDA_RASTER_DATA_ACCESS_ROLE_ARN` | Optional arn of IAM Role to be assumed by raster-api for S3 bucket data access, if not provided default role for the lambda construct is used |
+| `VEDA_RASTER_AWS_REQUEST_PAYER` | Set this optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs |

raster_api/infrastructure/config.py

@@ -59,6 +59,11 @@ class vedaRasterSettings(BaseSettings):
         description="Resource name of role permitting access to specified external S3 buckets",
     )
 
+    aws_request_payer: Optional[str] = Field(
+        None,
+        description="Set optional global parameter to 'requester' if the requester agrees to pay S3 transfer costs",
+    )
+
     class Config:
         """model config"""
 

raster_api/infrastructure/construct.py

@@ -69,6 +69,12 @@ def __init__(
             "VEDA_RASTER_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn
         )
 
+        # Optional AWS S3 requester pays global setting
+        if veda_raster_settings.aws_request_payer:
+            veda_raster_function.add_environment(
+                "AWS_REQUEST_PAYER", veda_raster_settings.aws_request_payer
+            )
+
         raster_api_integration = (
             aws_apigatewayv2_integrations_alpha.HttpLambdaIntegration(
                 construct_id, veda_raster_function

raster_api/runtime/setup.py

@@ -9,6 +9,7 @@
     "titiler.pgstac==0.1.0.a9",
     "titiler.application>=0.5,<0.6",
     "starlette-cramjam>=0.1.0,<0.2",
+    "fastapi==0.93.0",
     "importlib_resources>=1.1.0;python_version<'3.9'",
     "aws_xray_sdk>=2.6.0,<3",
     "aws-lambda-powertools>=1.18.0",

setup.py

@@ -19,7 +19,7 @@
         "pytest",
         "pytest-asyncio",
         "httpx",
-        "pypgstac==0.6.6",
+        "pypgstac==0.7.1",
         "psycopg[binary, pool]",
     ],
 }

stac_api/runtime/Dockerfile

@@ -3,7 +3,7 @@ FROM lambci/lambda:build-python3.8
 WORKDIR /tmp
 
 COPY stac_api/runtime /tmp/stac
-RUN pip install "mangum>=0.14,<0.15" /tmp/stac -t /asset --no-binary pydantic
+RUN pip install "mangum>=0.14,<0.15" "plpygis>=0.2.1" /tmp/stac -t /asset --no-binary pydantic
 RUN rm -rf /tmp/stac
 
 # Reduce package size and remove useless files