Fetch upstream changes

.example.env

@@ -19,9 +19,16 @@ VEDA_DOMAIN_ALT_HOSTED_ZONE_ID=[OPTIONAL SECOND HOSTED ZONE]
 VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME=[OPTIONAL SECOND DOMAIN]
 
 VEDA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
-VEDA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API] 
+VEDA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API]
+VEDA_RASTER_EXPORT_ASSUME_ROLE_CREDS_AS_ENVS=False
 
-VEDA_RASTER_PATH_PREFIX=[OPTIONAL PATH PREFIX TO ADD TO TITILER ENDPOINTS]
-VEDA_STAC_PATH_PREFIX=[OPTIONAL PATH PREFIX TO ADD TO TITILER ENDPOINTS]
+VEDA_DB_PUBLICLY_ACCESSIBLE=TRUE
+
+VEDA_RASTER_ROOT_PATH=
+VEDA_STAC_ROOT_PATH=
 
-VEDA_HOST=[OPTIONAL HOST/DOMAIN_NAME TO PROPAGATE TO STAC AND RASTER APIS]
+STAC_BROWSER_BUCKET=
+STAC_URL=
+CERT_ARN=
+VEDA_CLOUDFRONT=
+VEDA_CUSTOM_HOST=

.github/workflows/pr.yml

@@ -61,6 +61,7 @@ jobs:
         with:
           python-version: '3.9'
 
+
       - uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
@@ -69,7 +70,7 @@ jobs:
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install -e .[dev,deploy,test] 
+          python -m pip install -e .[dev,deploy,test]
 
       - name: Launch services
         run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d
@@ -82,6 +83,7 @@ jobs:
         run: sleep 10s
         shell: bash
 
+
       - name: Integrations tests
         run: python -m pytest .github/workflows/tests/ -vv -s
 

.github/workflows/tests/test_raster.py

@@ -33,15 +33,15 @@ def test_mosaic_api():
     assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
     assert resp.json()[0]["id"] == "20200307aC0853900w361030"
 
-    resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/15/8589/12849/assets")
+    resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/tiles/15/8589/12849/assets")
     assert resp.status_code == 200
     assert len(resp.json()) == 1
     assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
     assert resp.json()[0]["id"] == "20200307aC0853900w361030"
 
     z, x, y = 15, 8589, 12849
     resp = httpx.get(
-        f"{raster_endpoint}/mosaic/tiles/{searchid}/{z}/{x}/{y}",
+        f"{raster_endpoint}/mosaic/{searchid}/tiles/{z}/{x}/{y}",
         params={"assets": "cog"},
         headers={"Accept-Encoding": "br, gzip"},
         timeout=10.0,

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
         language_version: python
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 3.8.3
+    rev: 6.1.0
     hooks:
       - id: flake8
         language_version: python

README.md

@@ -43,16 +43,42 @@ To retrieve the variables for a stage that has been previously deployed, the sec
 | `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.7.9 |
 | `VEDA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom veda-backend schema, i.e. 0.1.1 |
 | `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot |
-> **Note** See [Advanced Configuration](docs/advanced_configuration.md) for details about custom configuration options.
+
+### Advanced configuration
+The constructs and applications in this project are configured using pydantic. The settings are defined in config.py files stored alongside the associated construct or application--for example the settings for the RDS PostgreSQL construct are defined in database/infrastructure/config.py. For custom configuration, use environment variables to override the pydantic defaults.
+
+| Construct | Env Prefix | Configuration |
+| --- | --- | --- |
+| Database | `VEDA_DB` | [database/infrastructure/config.py](database/infrastructure/config.py) |
+| Domain | `VEDA_DOMAIN` | [domain/infrastructure/config.py](domain/infrastructure/config.py) |
+| Network | `N/A` | [network/infrastructure/config.py](network/infrastructure/config.py) |
+| Raster API (TiTiler) | `VEDA_RASTER` | [raster_api/infrastructure/config.py](raster_-_api/infrastructure/config.py) |
+| STAC API | `VEDA` | [stac_api/infrastructure/config.py](stac_api/infrastructure/config.py) |
+| Routes | `VEDA` | [routes/infrastructure/config.py](routes/infrastructure/config.py) |
 
 ### Deploying to the cloud
 
-#### Install pre-requisites
+#### Install deployment pre-requisites
+- [Node](https://nodejs.org/)
+- [NVM](https://github.com/nvm-sh/nvm#node-version-manager---)
+- [jq](https://jqlang.github.io/jq/) (used for exporting environment variable secrets to `.env` in [scripts/sync-env-local.sh](/scripts/sync-env-local.sh))
+
+These can be installed with [homebrew](https://brew.sh/) on MacOS
+```
+brew install node
+brew install nvm
+brew install jq
+```
+
+#### Virtual environment example
+```
+python3 -m venv .venv
+source .venv/bin/activate
+```
 
+#### Install requirements
 ```bash
-nvm install 17
-nvm use 17
-node --version
+nvm use --lts
 npm install --location=global aws-cdk
 python3 -m pip install --upgrade pip
 python3 -m pip install -e ".[dev,deploy,test]"
@@ -72,8 +98,9 @@ cdk deploy
 If this is a development stack that is safe to delete, you can delete the stack in CloudFormation console or via `cdk destroy`, however, the additional manual steps were required to completely delete the stack resources:
 
 1. You will need to disable deletion protection of the RDS database and delete the database.
-2. Detach the Internet Gateway (IGW) from the VPC and delete it.
-3. If this stack created a new VPC, delete the VPC (this should delete a subnet and security group too).
+2. Identify and delete the RDS subnet group associated with the RDS database you just deleted (it will not be automatically removed because of the RDS deletion protection in place when the group was created).
+3. If this stack created a new VPC, detach the Internet Gateway (IGW) from the VPC and delete it.
+4. If this stack created a new VPC, delete the VPC (this should delete a subnet and security group too).
 
 ## Custom deployments
 
@@ -98,7 +125,7 @@ docker compose down
 
 > **Warning** PgSTAC records should be loaded in the database using [pypgstac](https://github.com/stac-utils/pgstac#pypgstac) for proper indexing and partitioning.
 
-The VEDA ecosystem includes tools specifially created for loading PgSTAC records and optimizing data assets. The [veda-data-pipelines](https://github.com/NASA-IMPACT/veda-data-pipelines) project provides examples of cloud pipelines that transform data to cloud optimized formats, generate STAC metadata, and submit records for publication to the veda-backend database using the [veda-stac-ingestor](https://github.com/NASA-IMPACT/veda-stac-ingestor).
+The VEDA ecosystem includes tools specifially created for loading PgSTAC records and optimizing data assets. The [veda-data-airflow](https://github.com/NASA-IMPACT/veda-data-airflow) project provides examples of cloud pipelines that transform data to cloud optimized formats, generate STAC metadata, and submit records for publication to the veda-backend database using the [veda-stac-ingestor](https://github.com/NASA-IMPACT/veda-stac-ingestor).
 
 ## Support scripts
 Support scripts are provided for manual system operations.
@@ -118,6 +145,7 @@ Support scripts are provided for manual system operations.
 
 ## VEDA usage examples
 
+### [VEDA documentation](https://nasa-impact.github.io/veda-docs)
 ### [VEDA documentation](https://nasa-impact.github.io/veda-docs)
 
 ### [VEDA dashboard](https://www.earthdata.nasa.gov/dashboard)
@@ -129,3 +157,4 @@ Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is
 
 # License
 This project is licensed under **Apache 2**, see the [LICENSE](LICENSE) file for more details.
+

app.py

@@ -9,6 +9,7 @@
 from domain.infrastructure.construct import DomainConstruct
 from network.infrastructure.construct import VpcConstruct
 from raster_api.infrastructure.construct import RasterApiLambdaConstruct
+from routes.infrastructure.construct import CloudfrontDistributionConstruct
 from stac_api.infrastructure.construct import StacApiLambdaConstruct
 
 app = App()
@@ -24,7 +25,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
         if veda_app_settings.permissions_boundary_policy_name:
             permission_boundary_policy = aws_iam.ManagedPolicy.from_managed_policy_name(
                 self,
-                "permission-boundary",
+                "permissions-boundary",
                 veda_app_settings.permissions_boundary_policy_name,
             )
             aws_iam.PermissionsBoundary.of(self).apply(permission_boundary_policy)
@@ -59,18 +60,29 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
 raster_api = RasterApiLambdaConstruct(
     veda_stack,
     "raster-api",
+    stage=veda_app_settings.stage_name(),
     vpc=vpc.vpc,
     database=database,
-    domain_name=domain.raster_domain_name,
+    domain=domain,
 )
 
 stac_api = StacApiLambdaConstruct(
     veda_stack,
     "stac-api",
+    stage=veda_app_settings.stage_name(),
     vpc=vpc.vpc,
     database=database,
     raster_api=raster_api,
-    domain_name=domain.stac_domain_name,
+    domain=domain,
+)
+
+veda_routes = CloudfrontDistributionConstruct(
+    veda_stack,
+    "routes",
+    stage=veda_app_settings.stage_name(),
+    raster_api_id=raster_api.raster_api.api_id,
+    stac_api_id=stac_api.stac_api.api_id,
+    region=veda_app_settings.cdk_default_region,
 )
 
 # TODO this conditional supports deploying a second set of APIs to a separate custom domain and should be removed if no longer necessary
@@ -85,6 +97,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
     alt_raster_api = RasterApiLambdaConstruct(
         veda_stack,
         "alt-raster-api",
+        stage=veda_app_settings.stage_name(),
         vpc=vpc.vpc,
         database=database,
         domain_name=alt_domain.raster_domain_name,
@@ -93,6 +106,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
     alt_stac_api = StacApiLambdaConstruct(
         veda_stack,
         "alt-stac-api",
+        stage=veda_app_settings.stage_name(),
         vpc=vpc.vpc,
         database=database,
         raster_api=raster_api,

database/infrastructure/config.py

@@ -1,6 +1,7 @@
 """Veda-backend database construct configuration."""
 from typing import Optional
 
+from aws_cdk import aws_ec2, aws_rds
 from pydantic import BaseSettings, Field
 
 
@@ -57,12 +58,37 @@ class vedaDBSettings(BaseSettings):
         False,
         description="Boolean if the RDS should be accessed through a proxy",
     )
-    rds_type: str = Field(
-        "t3.small",
-        description="Postgres database type",
+    rds_instance_class: Optional[str] = Field(
+        aws_ec2.InstanceClass.BURSTABLE3.value,
+        description=(
+            "The instance class of the RDS instance "
+            "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/InstanceClass.html"
+        ),
+    )
+    rds_instance_size: Optional[str] = Field(
+        aws_ec2.InstanceSize.SMALL.value,
+        description=(
+            "The size of the RDS instance "
+            "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_ec2/InstanceSize.html"
+        ),
+    )
+    rds_engine_full_version: Optional[str] = Field(
+        aws_rds.PostgresEngineVersion.VER_14.postgres_full_version,
+        description=(
+            "The version of the RDS Postgres engine "
+            "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_rds/PostgresEngineVersion.html"
+        ),
+    )
+    rds_engine_major_version: Optional[str] = Field(
+        aws_rds.PostgresEngineVersion.VER_14.postgres_major_version,
+        description=(
+            "The version of the RDS Postgres engine "
+            "https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_rds/PostgresEngineVersion.html"
+        ),
     )
-    storage_encrypted: bool = Field(
-        False, description="Boolean if the RDS should be storage encrypted"
+    rds_encryption: Optional[bool] = Field(
+        False,
+        description="Boolean if the RDS should be encrypted",
     )
 
     class Config:

database/infrastructure/construct.py

@@ -139,8 +139,19 @@ def __init__(
 
         # Custom parameter group
         engine = aws_rds.DatabaseInstanceEngine.postgres(
-            version=aws_rds.PostgresEngineVersion.VER_14
+            version=aws_rds.PostgresEngineVersion.of(
+                veda_db_settings.rds_engine_full_version,
+                veda_db_settings.rds_engine_major_version,
+            )
+        )
+
+        # RDS Instance Type
+        rds_instance_type = aws_ec2.InstanceType.of(
+            aws_ec2.InstanceClass[veda_db_settings.rds_instance_class],
+            aws_ec2.InstanceSize[veda_db_settings.rds_instance_size],
         )
+
+        #  version=aws_rds.PostgresEngineVersion.postgres_major_version(veda_db_settings.rds_engine_version)
         parameter_group = aws_rds.ParameterGroup(
             self,
             "parameter-group",
@@ -152,35 +163,33 @@ def __init__(
             },
         )
 
+        # Database Configurations
         database_config = {
             "id": "rds",
             "instance_identifier": f"{stack_name}-postgres",
             "vpc": vpc,
             "engine": engine,
-            "instance_type": aws_ec2.InstanceType(
-                instance_type_identifier=veda_db_settings.rds_type
-            ),
+            "instance_type": rds_instance_type,
             "vpc_subnets": aws_ec2.SubnetSelection(subnet_type=subnet_type),
             "deletion_protection": True,
             "removal_policy": RemovalPolicy.RETAIN,
             "publicly_accessible": veda_db_settings.publicly_accessible,
             "parameter_group": parameter_group,
         }
-
-        if storage_encrypted := veda_db_settings.storage_encrypted:
-            database_config["storage_encrypted"] = storage_encrypted
+        if veda_db_settings.rds_encryption:
+            database_config["storage_encrypted"] = veda_db_settings.rds_encryption
 
         # Create a new database instance from snapshot if provided
         if veda_db_settings.snapshot_id:
             # For the database from snapshot we will need a new master secret
-            credentials = aws_rds.SnapshotCredentials.from_generated_secret(
+            snapshot_credentials = aws_rds.SnapshotCredentials.from_generated_secret(
                 username=veda_db_settings.admin_user
             )
 
             database = aws_rds.DatabaseInstanceFromSnapshot(
                 self,
                 snapshot_identifier=veda_db_settings.snapshot_id,
-                credentials=credentials,
+                credentials=snapshot_credentials,
                 **database_config,
             )
         # Or create/update RDS Resource

database/runtime/Dockerfile

@@ -5,12 +5,7 @@ RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"
 
 WORKDIR /tmp
 
-COPY database/runtime/requirements.txt /tmp/requirements.txt
-
-RUN pip install --upgrade pip
-RUN pip install -r /tmp/requirements.txt -t /asset
-
-RUN pip install pypgstac==${PGSTAC_VERSION} -t /asset
+RUN pip install requests "urllib3<2" psycopg[binary,pool] pypgstac==${PGSTAC_VERSION} -t /asset
 
 COPY database/runtime/handler.py /asset/handler.py
 

docker-compose.yml

@@ -84,6 +84,8 @@ services:
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
       # API Config
       - VEDA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
+      - VEDA_RASTER_EXPORT_ASSUME_ROLE_CREDS_AS_ENVS=TRUE
+
 
     depends_on:
       - database
@@ -95,7 +97,7 @@ services:
   database:
     container_name: veda.db
     platform: linux/amd64
-    image: ghcr.io/stac-utils/pgstac:v0.7.9
+    image: ghcr.io/stac-utils/pgstac:v0.7.10
     environment:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password