From b95661eac64fec82dab94df74f136750ce23a850 Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Mon, 31 Oct 2022 11:23:37 -0500 Subject: [PATCH 01/23] Enable context for stac searches in pgstac --- database/runtime/handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/database/runtime/handler.py b/database/runtime/handler.py index 8ef00b5f..c285c733 100644 --- a/database/runtime/handler.py +++ b/database/runtime/handler.py @@ -109,7 +109,7 @@ def create_user(cursor, username: str, password: str) -> None: def create_permissions(cursor, db_name: str, username: str) -> None: - """Add permissions.""" + """Add permissions and user-specific pgstac configuration.""" cursor.execute( sql.SQL( "GRANT CONNECT ON DATABASE {db_name} TO {username};" @@ -122,6 +122,11 @@ def create_permissions(cursor, db_name: str, username: str) -> None: "GRANT pgstac_read TO {username};" "GRANT pgstac_ingest TO {username};" "GRANT pgstac_admin TO {username};" + "ALTER ROLE {username} SET SEARCH_PATH to pgstac, public;" + "ALTER ROLE {username} SET pgstac.context TO 'auto';" + "ALTER ROLE {username} SET pgstac.context_estimated_count TO '100000';" + "ALTER ROLE {username} SET pgstac.context_estimated_cost TO '100000';" + "ALTER ROLE {username} SET pgstac.context_stats_ttl TO '1 day';" ).format( db_name=sql.Identifier(db_name), username=sql.Identifier(username), From 9564631c70c4f2e130e4d1b154e1431d3707c52d Mon Sep 17 00:00:00 2001 From: Prakash Chaudhary Date: Wed, 16 Nov 2022 15:30:23 -0600 Subject: [PATCH 02/23] [Chore]: Add Apache 2.0 License --- LICENSE | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/LICENSE b/LICENSE index ec8b8050..224519be 100644 --- a/LICENSE +++ b/LICENSE @@ -1,29 +1,13 @@ -BSD 3-Clause License +Copyright 2022 NASA -Copyright (c) 2021, Inter Agency Implementation and Advanced Concepts -All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: + http://www.apache.org/licenses/LICENSE-2.0 -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file From aad06732ed3db567f3beb658c66f6c6626bb3a04 Mon Sep 17 00:00:00 2001 From: Prakash Chaudhary Date: Wed, 16 Nov 2022 15:30:23 -0600 Subject: [PATCH 03/23] [Chore]: Add Apache 2.0 License info in README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d02072b6..3120033c 100644 --- a/README.md +++ b/README.md @@ -186,4 +186,7 @@ https://github.com/NASA-IMPACT/veda-documentation # STAC community resources ## STAC browser -Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is a browser for STAC catalogs. The demo version of this browser [radiantearth.github.io/stac-browser](https://radiantearth.github.io/stac-browser/#/) can be used to browse the contents of the delta-backend STAC catalog, paste the delta-backend stac-api URL deployed by this project in the demo and click load. Read more about the recent developments and usage of stac-browser [here](https://medium.com/radiant-earth-insights/the-exciting-future-of-the-stac-browser-2351143aa24b). \ No newline at end of file +Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is a browser for STAC catalogs. The demo version of this browser [radiantearth.github.io/stac-browser](https://radiantearth.github.io/stac-browser/#/) can be used to browse the contents of the delta-backend STAC catalog, paste the delta-backend stac-api URL deployed by this project in the demo and click load. Read more about the recent developments and usage of stac-browser [here](https://medium.com/radiant-earth-insights/the-exciting-future-of-the-stac-browser-2351143aa24b). + +# License +This project is licensed under **Apache 2**, see the [LICENSE](LICENSE) file for more details. \ No newline at end of file From 112e70c008c61a717be9bd5dad7b2f28f5b59ee7 Mon Sep 17 00:00:00 2001 From: ividito Date: Tue, 1 Nov 2022 14:41:02 -0300 Subject: [PATCH 04/23] Adds monitoring, metrics collection, and enhanced logging --- .gitignore | 2 + raster_api/runtime/handler.py | 12 +++++ raster_api/runtime/setup.py | 2 + raster_api/runtime/src/app.py | 44 ++++++++++++++++-- raster_api/runtime/src/factory.py | 4 +- raster_api/runtime/src/monitoring.py | 29 ++++++++++++ raster_api/runtime/src/tracing.py | 69 ++++++++++++++++++++++++++++ stac_api/runtime/handler.py | 10 ++++ stac_api/runtime/setup.py | 3 ++ stac_api/runtime/src/app.py | 33 ++++++++++++- stac_api/runtime/src/extension.py | 8 +++- stac_api/runtime/src/monitoring.py | 29 ++++++++++++ 12 files changed, 236 insertions(+), 9 deletions(-) create mode 100644 raster_api/runtime/src/monitoring.py create mode 100644 raster_api/runtime/src/tracing.py create mode 100644 stac_api/runtime/src/monitoring.py diff --git a/.gitignore b/.gitignore index 14199335..348981ef 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,5 @@ Pipfile.lock cdk.context.json node_modules/ package-lock.json +cdk-dev.out +**/--no-binary/ diff --git a/raster_api/runtime/handler.py b/raster_api/runtime/handler.py index b7c8bb83..3d98328f 100644 --- a/raster_api/runtime/handler.py +++ b/raster_api/runtime/handler.py @@ -4,8 +4,20 @@ from mangum import Mangum from src.app import app +from src.monitoring import logger, metrics, tracer logging.getLogger("mangum.lifespan").setLevel(logging.ERROR) logging.getLogger("mangum.http").setLevel(logging.ERROR) + + + handler = Mangum(app, lifespan="auto") + +# Add tracing +handler.__name__ = "handler" # tracer requires __name__ to be set +handler = tracer.capture_lambda_handler(handler) +# Add logging +handler = logger.inject_lambda_context(handler, clear_state=True) +# Add metrics last to properly flush metrics. +handler = metrics.log_metrics(handler, capture_cold_start_metric=True) \ No newline at end of file diff --git a/raster_api/runtime/setup.py b/raster_api/runtime/setup.py index acaad400..034b6ec9 100644 --- a/raster_api/runtime/setup.py +++ b/raster_api/runtime/setup.py @@ -10,6 +10,8 @@ "titiler.application>=0.5,<0.6", "starlette-cramjam>=0.1.0,<0.2", "importlib_resources>=1.1.0;python_version<'3.9'", + "aws_xray_sdk>=2.6.0,<3", + "aws-lambda-powertools>=1.18.0", ] extra_reqs = { diff --git a/raster_api/runtime/src/app.py b/raster_api/runtime/src/app.py index 5c0056b9..79747649 100644 --- a/raster_api/runtime/src/app.py +++ b/raster_api/runtime/src/app.py @@ -8,10 +8,11 @@ from src.factory import MosaicTilerFactory, MultiBaseTilerFactory from src.version import __version__ as delta_raster_version -from fastapi import Depends, FastAPI, Query +from fastapi import Depends, FastAPI, Query, APIRouter +from starlette.middleware.base import BaseHTTPMiddleware from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request -from starlette.responses import HTMLResponse +from starlette.responses import HTMLResponse, JSONResponse from starlette_cramjam.middleware import CompressionMiddleware from titiler.application.custom import templates from titiler.core.dependencies import DatasetPathParams @@ -23,6 +24,9 @@ from titiler.pgstac.db import close_db_connection, connect_to_db from titiler.pgstac.dependencies import ItemPathParams from titiler.pgstac.reader import PgSTACReader +from aws_lambda_powertools.metrics import MetricUnit + +from .monitoring import logger, metrics, tracer, LoggerRouteHandler logging.getLogger("botocore.credentials").disabled = True logging.getLogger("botocore.utils").disabled = True @@ -35,12 +39,12 @@ else: optional_headers = [] - app = FastAPI(title=settings.name, version=delta_raster_version) +# router to be applied to all titiler route factories (improves logs with FastAPI context) +router = APIRouter(route_class=LoggerRouteHandler) add_exception_handlers(app, DEFAULT_STATUS_CODES) add_exception_handlers(app, MOSAIC_STATUS_CODES) - # Custom PgSTAC mosaic tiler mosaic = MosaicTilerFactory( router_prefix="/mosaic", @@ -48,6 +52,7 @@ optional_headers=optional_headers, gdal_config=settings.get_gdal_config(), dataset_dependency=DatasetParams, + router=router ) app.include_router(mosaic.router, prefix="/mosaic", tags=["Mosaic"]) @@ -58,6 +63,7 @@ optional_headers=optional_headers, router_prefix="/stac", gdal_config=settings.get_gdal_config(), + router=router ) app.include_router(stac.router, tags=["Items"], prefix="/stac") @@ -65,9 +71,9 @@ router_prefix="/cog", optional_headers=optional_headers, gdal_config=settings.get_gdal_config(), + router=router ) - @cog.router.get("/validate", response_model=Info) def cog_validate( src_path: str = Depends(DatasetPathParams), @@ -127,6 +133,34 @@ def ping(): }, ) +# If the correlation header is used in the UI, we can analyze traces that originate from a given user or client +@app.middleware("http") +async def add_correlation_id(request: Request, call_next): + # Get correlation id from X-Correlation-Id header + corr_id = request.headers.get("x-correlation-id") + if not corr_id: + try: + # If empty, use request id from aws context + corr_id = request.scope["aws.context"].aws_request_id + except KeyError: + # If empty, use uuid + corr_id = 'local' + # Add correlation id to logs + logger.set_correlation_id(corr_id) + # Add correlation id to traces + tracer.put_annotation(key="correlation_id", value=corr_id) + + response = await tracer.capture_method(call_next)(request) + # Return correlation header in response + response.headers["X-Correlation-Id"] = corr_id + logger.info("Request completed") + return response + +@app.exception_handler(Exception) +async def validation_exception_handler(request, err): + metrics.add_metric(name="UnhandledExceptions", unit=MetricUnit.Count, value=1) + logger.exception("Unhandled exception") + return JSONResponse(status_code=500, content={"detail": "Internal Server Error"}) @app.on_event("startup") async def startup_event() -> None: diff --git a/raster_api/runtime/src/factory.py b/raster_api/runtime/src/factory.py index 34d13f92..d39b1e2d 100644 --- a/raster_api/runtime/src/factory.py +++ b/raster_api/runtime/src/factory.py @@ -18,6 +18,8 @@ from titiler.pgstac import factory as TitilerPgSTACFactory from titiler.pgstac import model +from .monitoring import tracer + try: from importlib.resources import files as resources_files # type: ignore except ImportError: @@ -188,7 +190,7 @@ def parse_sort_by(sortby: str) -> Generator[sql.Composable, None, None]: *order_by, sql.SQL("LIMIT %(limit)s OFFSET %(offset)s"), ] - cursor.execute( + tracer.capture_method(cursor.execute)( sql.SQL(" ").join(query), {"limit": limit, "offset": offset} ) diff --git a/raster_api/runtime/src/monitoring.py b/raster_api/runtime/src/monitoring.py new file mode 100644 index 00000000..ebb49b75 --- /dev/null +++ b/raster_api/runtime/src/monitoring.py @@ -0,0 +1,29 @@ +from aws_lambda_powertools import Logger, Metrics, Tracer +from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 +from fastapi import Request, Response +from fastapi.routing import APIRoute +from typing import Callable +from functools import wraps + +logger: Logger = Logger(service='raster-api', namespace='veda-backend') +metrics: Metrics = Metrics(service='raster-api', namespace='veda-backend') +tracer: Tracer = Tracer() + +class LoggerRouteHandler(APIRoute): + def get_route_handler(self) -> Callable: + original_route_handler = super().get_route_handler() + async def route_handler(request: Request) -> Response: + # Add fastapi context to logs + ctx = { + "path": request.url.path, + "route": self.path, + "method": request.method, + } + logger.append_keys(fastapi=ctx) + logger.info("Received request") + metrics.add_metric(name='/'.join(str(request.url).split('/')[:2]), unit=MetricUnit.Count, value=1) + tracer.put_annotation(key="path", value=request.url.path) + tracer.capture_method(original_route_handler)(request) + return await original_route_handler(request) + + return route_handler diff --git a/raster_api/runtime/src/tracing.py b/raster_api/runtime/src/tracing.py new file mode 100644 index 00000000..a0ccc589 --- /dev/null +++ b/raster_api/runtime/src/tracing.py @@ -0,0 +1,69 @@ +import copy +import re +import wrapt +from operator import methodcaller + +from aws_xray_sdk.ext.dbapi2 import XRayTracedConn, XRayTracedCursor + + +def patch_psycopg_3(): + wrapt.wrap_function_wrapper( + 'psycopg', + 'connect', + _xray_traced_connect + ) + wrapt.wrap_function_wrapper( + 'psycopg.extensions', + 'register_type', + _xray_register_type_fix + ) + wrapt.wrap_function_wrapper( + 'psycopg.extensions', + 'quote_ident', + _xray_register_type_fix + ) + + wrapt.wrap_function_wrapper( + 'psycopg.extras', + 'register_default_jsonb', + _xray_register_default_jsonb_fix + ) + + +def _xray_traced_connect(wrapped, instance, args, kwargs): + conn = wrapped(*args, **kwargs) + parameterized_dsn = {c[0]: c[-1] for c in map(methodcaller('split', '='), conn.dsn.split(' '))} + meta = { + 'database_type': 'PostgreSQL', + 'url': 'postgresql://{}@{}:{}/{}'.format( + parameterized_dsn.get('user', 'unknown'), + parameterized_dsn.get('host', 'unknown'), + parameterized_dsn.get('port', 'unknown'), + parameterized_dsn.get('dbname', 'unknown'), + ), + 'user': parameterized_dsn.get('user', 'unknown'), + 'database_version': str(conn.server_version), + 'driver_version': 'Psycopg' + } + + return XRayTracedConn(conn, meta) + + +def _xray_register_type_fix(wrapped, instance, args, kwargs): + """Send the actual connection or curser to register type.""" + our_args = list(copy.copy(args)) + if len(our_args) == 2 and isinstance(our_args[1], (XRayTracedConn, XRayTracedCursor)): + our_args[1] = our_args[1].__wrapped__ + + return wrapped(*our_args, **kwargs) + + +def _xray_register_default_jsonb_fix(wrapped, instance, args, kwargs): + our_kwargs = dict() + for key, value in kwargs.items(): + if key == "conn_or_curs" and isinstance(value, (XRayTracedConn, XRayTracedCursor)): + # unwrap the connection or cursor to be sent to register_default_jsonb + value = value.__wrapped__ + our_kwargs[key] = value + + return wrapped(*args, **our_kwargs) \ No newline at end of file diff --git a/stac_api/runtime/handler.py b/stac_api/runtime/handler.py index b7c8bb83..a46554e2 100644 --- a/stac_api/runtime/handler.py +++ b/stac_api/runtime/handler.py @@ -4,8 +4,18 @@ from mangum import Mangum from src.app import app +from src.monitoring import logger, metrics, tracer + logging.getLogger("mangum.lifespan").setLevel(logging.ERROR) logging.getLogger("mangum.http").setLevel(logging.ERROR) handler = Mangum(app, lifespan="auto") + +# Add tracing +handler.__name__ = "handler" # tracer requires __name__ to be set +handler = tracer.capture_lambda_handler(handler) +# Add logging +handler = logger.inject_lambda_context(handler, clear_state=True) +# Add metrics last to properly flush metrics. +handler = metrics.log_metrics(handler, capture_cold_start_metric=True) \ No newline at end of file diff --git a/stac_api/runtime/setup.py b/stac_api/runtime/setup.py index f761f9f5..13b81c91 100644 --- a/stac_api/runtime/setup.py +++ b/stac_api/runtime/setup.py @@ -15,6 +15,9 @@ "jinja2>=2.11.2,<4.0.0", "starlette-cramjam>=0.1.0.a0,<0.2", "importlib_resources>=1.1.0;python_version<'3.9'", + "pygeoif<=0.8", # newest release (1.0+ / 09-22-2022) breaks a number of other geo libs + "aws-lambda-powertools>=1.18.0", + "aws_xray_sdk>=2.6.0,<3", ] extra_reqs = { diff --git a/stac_api/runtime/src/app.py b/stac_api/runtime/src/app.py index 4bd873bb..c339e9be 100644 --- a/stac_api/runtime/src/app.py +++ b/stac_api/runtime/src/app.py @@ -14,9 +14,12 @@ from stac_fastapi.pgstac.db import close_db_connection, connect_to_db from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request -from starlette.responses import HTMLResponse +from starlette.responses import HTMLResponse, JSONResponse from starlette.templating import Jinja2Templates from starlette_cramjam.middleware import CompressionMiddleware +from aws_lambda_powertools.metrics import MetricUnit + +from .monitoring import logger, metrics, tracer, LoggerRouteHandler try: from importlib.resources import files as resources_files # type: ignore @@ -72,6 +75,34 @@ async def viewer_page(request: Request): media_type="text/html", ) +# If the correlation header is used in the UI, we can analyze traces that originate from a given user or client +@app.middleware("http") +async def add_correlation_id(request: Request, call_next): + # Get correlation id from X-Correlation-Id header + corr_id = request.headers.get("x-correlation-id") + if not corr_id: + try: + # If empty, use request id from aws context + corr_id = request.scope["aws.context"].aws_request_id + except KeyError: + # If empty, use uuid + corr_id = 'local' + # Add correlation id to logs + logger.set_correlation_id(corr_id) + # Add correlation id to traces + tracer.put_annotation(key="correlation_id", value=corr_id) + + response = await tracer.capture_method(call_next)(request) + # Return correlation header in response + response.headers["X-Correlation-Id"] = corr_id + logger.info("Request completed") + return response + +@app.exception_handler(Exception) +async def validation_exception_handler(request, err): + metrics.add_metric(name="UnhandledExceptions", unit=MetricUnit.Count, value=1) + logger.error("Unhandled exception") + return JSONResponse(status_code=500, content={"detail": "Internal Server Error"}) @app.on_event("startup") async def startup_event(): diff --git a/stac_api/runtime/src/extension.py b/stac_api/runtime/src/extension.py index a5185657..a5ac7a3c 100644 --- a/stac_api/runtime/src/extension.py +++ b/stac_api/runtime/src/extension.py @@ -10,7 +10,9 @@ from stac_fastapi.types.extension import ApiExtension from starlette.requests import Request -router = APIRouter() +from .monitoring import LoggerRouteHandler, logger, metrics, tracer + +router = APIRouter(route_class=LoggerRouteHandler) MAX_B64_ITEM_SIZE = 2000 @@ -27,8 +29,9 @@ def register(self, app: FastAPI, titiler_endpoint: str) -> None: None """ - router = APIRouter() + router = APIRouter(route_class=LoggerRouteHandler) + @tracer.capture_method @router.get( "/collections/{collectionId}/items/{itemId}/tilejson.json", ) @@ -90,6 +93,7 @@ async def tilejson( f"{titiler_endpoint}/stac/tilejson.json?{urlencode(qs)}" ) + @tracer.capture_method @router.get( "/collections/{collectionId}/items/{itemId}/viewer", responses={ diff --git a/stac_api/runtime/src/monitoring.py b/stac_api/runtime/src/monitoring.py new file mode 100644 index 00000000..2eab9f30 --- /dev/null +++ b/stac_api/runtime/src/monitoring.py @@ -0,0 +1,29 @@ +from aws_lambda_powertools import Logger, Metrics, Tracer +from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 +from fastapi import Request, Response +from fastapi.routing import APIRoute +from typing import Callable +from functools import wraps + +logger: Logger = Logger(service='stac-api', namespace='veda-backend') +metrics: Metrics = Metrics(service='stac-api', namespace='veda-backend') +tracer: Tracer = Tracer() + +class LoggerRouteHandler(APIRoute): + def get_route_handler(self) -> Callable: + original_route_handler = super().get_route_handler() + async def route_handler(request: Request) -> Response: + # Add fastapi context to logs + ctx = { + "path": request.url.path, + "route": self.path, + "method": request.method, + } + logger.append_keys(fastapi=ctx) + logger.info("Received request") + metrics.add_metric(name='/'.join(str(request.url).split('/')[:2]), unit=MetricUnit.Count, value=1) + tracer.put_annotation(key="path", value=request.url.path) + tracer.capture_method(original_route_handler)(request) + return await original_route_handler(request) + + return route_handler From 60c7bc99b71465ac4f9792e23e5b32c809927342 Mon Sep 17 00:00:00 2001 From: ividito Date: Tue, 1 Nov 2022 14:44:35 -0300 Subject: [PATCH 05/23] Enable tracing in lambda construct --- raster_api/infrastructure/construct.py | 1 + stac_api/infrastructure/construct.py | 1 + 2 files changed, 2 insertions(+) diff --git a/raster_api/infrastructure/construct.py b/raster_api/infrastructure/construct.py index 5970c465..177c0536 100644 --- a/raster_api/infrastructure/construct.py +++ b/raster_api/infrastructure/construct.py @@ -52,6 +52,7 @@ def __init__( timeout=Duration.seconds(delta_raster_settings.timeout), log_retention=aws_logs.RetentionDays.ONE_WEEK, environment=delta_raster_settings.env or {}, + tracing=aws_lambda.Tracing.ACTIVE ) database.pgstac.secret.grant_read(delta_raster_function) diff --git a/stac_api/infrastructure/construct.py b/stac_api/infrastructure/construct.py index e7fbc2bd..dc6537c1 100644 --- a/stac_api/infrastructure/construct.py +++ b/stac_api/infrastructure/construct.py @@ -55,6 +55,7 @@ def __init__( **{k.upper(): v for k, v in delta_stac_settings.env.items()}, }, log_retention=aws_logs.RetentionDays.ONE_WEEK, + tracing=aws_lambda.Tracing.ACTIVE ) # # lambda_function.add_environment(key="TITILER_ENDPOINT", value=raster_api.url) From f2a90320f2e88f58c10ec2090678c5f3f5205950 Mon Sep 17 00:00:00 2001 From: Isayah Vidito Date: Wed, 2 Nov 2022 10:23:00 -0300 Subject: [PATCH 06/23] lint+cleanup --- raster_api/infrastructure/construct.py | 2 +- raster_api/runtime/handler.py | 4 +- raster_api/runtime/src/app.py | 23 +++++---- raster_api/runtime/src/monitoring.py | 27 +++++++--- raster_api/runtime/src/tracing.py | 69 -------------------------- stac_api/infrastructure/construct.py | 2 +- stac_api/runtime/handler.py | 3 +- stac_api/runtime/src/app.py | 11 ++-- stac_api/runtime/src/extension.py | 2 +- stac_api/runtime/src/monitoring.py | 27 +++++++--- 10 files changed, 67 insertions(+), 103 deletions(-) delete mode 100644 raster_api/runtime/src/tracing.py diff --git a/raster_api/infrastructure/construct.py b/raster_api/infrastructure/construct.py index 177c0536..4dc6d78b 100644 --- a/raster_api/infrastructure/construct.py +++ b/raster_api/infrastructure/construct.py @@ -52,7 +52,7 @@ def __init__( timeout=Duration.seconds(delta_raster_settings.timeout), log_retention=aws_logs.RetentionDays.ONE_WEEK, environment=delta_raster_settings.env or {}, - tracing=aws_lambda.Tracing.ACTIVE + tracing=aws_lambda.Tracing.ACTIVE, ) database.pgstac.secret.grant_read(delta_raster_function) diff --git a/raster_api/runtime/handler.py b/raster_api/runtime/handler.py index 3d98328f..5de94a8f 100644 --- a/raster_api/runtime/handler.py +++ b/raster_api/runtime/handler.py @@ -10,8 +10,6 @@ logging.getLogger("mangum.http").setLevel(logging.ERROR) - - handler = Mangum(app, lifespan="auto") # Add tracing @@ -20,4 +18,4 @@ # Add logging handler = logger.inject_lambda_context(handler, clear_state=True) # Add metrics last to properly flush metrics. -handler = metrics.log_metrics(handler, capture_cold_start_metric=True) \ No newline at end of file +handler = metrics.log_metrics(handler, capture_cold_start_metric=True) diff --git a/raster_api/runtime/src/app.py b/raster_api/runtime/src/app.py index 79747649..1b4ee895 100644 --- a/raster_api/runtime/src/app.py +++ b/raster_api/runtime/src/app.py @@ -1,6 +1,7 @@ """TiTiler+PgSTAC FastAPI application.""" import logging +from aws_lambda_powertools.metrics import MetricUnit from rio_cogeo.cogeo import cog_info as rio_cogeo_info from rio_cogeo.models import Info from src.config import ApiSettings @@ -8,8 +9,7 @@ from src.factory import MosaicTilerFactory, MultiBaseTilerFactory from src.version import __version__ as delta_raster_version -from fastapi import Depends, FastAPI, Query, APIRouter -from starlette.middleware.base import BaseHTTPMiddleware +from fastapi import APIRouter, Depends, FastAPI, Query from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request from starlette.responses import HTMLResponse, JSONResponse @@ -24,9 +24,8 @@ from titiler.pgstac.db import close_db_connection, connect_to_db from titiler.pgstac.dependencies import ItemPathParams from titiler.pgstac.reader import PgSTACReader -from aws_lambda_powertools.metrics import MetricUnit -from .monitoring import logger, metrics, tracer, LoggerRouteHandler +from .monitoring import LoggerRouteHandler, logger, metrics, tracer logging.getLogger("botocore.credentials").disabled = True logging.getLogger("botocore.utils").disabled = True @@ -52,7 +51,7 @@ optional_headers=optional_headers, gdal_config=settings.get_gdal_config(), dataset_dependency=DatasetParams, - router=router + router=router, ) app.include_router(mosaic.router, prefix="/mosaic", tags=["Mosaic"]) @@ -63,7 +62,7 @@ optional_headers=optional_headers, router_prefix="/stac", gdal_config=settings.get_gdal_config(), - router=router + router=router, ) app.include_router(stac.router, tags=["Items"], prefix="/stac") @@ -71,9 +70,10 @@ router_prefix="/cog", optional_headers=optional_headers, gdal_config=settings.get_gdal_config(), - router=router + router=router, ) + @cog.router.get("/validate", response_model=Info) def cog_validate( src_path: str = Depends(DatasetPathParams), @@ -133,10 +133,12 @@ def ping(): }, ) + # If the correlation header is used in the UI, we can analyze traces that originate from a given user or client @app.middleware("http") async def add_correlation_id(request: Request, call_next): - # Get correlation id from X-Correlation-Id header + """Add correlation ids to all requests and subsequent logs/traces""" + # Get correlation id from X-Correlation-Id header if provided corr_id = request.headers.get("x-correlation-id") if not corr_id: try: @@ -144,7 +146,7 @@ async def add_correlation_id(request: Request, call_next): corr_id = request.scope["aws.context"].aws_request_id except KeyError: # If empty, use uuid - corr_id = 'local' + corr_id = "local" # Add correlation id to logs logger.set_correlation_id(corr_id) # Add correlation id to traces @@ -156,12 +158,15 @@ async def add_correlation_id(request: Request, call_next): logger.info("Request completed") return response + @app.exception_handler(Exception) async def validation_exception_handler(request, err): + """Handle exceptions that aren't caught elsewhere""" metrics.add_metric(name="UnhandledExceptions", unit=MetricUnit.Count, value=1) logger.exception("Unhandled exception") return JSONResponse(status_code=500, content={"detail": "Internal Server Error"}) + @app.on_event("startup") async def startup_event() -> None: """Connect to database on startup.""" diff --git a/raster_api/runtime/src/monitoring.py b/raster_api/runtime/src/monitoring.py index ebb49b75..40b24794 100644 --- a/raster_api/runtime/src/monitoring.py +++ b/raster_api/runtime/src/monitoring.py @@ -1,17 +1,24 @@ +"""Observability utils""" +from typing import Callable + from aws_lambda_powertools import Logger, Metrics, Tracer from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 + from fastapi import Request, Response from fastapi.routing import APIRoute -from typing import Callable -from functools import wraps - -logger: Logger = Logger(service='raster-api', namespace='veda-backend') -metrics: Metrics = Metrics(service='raster-api', namespace='veda-backend') + +logger: Logger = Logger(service="raster-api", namespace="veda-backend") +metrics: Metrics = Metrics(service="raster-api", namespace="veda-backend") tracer: Tracer = Tracer() + class LoggerRouteHandler(APIRoute): + """Extension of base APIRoute to add context to log statements, as well as record usage metricss""" + def get_route_handler(self) -> Callable: + """Overide route handler method to add logs, metrics, tracing""" original_route_handler = super().get_route_handler() + async def route_handler(request: Request) -> Response: # Add fastapi context to logs ctx = { @@ -21,9 +28,15 @@ async def route_handler(request: Request) -> Response: } logger.append_keys(fastapi=ctx) logger.info("Received request") - metrics.add_metric(name='/'.join(str(request.url).split('/')[:2]), unit=MetricUnit.Count, value=1) + metrics.add_metric( + name="/".join( + str(request.url).split("/")[:2] + ), # enough detail to capture search IDs, but not individual tile coords + unit=MetricUnit.Count, + value=1, + ) tracer.put_annotation(key="path", value=request.url.path) tracer.capture_method(original_route_handler)(request) return await original_route_handler(request) - + return route_handler diff --git a/raster_api/runtime/src/tracing.py b/raster_api/runtime/src/tracing.py deleted file mode 100644 index a0ccc589..00000000 --- a/raster_api/runtime/src/tracing.py +++ /dev/null @@ -1,69 +0,0 @@ -import copy -import re -import wrapt -from operator import methodcaller - -from aws_xray_sdk.ext.dbapi2 import XRayTracedConn, XRayTracedCursor - - -def patch_psycopg_3(): - wrapt.wrap_function_wrapper( - 'psycopg', - 'connect', - _xray_traced_connect - ) - wrapt.wrap_function_wrapper( - 'psycopg.extensions', - 'register_type', - _xray_register_type_fix - ) - wrapt.wrap_function_wrapper( - 'psycopg.extensions', - 'quote_ident', - _xray_register_type_fix - ) - - wrapt.wrap_function_wrapper( - 'psycopg.extras', - 'register_default_jsonb', - _xray_register_default_jsonb_fix - ) - - -def _xray_traced_connect(wrapped, instance, args, kwargs): - conn = wrapped(*args, **kwargs) - parameterized_dsn = {c[0]: c[-1] for c in map(methodcaller('split', '='), conn.dsn.split(' '))} - meta = { - 'database_type': 'PostgreSQL', - 'url': 'postgresql://{}@{}:{}/{}'.format( - parameterized_dsn.get('user', 'unknown'), - parameterized_dsn.get('host', 'unknown'), - parameterized_dsn.get('port', 'unknown'), - parameterized_dsn.get('dbname', 'unknown'), - ), - 'user': parameterized_dsn.get('user', 'unknown'), - 'database_version': str(conn.server_version), - 'driver_version': 'Psycopg' - } - - return XRayTracedConn(conn, meta) - - -def _xray_register_type_fix(wrapped, instance, args, kwargs): - """Send the actual connection or curser to register type.""" - our_args = list(copy.copy(args)) - if len(our_args) == 2 and isinstance(our_args[1], (XRayTracedConn, XRayTracedCursor)): - our_args[1] = our_args[1].__wrapped__ - - return wrapped(*our_args, **kwargs) - - -def _xray_register_default_jsonb_fix(wrapped, instance, args, kwargs): - our_kwargs = dict() - for key, value in kwargs.items(): - if key == "conn_or_curs" and isinstance(value, (XRayTracedConn, XRayTracedCursor)): - # unwrap the connection or cursor to be sent to register_default_jsonb - value = value.__wrapped__ - our_kwargs[key] = value - - return wrapped(*args, **our_kwargs) \ No newline at end of file diff --git a/stac_api/infrastructure/construct.py b/stac_api/infrastructure/construct.py index dc6537c1..c6982025 100644 --- a/stac_api/infrastructure/construct.py +++ b/stac_api/infrastructure/construct.py @@ -55,7 +55,7 @@ def __init__( **{k.upper(): v for k, v in delta_stac_settings.env.items()}, }, log_retention=aws_logs.RetentionDays.ONE_WEEK, - tracing=aws_lambda.Tracing.ACTIVE + tracing=aws_lambda.Tracing.ACTIVE, ) # # lambda_function.add_environment(key="TITILER_ENDPOINT", value=raster_api.url) diff --git a/stac_api/runtime/handler.py b/stac_api/runtime/handler.py index a46554e2..176be13b 100644 --- a/stac_api/runtime/handler.py +++ b/stac_api/runtime/handler.py @@ -6,7 +6,6 @@ from src.app import app from src.monitoring import logger, metrics, tracer - logging.getLogger("mangum.lifespan").setLevel(logging.ERROR) logging.getLogger("mangum.http").setLevel(logging.ERROR) @@ -18,4 +17,4 @@ # Add logging handler = logger.inject_lambda_context(handler, clear_state=True) # Add metrics last to properly flush metrics. -handler = metrics.log_metrics(handler, capture_cold_start_metric=True) \ No newline at end of file +handler = metrics.log_metrics(handler, capture_cold_start_metric=True) diff --git a/stac_api/runtime/src/app.py b/stac_api/runtime/src/app.py index c339e9be..96313638 100644 --- a/stac_api/runtime/src/app.py +++ b/stac_api/runtime/src/app.py @@ -1,6 +1,7 @@ """FastAPI application using PGStac. Based on https://github.com/developmentseed/eoAPI/tree/master/src/eoapi/stac """ +from aws_lambda_powertools.metrics import MetricUnit from src.config import ApiSettings, TilesApiSettings from src.config import extensions as PgStacExtensions from src.config import get_request_model as GETModel @@ -17,9 +18,8 @@ from starlette.responses import HTMLResponse, JSONResponse from starlette.templating import Jinja2Templates from starlette_cramjam.middleware import CompressionMiddleware -from aws_lambda_powertools.metrics import MetricUnit -from .monitoring import logger, metrics, tracer, LoggerRouteHandler +from .monitoring import logger, metrics, tracer try: from importlib.resources import files as resources_files # type: ignore @@ -75,9 +75,11 @@ async def viewer_page(request: Request): media_type="text/html", ) + # If the correlation header is used in the UI, we can analyze traces that originate from a given user or client @app.middleware("http") async def add_correlation_id(request: Request, call_next): + """Add correlation ids to all requests and subsequent logs/traces""" # Get correlation id from X-Correlation-Id header corr_id = request.headers.get("x-correlation-id") if not corr_id: @@ -86,7 +88,7 @@ async def add_correlation_id(request: Request, call_next): corr_id = request.scope["aws.context"].aws_request_id except KeyError: # If empty, use uuid - corr_id = 'local' + corr_id = "local" # Add correlation id to logs logger.set_correlation_id(corr_id) # Add correlation id to traces @@ -98,12 +100,15 @@ async def add_correlation_id(request: Request, call_next): logger.info("Request completed") return response + @app.exception_handler(Exception) async def validation_exception_handler(request, err): + """Handle exceptions that aren't caught elsewhere""" metrics.add_metric(name="UnhandledExceptions", unit=MetricUnit.Count, value=1) logger.error("Unhandled exception") return JSONResponse(status_code=500, content={"detail": "Internal Server Error"}) + @app.on_event("startup") async def startup_event(): """Connect to database on startup.""" diff --git a/stac_api/runtime/src/extension.py b/stac_api/runtime/src/extension.py index a5ac7a3c..52cc346c 100644 --- a/stac_api/runtime/src/extension.py +++ b/stac_api/runtime/src/extension.py @@ -10,7 +10,7 @@ from stac_fastapi.types.extension import ApiExtension from starlette.requests import Request -from .monitoring import LoggerRouteHandler, logger, metrics, tracer +from .monitoring import LoggerRouteHandler, tracer router = APIRouter(route_class=LoggerRouteHandler) diff --git a/stac_api/runtime/src/monitoring.py b/stac_api/runtime/src/monitoring.py index 2eab9f30..67fa4a43 100644 --- a/stac_api/runtime/src/monitoring.py +++ b/stac_api/runtime/src/monitoring.py @@ -1,17 +1,24 @@ +"""Observability utils""" +from typing import Callable + from aws_lambda_powertools import Logger, Metrics, Tracer from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 + from fastapi import Request, Response from fastapi.routing import APIRoute -from typing import Callable -from functools import wraps - -logger: Logger = Logger(service='stac-api', namespace='veda-backend') -metrics: Metrics = Metrics(service='stac-api', namespace='veda-backend') + +logger: Logger = Logger(service="stac-api", namespace="veda-backend") +metrics: Metrics = Metrics(service="stac-api", namespace="veda-backend") tracer: Tracer = Tracer() + class LoggerRouteHandler(APIRoute): + """Extension of base APIRoute to add context to log statements, as well as record usage metricss""" + def get_route_handler(self) -> Callable: + """Overide route handler method to add logs, metrics, tracing""" original_route_handler = super().get_route_handler() + async def route_handler(request: Request) -> Response: # Add fastapi context to logs ctx = { @@ -21,9 +28,15 @@ async def route_handler(request: Request) -> Response: } logger.append_keys(fastapi=ctx) logger.info("Received request") - metrics.add_metric(name='/'.join(str(request.url).split('/')[:2]), unit=MetricUnit.Count, value=1) + metrics.add_metric( + name="/".join( + str(request.url).split("/")[:2] + ), # enough detail to capture search IDs, but not individual tile coords + unit=MetricUnit.Count, + value=1, + ) tracer.put_annotation(key="path", value=request.url.path) tracer.capture_method(original_route_handler)(request) return await original_route_handler(request) - + return route_handler From 632a919d2e18fb3743e69c7a2fda88291b8f20d0 Mon Sep 17 00:00:00 2001 From: ividito Date: Thu, 10 Nov 2022 12:27:14 -0400 Subject: [PATCH 07/23] Fix cog viewer not working --- raster_api/runtime/src/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/raster_api/runtime/src/app.py b/raster_api/runtime/src/app.py index 1b4ee895..495b0af0 100644 --- a/raster_api/runtime/src/app.py +++ b/raster_api/runtime/src/app.py @@ -51,7 +51,7 @@ optional_headers=optional_headers, gdal_config=settings.get_gdal_config(), dataset_dependency=DatasetParams, - router=router, + router=APIRouter(route_class=LoggerRouteHandler), ) app.include_router(mosaic.router, prefix="/mosaic", tags=["Mosaic"]) @@ -62,7 +62,7 @@ optional_headers=optional_headers, router_prefix="/stac", gdal_config=settings.get_gdal_config(), - router=router, + router=APIRouter(route_class=LoggerRouteHandler), ) app.include_router(stac.router, tags=["Items"], prefix="/stac") @@ -70,7 +70,7 @@ router_prefix="/cog", optional_headers=optional_headers, gdal_config=settings.get_gdal_config(), - router=router, + router=APIRouter(route_class=LoggerRouteHandler), ) From 6fd51cfb9b2e201977da4e9cee083441f622a467 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Sun, 11 Dec 2022 16:42:41 -0700 Subject: [PATCH 08/23] rename variables: delta->veda --- .example.env | 20 ++++---- .github/workflows/tests/test_raster.py | 2 +- .github/workflows/tests/test_stac.py | 2 +- README.md | 38 +++++++------- app.py | 50 +++++++++---------- config.py | 16 +++--- database/infrastructure/config.py | 12 ++--- database/infrastructure/construct.py | 30 +++++------ docker-compose.yml | 10 ++-- domain/infrastructure/config.py | 10 ++-- domain/infrastructure/construct.py | 28 +++++------ network/infrastructure/construct.py | 20 ++++---- raster_api/infrastructure/config.py | 8 +-- raster_api/infrastructure/construct.py | 46 ++++++++--------- raster_api/runtime/README.md | 2 +- raster_api/runtime/setup.py | 6 +-- raster_api/runtime/src/__init__.py | 2 +- raster_api/runtime/src/app.py | 4 +- raster_api/runtime/src/config.py | 4 +- raster_api/runtime/src/dependencies.py | 2 +- setup.py | 8 +-- stac_api/infrastructure/config.py | 6 +-- stac_api/infrastructure/construct.py | 10 ++-- stac_api/runtime/README.md | 2 +- stac_api/runtime/setup.py | 4 +- stac_api/runtime/src/__init__.py | 2 +- stac_api/runtime/src/config.py | 4 +- .../standalone_app.py | 2 +- support_scripts/README.md | 4 +- support_scripts/rotate_pgstac_pasword.py | 2 +- 30 files changed, 178 insertions(+), 178 deletions(-) diff --git a/.example.env b/.example.env index 8b5f2f6d..02344b94 100644 --- a/.example.env +++ b/.example.env @@ -4,16 +4,16 @@ CDK_DEFAULT_REGION=[REQUIRED IF DEPLOYING TO EXISTING VPC] STAGE=[FILL ME IN] -DELTA_DB_PGSTAC_VERSION=0.6.6 -DELTA_DB_SCHEMA_VERSION=0.1.0 -DELTA_DB_SNAPSHOT_ID=[OPTIONAL BUT **REQUIRED** FOR ALL DEPLOYMENTS AFTER BASING DEPLOYMENT ON SNAPSHOT] +VEDA_DB_PGSTAC_VERSION=0.6.6 +VEDA_DB_SCHEMA_VERSION=0.1.0 +VEDA_DB_SNAPSHOT_ID=[OPTIONAL BUT **REQUIRED** FOR ALL DEPLOYMENTS AFTER BASING DEPLOYMENT ON SNAPSHOT] -DELTA_DOMAIN_HOSTED_ZONE_ID=[OPTIONAL] -DELTA_DOMAIN_HOSTED_ZONE_NAME=[OPTIONAL] -DELTA_DOMAIN_API_PREFIX=[OPTIONAL CUSTOM PREFIX DECOUPLES API URL FROM A/B VERSION OF STACK DEPLOYED] +VEDA_DOMAIN_HOSTED_ZONE_ID=[OPTIONAL] +VEDA_DOMAIN_HOSTED_ZONE_NAME=[OPTIONAL] +VEDA_DOMAIN_API_PREFIX=[OPTIONAL CUSTOM PREFIX DECOUPLES API URL FROM A/B VERSION OF STACK DEPLOYED] -DELTA_DOMAIN_ALT_HOSTED_ZONE_ID=[OPTIONAL SECOND HOSTED ZONE] -DELTA_DOMAIN_ALT_HOSTED_ZONE_NAME=[OPTIONAL SECOND DOMAIN] +VEDA_DOMAIN_ALT_HOSTED_ZONE_ID=[OPTIONAL SECOND HOSTED ZONE] +VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME=[OPTIONAL SECOND DOMAIN] -DELTA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE -DELTA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API] +VEDA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE +VEDA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API] diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py index 19019736..0da4adc8 100644 --- a/.github/workflows/tests/test_raster.py +++ b/.github/workflows/tests/test_raster.py @@ -1,4 +1,4 @@ -"""test delta-backend.raster.""" +"""test veda-backend.raster.""" import httpx diff --git a/.github/workflows/tests/test_stac.py b/.github/workflows/tests/test_stac.py index bd94ec5d..0bdc80d7 100644 --- a/.github/workflows/tests/test_stac.py +++ b/.github/workflows/tests/test_stac.py @@ -1,4 +1,4 @@ -"""test Delta-backend STAC.""" +"""test veda-backend STAC.""" import httpx diff --git a/README.md b/README.md index 3120033c..586d2087 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# delta-backend -This project deploys a complete backend for a [SpatioTemporal Asset Catalog](https://stacspec.org/) including a postgres database, a metadata API, and raster tiling API. Delta-backend is a non-forked version of the [eoAPI](https://github.com/developmentseed/eoAPI) demo project. Delta-backend is decoupled from the demo project to selectively incorporate new stable functionality from the fast moving development in eoAPI while providing a continuous baseline for delta-backend users and to support project specific business and deployment logic. +# veda-backend +This project deploys a complete backend for a [SpatioTemporal Asset Catalog](https://stacspec.org/) including a postgres database, a metadata API, and raster tiling API. Veda-backend is a non-forked version of the [eoAPI](https://github.com/developmentseed/eoAPI) demo project. Veda-backend is decoupled from the demo project to selectively incorporate new stable functionality from the fast moving development in eoAPI while providing a continuous baseline for veda-backend users and to support project specific business and deployment logic. The primary tools employed in the [eoAPI demo](https://github.com/developmentseed/eoAPI) and this project are: - [stac-spec](https://github.com/radiantearth/stac-spec) @@ -20,7 +20,7 @@ This repo includes CDK scripts to deploy a PgStac AWS RDS database and other res ### Enviroment variables -An [.example.env](.example.env) template is supplied for for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `-backend/-env`, for example `delta-backend/dev-env`. +An [.example.env](.example.env) template is supplied for for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `-backend/-env`, for example `veda-backend/dev-env`. ### Fetch environment variables using AWS CLI @@ -35,23 +35,23 @@ aws secretsmanager get-secret-value --secret-id ${AWS_SECRET_ID} --query SecretS | Name | Explanation | | --- | --- | -| `APP_NAME` | Optional app name used to name stack and resources, defaults to `delta` | +| `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda` | | `STAGE` | **REQUIRED** Deployment stage used to name stack and resources, i.e. `dev`, `staging`, `prod` | | `VPC_ID` | Optional resource identifier of VPC, if none a new VPC with public and private subnets will be provisioned. | | `PERMISSIONS_BOUNDARY_POLICY_NAME` | Optional name of IAM policy to define stack permissions boundary | | `CDK_DEFAULT_ACCOUNT` | When deploying from a local machine the AWS account id is required to deploy to an exiting VPC | | `CDK_DEFAULT_REGION` | When deploying from a local machine the AWS region id is required to deploy to an exiting VPC | -| `DELTA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.5 | -| `DELTA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom delta-backend schema, i.e. 0.1.1 | -| `DELTA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot | -| `DELTA_DB_PRIVATE_SUBNETS` | Optional boolean to deploy database to private subnet | -| `DELTA_DOMAIN_HOSTED_ZONE_ID` | Optional Route53 zone identifier if using a custom domain name | -| `DELTA_DOMAIN_HOSTED_ZONE_NAME` | Optional custom domain name, i.e. delta-backend.xyz | -| `DELTA_DOMAIN_ALT_HOSTED_ZONE_ID` | Optional second Route53 zone identifier if using a custom domain name | -| `DELTA_DOMAIN_ALT_HOSTED_ZONE_NAME` | Optional second custom domain name, i.e. alt-delta-backend.xyz | -| `DELTA_DOMAIN_API_PREFIX` | Optional domain prefix override supports using a custom prefix instead of the STAGE variabe (an alternate version of the stack can be deployed with a unique STAGE=altprod and after testing prod API traffic can be cut over to the alternate version of the stack by setting the prefix to prod) | -| `DELTA_RASTER_ENABLE_MOSAIC_SEARCH` | Optional deploy the raster API with the mosaic/list endpoint TRUE/FALSE | -| `DELTA_RASTER_DATA_ACCESS_ROLE_ARN` | Optional arn of IAM Role to be assumed by raster-api for S3 bucket data access, if not provided default role for the lambda construct is used | +| `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.5 | +| `VEDA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom veda-backend schema, i.e. 0.1.1 | +| `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot | +| `VEDA_DB_PRIVATE_SUBNETS` | Optional boolean to deploy database to private subnet | +| `VEDA_DOMAIN_HOSTED_ZONE_ID` | Optional Route53 zone identifier if using a custom domain name | +| `VEDA_DOMAIN_HOSTED_ZONE_NAME` | Optional custom domain name, i.e. veda-backend.xyz | +| `VEDA_DOMAIN_ALT_HOSTED_ZONE_ID` | Optional second Route53 zone identifier if using a custom domain name | +| `VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME` | Optional second custom domain name, i.e. alt-veda-backend.xyz | +| `VEDA_DOMAIN_API_PREFIX` | Optional domain prefix override supports using a custom prefix instead of the STAGE variabe (an alternate version of the stack can be deployed with a unique STAGE=altprod and after testing prod API traffic can be cut over to the alternate version of the stack by setting the prefix to prod) | +| `VEDA_RASTER_ENABLE_MOSAIC_SEARCH` | Optional deploy the raster API with the mosaic/list endpoint TRUE/FALSE | +| `VEDA_RASTER_DATA_ACCESS_ROLE_ARN` | Optional arn of IAM Role to be assumed by raster-api for S3 bucket data access, if not provided default role for the lambda construct is used | ### Deploying to the cloud @@ -69,15 +69,15 @@ python3 -m pip install -e ".[dev,deploy,test]" #### Run the deployment ``` -cdk synth +# Review what infrastructure changes your deployment will cause +cdk diff +# Execute deployment, security changes will require approval for deployment cdk deploy ``` #### Check CloudFormation deployment status After logging in to the console at https://.signin.aws.amazon.com/console the status of the CloudFormation stack can be viewed here: https://.console.aws.amazon.com/cloudformation/home. - -The CloudFormation stack name is the combination of the app name and deployment stage environment variables https://github.com/NASA-IMPACT/delta-backend/blob/develop/config.py#L11 ## Deleting the CloudFormation stack @@ -186,7 +186,7 @@ https://github.com/NASA-IMPACT/veda-documentation # STAC community resources ## STAC browser -Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is a browser for STAC catalogs. The demo version of this browser [radiantearth.github.io/stac-browser](https://radiantearth.github.io/stac-browser/#/) can be used to browse the contents of the delta-backend STAC catalog, paste the delta-backend stac-api URL deployed by this project in the demo and click load. Read more about the recent developments and usage of stac-browser [here](https://medium.com/radiant-earth-insights/the-exciting-future-of-the-stac-browser-2351143aa24b). +Radiant Earth's [stac-browser](https://github.com/radiantearth/stac-browser) is a browser for STAC catalogs. The demo version of this browser [radiantearth.github.io/stac-browser](https://radiantearth.github.io/stac-browser/#/) can be used to browse the contents of the veda-backend STAC catalog, paste the veda-backend stac-api URL deployed by this project in the demo and click load. Read more about the recent developments and usage of stac-browser [here](https://medium.com/radiant-earth-insights/the-exciting-future-of-the-stac-browser-2351143aa24b). # License This project is licensed under **Apache 2**, see the [LICENSE](LICENSE) file for more details. \ No newline at end of file diff --git a/app.py b/app.py index bec2fc90..8f28356b 100644 --- a/app.py +++ b/app.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -""" CDK Configuration for the delta-backend stack.""" +""" CDK Configuration for the veda-backend stack.""" from aws_cdk import App, Stack, Tags, aws_iam from constructs import Construct -from config import delta_app_settings +from config import veda_app_settings from database.infrastructure.construct import RdsConstruct from domain.infrastructure.construct import DomainConstruct from network.infrastructure.construct import VpcConstruct @@ -14,46 +14,46 @@ app = App() -class DeltaStack(Stack): - """CDK stack for the delta-backend stack.""" +class VedaStack(Stack): + """CDK stack for the veda-backend stack.""" def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: """.""" super().__init__(scope, construct_id, **kwargs) - if delta_app_settings.permissions_boundary_policy_name: + if veda_app_settings.permissions_boundary_policy_name: permission_boundary_policy = aws_iam.Policy.from_policy_name( self, "permission-boundary", - delta_app_settings.permissions_boundary_policy_name, + veda_app_settings.permissions_boundary_policy_name, ) aws_iam.PermissionsBoundary.of(self).apply(permission_boundary_policy) -delta_stack = DeltaStack( +veda_stack = VedaStack( app, - f"{delta_app_settings.app_name}-{delta_app_settings.stage_name()}", - env=delta_app_settings.cdk_env(), + f"{veda_app_settings.app_name}-{veda_app_settings.stage_name()}", + env=veda_app_settings.cdk_env(), ) -if delta_app_settings.vpc_id: +if veda_app_settings.vpc_id: vpc = VpcConstruct( - delta_stack, + veda_stack, "network", - vpc_id=delta_app_settings.vpc_id, - stage=delta_app_settings.stage_name(), + vpc_id=veda_app_settings.vpc_id, + stage=veda_app_settings.stage_name(), ) else: - vpc = VpcConstruct(delta_stack, "network", stage=delta_app_settings.stage_name()) + vpc = VpcConstruct(veda_stack, "network", stage=veda_app_settings.stage_name()) database = RdsConstruct( - delta_stack, "database", vpc.vpc, stage=delta_app_settings.stage_name() + veda_stack, "database", vpc.vpc, stage=veda_app_settings.stage_name() ) -domain = DomainConstruct(delta_stack, "domain", stage=delta_app_settings.stage_name()) +domain = DomainConstruct(veda_stack, "domain", stage=veda_app_settings.stage_name()) raster_api = RasterApiLambdaConstruct( - delta_stack, + veda_stack, "raster-api", vpc=vpc.vpc, database=database, @@ -61,7 +61,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ) stac_api = StacApiLambdaConstruct( - delta_stack, + veda_stack, "stac-api", vpc=vpc.vpc, database=database, @@ -70,17 +70,17 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ) # TODO this conditional supports deploying a second set of APIs to a separate custom domain and should be removed if no longer necessary -if delta_app_settings.alt_domain(): +if veda_app_settings.alt_domain(): alt_domain = DomainConstruct( - delta_stack, + veda_stack, "alt-domain", - stage=delta_app_settings.stage_name(), + stage=veda_app_settings.stage_name(), alt_domain=True, ) alt_raster_api = RasterApiLambdaConstruct( - delta_stack, + veda_stack, "alt-raster-api", vpc=vpc.vpc, database=database, @@ -88,7 +88,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ) alt_stac_api = StacApiLambdaConstruct( - delta_stack, + veda_stack, "alt-stac-api", vpc=vpc.vpc, database=database, @@ -97,8 +97,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ) for key, value in { - "Project": delta_app_settings.app_name, - "Stack": delta_app_settings.stage_name(), + "Project": veda_app_settings.app_name, + "Stack": veda_app_settings.stage_name(), "Client": "nasa-impact", "Owner": "ds", }.items(): diff --git a/config.py b/config.py index fcda022e..cd461c6a 100644 --- a/config.py +++ b/config.py @@ -4,12 +4,12 @@ from pydantic import BaseSettings, Field -class deltaAppSettings(BaseSettings): +class vedaAppSettings(BaseSettings): """Application settings.""" # App name and deployment stage app_name: Optional[str] = Field( - "delta-backend", + "veda-backend", description="Optional app name used to name stack and resources", ) stage: str = Field( @@ -38,13 +38,13 @@ class deltaAppSettings(BaseSettings): None, description="Name of IAM policy to define stack permissions boundary", ) - delta_domain_alt_hosted_zone_id: Optional[str] = Field( + veda_domain_alt_hosted_zone_id: Optional[str] = Field( None, description="Route53 zone identifier if using a custom domain name", ) - delta_domain_alt_hosted_zone_name: Optional[str] = Field( + veda_domain_alt_hosted_zone_name: Optional[str] = Field( None, - description="Custom domain name, i.e. delta-backend.xyz", + description="Custom domain name, i.e. veda-backend.xyz", ) def cdk_env(self) -> dict: @@ -62,8 +62,8 @@ def alt_domain(self) -> bool: """True if alternative domain and host parameters provided""" return all( [ - self.delta_domain_alt_hosted_zone_id, - self.delta_domain_alt_hosted_zone_name, + self.veda_domain_alt_hosted_zone_id, + self.veda_domain_alt_hosted_zone_name, ] ) @@ -77,4 +77,4 @@ class Config: env_file = ".env" -delta_app_settings = deltaAppSettings() +veda_app_settings = vedaAppSettings() diff --git a/database/infrastructure/config.py b/database/infrastructure/config.py index e440023d..6cb15baa 100644 --- a/database/infrastructure/config.py +++ b/database/infrastructure/config.py @@ -1,10 +1,10 @@ -"""Delta-backend database construct configuration.""" +"""Veda-backend database construct configuration.""" from typing import Optional from pydantic import BaseSettings, Field -class deltaDBSettings(BaseSettings): +class vedaDBSettings(BaseSettings): """Application settings.""" dbname: str = Field( @@ -16,7 +16,7 @@ class deltaDBSettings(BaseSettings): description="Name of admin role for postgres database", ) user: str = Field( - "delta", + "veda", description="Name of pgstac role for postgres database", ) pgstac_version: str = Field( @@ -25,7 +25,7 @@ class deltaDBSettings(BaseSettings): ) schema_version: str = Field( ..., - description="The version of the custom delta-backend schema, i.e. 0.1.1", + description="The version of the custom veda-backend schema, i.e. 0.1.1", ) snapshot_id: Optional[str] = Field( None, @@ -60,7 +60,7 @@ class Config: """model config.""" env_file = ".env" - env_prefix = "DELTA_DB_" + env_prefix = "VEDA_DB_" -delta_db_settings = deltaDBSettings() +veda_db_settings = vedaDBSettings() diff --git a/database/infrastructure/construct.py b/database/infrastructure/construct.py index f1257eca..6f17279f 100644 --- a/database/infrastructure/construct.py +++ b/database/infrastructure/construct.py @@ -1,4 +1,4 @@ -"""CDK Construct for delta-backend RDS instance.""" +"""CDK Construct for veda-backend RDS instance.""" import json import os from typing import Union @@ -17,7 +17,7 @@ ) from constructs import Construct -from .config import delta_db_settings +from .config import veda_db_settings # https://github.com/developmentseed/eoAPI/blob/master/deployment/cdk/app.py @@ -39,8 +39,8 @@ def __init__( """.""" super().__init__(scope, construct_id) - pgstac_version = delta_db_settings.pgstac_version - delta_schema_version = delta_db_settings.schema_version + pgstac_version = veda_db_settings.pgstac_version + veda_schema_version = veda_db_settings.schema_version handler = aws_lambda.Function( self, @@ -97,7 +97,7 @@ def __init__( "pgstac_version": pgstac_version, "conn_secret_arn": database.secret.secret_arn, "new_user_secret_arn": self.secret.secret_arn, - "delta_schema_version": delta_schema_version, + "veda_schema_version": veda_schema_version, }, removal_policy=RemovalPolicy.RETAIN, # This retains the custom resource (which doesn't really exist), not the database ) @@ -127,10 +127,10 @@ def __init__( stack_name = Stack.of(self).stack_name # Configure accessibility - publicly_accessible = False if delta_db_settings.private_subnets else True + publicly_accessible = False if veda_db_settings.private_subnets else True subnet_type = ( aws_ec2.SubnetType.PRIVATE_ISOLATED - if delta_db_settings.private_subnets is True + if veda_db_settings.private_subnets is True else aws_ec2.SubnetType.PUBLIC ) @@ -143,23 +143,23 @@ def __init__( "parameter-group", engine=engine, parameters={ - "max_locks_per_transaction": delta_db_settings.max_locks_per_transaction, - "work_mem": delta_db_settings.work_mem, - "temp_buffers": delta_db_settings.temp_buffers, + "max_locks_per_transaction": veda_db_settings.max_locks_per_transaction, + "work_mem": veda_db_settings.work_mem, + "temp_buffers": veda_db_settings.temp_buffers, }, ) # Create a new database instance from snapshot if provided - if delta_db_settings.snapshot_id: + if veda_db_settings.snapshot_id: # For the database from snapshot we will need a new master secret credentials = aws_rds.SnapshotCredentials.from_generated_secret( - username=delta_db_settings.admin_user + username=veda_db_settings.admin_user ) database = aws_rds.DatabaseInstanceFromSnapshot( self, id="rds", - snapshot_identifier=delta_db_settings.snapshot_id, + snapshot_identifier=veda_db_settings.snapshot_id, instance_identifier=f"{stack_name}-postgres", vpc=vpc, engine=engine, @@ -197,8 +197,8 @@ def __init__( self, "pgstac", database=database, - new_dbname=delta_db_settings.dbname, - new_username=delta_db_settings.user, + new_dbname=veda_db_settings.dbname, + new_username=veda_db_settings.user, secrets_prefix=stack_name, stage=stage, ) diff --git a/docker-compose.yml b/docker-compose.yml index 0522cb4a..395e6a54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: stac: - container_name: delta.stac + container_name: veda.stac build: context: . dockerfile: local/Dockerfile.stac @@ -40,7 +40,7 @@ services: - ./scripts:/tmp/scripts raster: - container_name: delta.raster + container_name: veda.raster platform: linux/amd64 build: context: . @@ -83,7 +83,7 @@ services: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} # API Config - - DELTA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE + - VEDA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE depends_on: - database @@ -93,7 +93,7 @@ services: - $HOME/.aws/credentials:/root/.aws/credentials database: - container_name: delta.db + container_name: veda.db platform: linux/amd64 image: ghcr.io/stac-utils/pgstac:v0.6.6 environment: @@ -112,4 +112,4 @@ services: networks: default: - name: delta-network + name: veda-network diff --git a/domain/infrastructure/config.py b/domain/infrastructure/config.py index bfb6d999..e2b52a87 100644 --- a/domain/infrastructure/config.py +++ b/domain/infrastructure/config.py @@ -5,14 +5,14 @@ from pydantic import BaseSettings, Field -class deltaDomainSettings(BaseSettings): +class vedaDomainSettings(BaseSettings): """Application settings""" hosted_zone_id: Optional[str] = Field( None, description="Route53 hosted zone identifier if using a custom domain name" ) hosted_zone_name: Optional[str] = Field( - None, description="Custom domain name, i.e. delta-backend.xyz" + None, description="Custom domain name, i.e. veda-backend.xyz" ) api_prefix: Optional[str] = Field( None, @@ -29,14 +29,14 @@ class deltaDomainSettings(BaseSettings): None, description="Second Route53 zone identifier if using a custom domain name" ) alt_hosted_zone_name: Optional[str] = Field( - None, description="Second custom domain name, i.e. alt-delta-backend.xyz" + None, description="Second custom domain name, i.e. alt-veda-backend.xyz" ) class Config: """model config""" env_file = ".env" - env_prefix = "DELTA_DOMAIN_" + env_prefix = "VEDA_DOMAIN_" -delta_domain_settings = deltaDomainSettings() +veda_domain_settings = vedaDomainSettings() diff --git a/domain/infrastructure/construct.py b/domain/infrastructure/construct.py index aea57d38..9efd0011 100644 --- a/domain/infrastructure/construct.py +++ b/domain/infrastructure/construct.py @@ -1,4 +1,4 @@ -"""CDK Construct for a custom API domain (under the route 53 domain: delta-backend.xyz).""" +"""CDK Construct for a custom API domain.""" from typing import Optional from aws_cdk import ( @@ -10,11 +10,11 @@ ) from constructs import Construct -from .config import delta_domain_settings +from .config import veda_domain_settings class DomainConstruct(Construct): - """CDK Construct for a custom API domain (under the route 53 domain: delta-backend.xyz).""" + """CDK Construct for a custom API domain.""" def __init__( self, @@ -31,16 +31,16 @@ def __init__( self.raster_domain_name = None if ( - delta_domain_settings.hosted_zone_id - and delta_domain_settings.hosted_zone_name + veda_domain_settings.hosted_zone_id + and veda_domain_settings.hosted_zone_name ): # If alternative custom domain provided, use it instead of the default if alt_domain is True: - hosted_zone_name = delta_domain_settings.alt_hosted_zone_name - hosted_zone_id = delta_domain_settings.alt_hosted_zone_id + hosted_zone_name = veda_domain_settings.alt_hosted_zone_name + hosted_zone_id = veda_domain_settings.alt_hosted_zone_id else: - hosted_zone_name = delta_domain_settings.hosted_zone_name - hosted_zone_id = delta_domain_settings.hosted_zone_id + hosted_zone_name = veda_domain_settings.hosted_zone_name + hosted_zone_id = veda_domain_settings.hosted_zone_id hosted_zone = aws_route53.HostedZone.from_hosted_zone_attributes( self, @@ -58,9 +58,9 @@ def __init__( ) # Use custom api prefix if provided or deployment stage if not - if delta_domain_settings.api_prefix: - raster_url_prefix = f"{delta_domain_settings.api_prefix.lower()}-raster" - stac_url_prefix = f"{delta_domain_settings.api_prefix.lower()}-stac" + if veda_domain_settings.api_prefix: + raster_url_prefix = f"{veda_domain_settings.api_prefix.lower()}-raster" + stac_url_prefix = f"{veda_domain_settings.api_prefix.lower()}-stac" else: raster_url_prefix = f"{stage.lower()}-raster" stac_url_prefix = f"{stage.lower()}-stac" @@ -84,7 +84,7 @@ def __init__( regional_hosted_zone_id=self.raster_domain_name.regional_hosted_zone_id, ) ), - # Note: CDK will append the hosted zone name (eg: `delta-backend.xyz` to this record name) + # Note: CDK will append the hosted zone name (eg: `veda-backend.xyz` to this record name) record_name=raster_url_prefix, ) @@ -105,7 +105,7 @@ def __init__( regional_hosted_zone_id=self.stac_domain_name.regional_hosted_zone_id, ) ), - # Note: CDK will append the hosted zone name (eg: `delta-backend.xyz` to this record name) + # Note: CDK will append the hosted zone name (eg: `veda-backend.xyz` to this record name) record_name=stac_url_prefix, ) diff --git a/network/infrastructure/construct.py b/network/infrastructure/construct.py index 88c48da9..fa8cd3c7 100644 --- a/network/infrastructure/construct.py +++ b/network/infrastructure/construct.py @@ -1,5 +1,5 @@ """ -CDK construct for delta-backend VPC. +CDK construct for veda-backend VPC. """ from typing import Optional @@ -12,7 +12,7 @@ # https://github.com/aws-samples/aws-cdk-examples/tree/master/python/new-vpc-alb-asg-mysql # https://github.com/aws-samples/aws-cdk-examples/tree/master/python/docker-app-with-asg-alb class VpcConstruct(Construct): - """CDK construct for delta-backend VPC.""" + """CDK construct for veda-backend VPC.""" def __init__( self, @@ -35,30 +35,30 @@ def __init__( else: # Union of pydantic base settings is unpredictable so set stage settings conditionally if stage == "prod": - delta_vpc_settings = prod_vpc_settings + veda_vpc_settings = prod_vpc_settings elif stage == "staging": - delta_vpc_settings = staging_vpc_settings + veda_vpc_settings = staging_vpc_settings else: - delta_vpc_settings = dev_vpc_settings + veda_vpc_settings = dev_vpc_settings public_subnet = aws_ec2.SubnetConfiguration( name="public", subnet_type=aws_ec2.SubnetType.PUBLIC, - cidr_mask=delta_vpc_settings.public_mask, + cidr_mask=veda_vpc_settings.public_mask, ) private_subnet = aws_ec2.SubnetConfiguration( name="private", subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_NAT, - cidr_mask=delta_vpc_settings.private_mask, + cidr_mask=veda_vpc_settings.private_mask, ) self.vpc = aws_ec2.Vpc( self, "vpc", - max_azs=delta_vpc_settings.max_azs, - cidr=delta_vpc_settings.cidr, + max_azs=veda_vpc_settings.max_azs, + cidr=veda_vpc_settings.cidr, subnet_configuration=[public_subnet, private_subnet], - nat_gateways=delta_vpc_settings.nat_gateways, + nat_gateways=veda_vpc_settings.nat_gateways, ) vpc_endpoints = { diff --git a/raster_api/infrastructure/config.py b/raster_api/infrastructure/config.py index 635e1657..586b5dc3 100644 --- a/raster_api/infrastructure/config.py +++ b/raster_api/infrastructure/config.py @@ -1,12 +1,12 @@ """Settings for Raster API - any environment variables starting with -`DELTA_RASTER_` will overwrite the values of variables in this file +`VEDA_RASTER_` will overwrite the values of variables in this file """ from typing import Dict, List, Optional from pydantic import BaseSettings, Field -class deltaRasterSettings(BaseSettings): +class vedaRasterSettings(BaseSettings): """Application settings""" # Default options are optimized for CloudOptimized GeoTIFF @@ -63,7 +63,7 @@ class Config: """model config""" env_file = ".env" - env_prefix = "DELTA_RASTER_" + env_prefix = "VEDA_RASTER_" -delta_raster_settings = deltaRasterSettings() +veda_raster_settings = vedaRasterSettings() diff --git a/raster_api/infrastructure/construct.py b/raster_api/infrastructure/construct.py index 4dc6d78b..118e4257 100644 --- a/raster_api/infrastructure/construct.py +++ b/raster_api/infrastructure/construct.py @@ -14,7 +14,7 @@ ) from constructs import Construct -from .config import delta_raster_settings +from .config import veda_raster_settings class RasterApiLambdaConstruct(Construct): @@ -36,7 +36,7 @@ def __init__( # TODO config stack_name = Stack.of(self).stack_name - delta_raster_function = aws_lambda.Function( + veda_raster_function = aws_lambda.Function( self, "lambda", runtime=aws_lambda.Runtime.PYTHON_3_8, @@ -48,30 +48,30 @@ def __init__( vpc=vpc, allow_public_subnet=True, handler="handler.handler", - memory_size=delta_raster_settings.memory, - timeout=Duration.seconds(delta_raster_settings.timeout), + memory_size=veda_raster_settings.memory, + timeout=Duration.seconds(veda_raster_settings.timeout), log_retention=aws_logs.RetentionDays.ONE_WEEK, - environment=delta_raster_settings.env or {}, + environment=veda_raster_settings.env or {}, tracing=aws_lambda.Tracing.ACTIVE, ) - database.pgstac.secret.grant_read(delta_raster_function) + database.pgstac.secret.grant_read(veda_raster_function) database.pgstac.connections.allow_from( - delta_raster_function, port_range=aws_ec2.Port.tcp(5432) + veda_raster_function, port_range=aws_ec2.Port.tcp(5432) ) - delta_raster_function.add_environment( - "DELTA_RASTER_ENABLE_MOSAIC_SEARCH", - str(delta_raster_settings.enable_mosaic_search), + veda_raster_function.add_environment( + "VEDA_RASTER_ENABLE_MOSAIC_SEARCH", + str(veda_raster_settings.enable_mosaic_search), ) - delta_raster_function.add_environment( - "DELTA_RASTER_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn + veda_raster_function.add_environment( + "VEDA_RASTER_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn ) raster_api_integration = ( aws_apigatewayv2_integrations_alpha.HttpLambdaIntegration( - construct_id, delta_raster_function + construct_id, veda_raster_function ) ) @@ -89,34 +89,34 @@ def __init__( ) CfnOutput(self, "raster-api", value=self.raster_api.url) - CfnOutput(self, "raster-api-arn", value=delta_raster_function.function_arn) + CfnOutput(self, "raster-api-arn", value=veda_raster_function.function_arn) - delta_raster_function.add_to_role_policy( + veda_raster_function.add_to_role_policy( aws_iam.PolicyStatement( actions=["s3:GetObject"], resources=[ - f"arn:aws:s3:::{bucket}/{delta_raster_settings.key}" - for bucket in delta_raster_settings.buckets + f"arn:aws:s3:::{bucket}/{veda_raster_settings.key}" + for bucket in veda_raster_settings.buckets ], ) ) # Optional use sts assume role with GetObject permissions for external S3 bucket(s) - if delta_raster_settings.data_access_role_arn: + if veda_raster_settings.data_access_role_arn: # Get the role for external data access data_access_role = aws_iam.Role.from_role_arn( self, "data-access-role", - delta_raster_settings.data_access_role_arn, + veda_raster_settings.data_access_role_arn, ) # Allow this lambda to assume the data access role data_access_role.grant( - delta_raster_function.grant_principal, + veda_raster_function.grant_principal, "sts:AssumeRole", ) - delta_raster_function.add_environment( - "DELTA_RASTER_DATA_ACCESS_ROLE_ARN", - delta_raster_settings.data_access_role_arn, + veda_raster_function.add_environment( + "VEDA_RASTER_DATA_ACCESS_ROLE_ARN", + veda_raster_settings.data_access_role_arn, ) diff --git a/raster_api/runtime/README.md b/raster_api/runtime/README.md index b0646eb9..202a6236 100644 --- a/raster_api/runtime/README.md +++ b/raster_api/runtime/README.md @@ -1 +1 @@ -## delta.raster_api +## veda.raster_api diff --git a/raster_api/runtime/setup.py b/raster_api/runtime/setup.py index 034b6ec9..9501c0b3 100644 --- a/raster_api/runtime/setup.py +++ b/raster_api/runtime/setup.py @@ -1,4 +1,4 @@ -"""Setup delta.raster_api.""" +"""Setup veda.raster_api.""" from setuptools import find_namespace_packages, setup @@ -24,11 +24,11 @@ setup( - name="delta.raster_api", + name="veda.raster_api", description="", python_requires=">=3.7", packages=find_namespace_packages(exclude=["tests*"]), - package_data={"delta": ["raster/templates/*.html"]}, + package_data={"veda": ["raster/templates/*.html"]}, include_package_data=True, zip_safe=False, install_requires=inst_reqs, diff --git a/raster_api/runtime/src/__init__.py b/raster_api/runtime/src/__init__.py index d50a93a9..5221bec5 100644 --- a/raster_api/runtime/src/__init__.py +++ b/raster_api/runtime/src/__init__.py @@ -1 +1 @@ -"""delta.raster_api""" +"""veda.raster_api""" diff --git a/raster_api/runtime/src/app.py b/raster_api/runtime/src/app.py index 495b0af0..f3b9f945 100644 --- a/raster_api/runtime/src/app.py +++ b/raster_api/runtime/src/app.py @@ -7,7 +7,7 @@ from src.config import ApiSettings from src.datasetparams import DatasetParams from src.factory import MosaicTilerFactory, MultiBaseTilerFactory -from src.version import __version__ as delta_raster_version +from src.version import __version__ as veda_raster_version from fastapi import APIRouter, Depends, FastAPI, Query from starlette.middleware.cors import CORSMiddleware @@ -38,7 +38,7 @@ else: optional_headers = [] -app = FastAPI(title=settings.name, version=delta_raster_version) +app = FastAPI(title=settings.name, version=veda_raster_version) # router to be applied to all titiler route factories (improves logs with FastAPI context) router = APIRouter(route_class=LoggerRouteHandler) add_exception_handlers(app, DEFAULT_STATUS_CODES) diff --git a/raster_api/runtime/src/config.py b/raster_api/runtime/src/config.py index fa9b89d8..28234882 100644 --- a/raster_api/runtime/src/config.py +++ b/raster_api/runtime/src/config.py @@ -48,7 +48,7 @@ def get_role_credentials(role_arn: str): class ApiSettings(BaseSettings): """API settings""" - name: str = "delta-raster" + name: str = "veda-raster" cors_origins: str = "*" cachecontrol: str = "public, max-age=3600" debug: bool = False @@ -113,4 +113,4 @@ class Config: """model config""" env_file = ".env" - env_prefix = "DELTA_RASTER_" + env_prefix = "VEDA_RASTER_" diff --git a/raster_api/runtime/src/dependencies.py b/raster_api/runtime/src/dependencies.py index 1ef8f60a..c4914138 100644 --- a/raster_api/runtime/src/dependencies.py +++ b/raster_api/runtime/src/dependencies.py @@ -1,4 +1,4 @@ -"""delta.raster.dependencies.""" +"""veda.raster.dependencies.""" import json from base64 import b64decode diff --git a/setup.py b/setup.py index 301d1db4..264d779a 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -"""Setup delta-backend.""" +"""Setup veda-backend.""" from setuptools import find_packages, setup @@ -26,7 +26,7 @@ setup( - name="delta-backend", + name="veda-backend", version="0.6.2", description="", long_description=long_description, @@ -42,10 +42,10 @@ keywords="", author="Development Seed", author_email="info@developmentseed.org", - url="https://github.com/NASA-IMPACT/delta-backend", + url="https://github.com/NASA-IMPACT/veda-backend", license="MIT", packages=find_packages(exclude=["ez_setup", "examples", "tests"]), - package_data={"delta-backend": ["templates/*.html", "templates/*.xml"]}, + package_data={"veda-backend": ["templates/*.html", "templates/*.xml"]}, include_package_data=True, zip_safe=False, install_requires=[], diff --git a/stac_api/infrastructure/config.py b/stac_api/infrastructure/config.py index cacf797d..7bbd4dde 100644 --- a/stac_api/infrastructure/config.py +++ b/stac_api/infrastructure/config.py @@ -4,7 +4,7 @@ from pydantic import BaseSettings, Field -class deltaSTACSettings(BaseSettings): +class vedaSTACSettings(BaseSettings): """Application settings""" env: Dict = {} @@ -22,7 +22,7 @@ class Config: """model config""" env_file = ".env" - env_prefix = "DELTA_STAC_" + env_prefix = "VEDA_STAC_" -delta_stac_settings = deltaSTACSettings() +veda_stac_settings = vedaSTACSettings() diff --git a/stac_api/infrastructure/construct.py b/stac_api/infrastructure/construct.py index c6982025..4b9d45f5 100644 --- a/stac_api/infrastructure/construct.py +++ b/stac_api/infrastructure/construct.py @@ -13,7 +13,7 @@ ) from constructs import Construct -from .config import delta_stac_settings +from .config import veda_stac_settings class StacApiLambdaConstruct(Construct): @@ -47,12 +47,12 @@ def __init__( ), vpc=vpc, allow_public_subnet=True, - memory_size=delta_stac_settings.memory, - timeout=Duration.seconds(delta_stac_settings.timeout), + memory_size=veda_stac_settings.memory, + timeout=Duration.seconds(veda_stac_settings.timeout), environment={ "DB_MIN_CONN_SIZE": "0", "DB_MAX_CONN_SIZE": "1", - **{k.upper(): v for k, v in delta_stac_settings.env.items()}, + **{k.upper(): v for k, v in veda_stac_settings.env.items()}, }, log_retention=aws_logs.RetentionDays.ONE_WEEK, tracing=aws_lambda.Tracing.ACTIVE, @@ -67,7 +67,7 @@ def __init__( lambda_function.add_environment("TITILER_ENDPOINT", raster_api.raster_api.url) lambda_function.add_environment( - "DELTA_STAC_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn + "VEDA_STAC_PGSTAC_SECRET_ARN", database.pgstac.secret.secret_full_arn ) stac_api_integration = ( diff --git a/stac_api/runtime/README.md b/stac_api/runtime/README.md index 006e8b14..a12aa1cb 100644 --- a/stac_api/runtime/README.md +++ b/stac_api/runtime/README.md @@ -1 +1 @@ -## delta.stac_api +## veda.stac_api diff --git a/stac_api/runtime/setup.py b/stac_api/runtime/setup.py index 13b81c91..04399fc8 100644 --- a/stac_api/runtime/setup.py +++ b/stac_api/runtime/setup.py @@ -26,11 +26,11 @@ setup( - name="delta.stac_api", + name="veda.stac_api", description="", python_requires=">=3.7", packages=find_namespace_packages(exclude=["tests*"]), - package_data={"delta": ["stac/templates/*.html"]}, + package_data={"veda": ["stac/templates/*.html"]}, include_package_data=True, zip_safe=False, install_requires=inst_reqs, diff --git a/stac_api/runtime/src/__init__.py b/stac_api/runtime/src/__init__.py index a2c0fac2..b6473049 100644 --- a/stac_api/runtime/src/__init__.py +++ b/stac_api/runtime/src/__init__.py @@ -1 +1 @@ -"""delta.stac_api""" +"""veda.stac_api""" diff --git a/stac_api/runtime/src/config.py b/stac_api/runtime/src/config.py index 32ded30c..b8386993 100644 --- a/stac_api/runtime/src/config.py +++ b/stac_api/runtime/src/config.py @@ -49,7 +49,7 @@ def get_secret_dict(secret_name: str): class _ApiSettings(pydantic.BaseSettings): """API settings""" - name: str = "delta-backend-stac" + name: str = "veda-stac" cors_origins: str = "*" cachecontrol: str = "public, max-age=3600" debug: bool = False @@ -82,7 +82,7 @@ class Config: """model config""" env_file = ".env" - env_prefix = "DELTA_STAC_" + env_prefix = "VEDA_STAC_" @lru_cache() diff --git a/standalone_base_infrastructure/standalone_app.py b/standalone_base_infrastructure/standalone_app.py index b48f5128..46a33934 100644 --- a/standalone_base_infrastructure/standalone_app.py +++ b/standalone_base_infrastructure/standalone_app.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -""" CDK Configuration for the delta-backend stack.""" +""" CDK Configuration for base deployment infrastructure fo the veda-backend stack.""" from aws_cdk import App, Stack from constructs import Construct diff --git a/support_scripts/README.md b/support_scripts/README.md index 15e426ce..67c0ed70 100644 --- a/support_scripts/README.md +++ b/support_scripts/README.md @@ -9,8 +9,8 @@ This cli script will rotate the postgres connection password in a specified AWS ### Required input parameters To use the script, the following AWS resource names are required -- **`--admin`** the AWS secret containing the *admin* postgres connection info. This role will be used to update a pgstac user role. This secret will have a description `Generated by the CDK for stack: delta-backend-` -- **`--pgstac`** the aws secret containing the pgstac user role to be updated. This secret will have a description `Pgstac database bootsrapped by delta-backend- stack` +- **`--admin`** the AWS secret containing the *admin* postgres connection info. This role will be used to update a pgstac user role. This secret will have a description `Generated by the CDK for stack: veda-backend-` +- **`--pgstac`** the aws secret containing the pgstac user role to be updated. This secret will have a description `Pgstac database bootsrapped by veda-backend- stack` - **`--stac`** and **`--raster`** the resource names of the STAC and Raster lambdas that are using the pgstac connection secret. ### Optional dry run diff --git a/support_scripts/rotate_pgstac_pasword.py b/support_scripts/rotate_pgstac_pasword.py index a2378500..517f6cd2 100644 --- a/support_scripts/rotate_pgstac_pasword.py +++ b/support_scripts/rotate_pgstac_pasword.py @@ -194,7 +194,7 @@ def force_update_lambda( admin_dsn = get_dsn_string(admin_secret_dict) # Get pgstac user secret to update -print(f"Loading pgstac delta user credentials from secret={args.pgstac_secret_name}") +print(f"Loading pgstac veda user credentials from secret={args.pgstac_secret_name}") pgstac_secret_dict = get_secret_dict( args.pgstac_secret_name, profile_name=args.profile_name ) From 57618334e41aba55a51a3ca6b7fc625c7314bf61 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 12 Dec 2022 13:31:48 -0700 Subject: [PATCH 09/23] fix enable context in database construct to unblock deployment --- database/runtime/handler.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/database/runtime/handler.py b/database/runtime/handler.py index c285c733..b27fbcdd 100644 --- a/database/runtime/handler.py +++ b/database/runtime/handler.py @@ -122,17 +122,23 @@ def create_permissions(cursor, db_name: str, username: str) -> None: "GRANT pgstac_read TO {username};" "GRANT pgstac_ingest TO {username};" "GRANT pgstac_admin TO {username};" - "ALTER ROLE {username} SET SEARCH_PATH to pgstac, public;" - "ALTER ROLE {username} SET pgstac.context TO 'auto';" - "ALTER ROLE {username} SET pgstac.context_estimated_count TO '100000';" - "ALTER ROLE {username} SET pgstac.context_estimated_cost TO '100000';" - "ALTER ROLE {username} SET pgstac.context_stats_ttl TO '1 day';" ).format( db_name=sql.Identifier(db_name), username=sql.Identifier(username), ) ) +def enable_context(cursor) -> None: + """Enable context extension for actual and estimated matches in item search and associated optimizations.""" + cursor.execute(sql.SQL( + "INSERT INTO pgstac.pgstac_settings (name, value) " + " VALUES " + " ('context', 'auto')," + " ('context_estimated_count', '100000')," + " ('context_estimated_cost', '100000')," + " ('context_stats_ttl', '1 day')" + " ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;" + )) def register_extensions(cursor) -> None: """Add PostGIS extension.""" @@ -351,6 +357,13 @@ def handler(event, context): username=user_params["username"], ) + with psycopg.connect(stac_db_conninfo, autocommit=True) as conn: + with conn.cursor() as cur: + print("Enabling and configuring item search context in pgstac_settings...") + enable_context( + cursor=cur, + ) + print("Adding mosaic index...") with psycopg.connect( stac_db_admin_dsn, From 53648c69c44a886f771c84e09a4ef89de19e963a Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 12 Dec 2022 13:34:07 -0700 Subject: [PATCH 10/23] lint --- database/runtime/handler.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/database/runtime/handler.py b/database/runtime/handler.py index b27fbcdd..52ea7c93 100644 --- a/database/runtime/handler.py +++ b/database/runtime/handler.py @@ -128,17 +128,21 @@ def create_permissions(cursor, db_name: str, username: str) -> None: ) ) + def enable_context(cursor) -> None: """Enable context extension for actual and estimated matches in item search and associated optimizations.""" - cursor.execute(sql.SQL( - "INSERT INTO pgstac.pgstac_settings (name, value) " - " VALUES " - " ('context', 'auto')," - " ('context_estimated_count', '100000')," - " ('context_estimated_cost', '100000')," - " ('context_stats_ttl', '1 day')" - " ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;" - )) + cursor.execute( + sql.SQL( + "INSERT INTO pgstac.pgstac_settings (name, value) " + " VALUES " + " ('context', 'auto')," + " ('context_estimated_count', '100000')," + " ('context_estimated_cost', '100000')," + " ('context_stats_ttl', '1 day')" + " ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;" + ) + ) + def register_extensions(cursor) -> None: """Add PostGIS extension.""" @@ -359,7 +363,9 @@ def handler(event, context): with psycopg.connect(stac_db_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print("Enabling and configuring item search context in pgstac_settings...") + print( + "Enabling and configuring item search context in pgstac_settings..." + ) enable_context( cursor=cur, ) From e97d6460adb6af68592719413b273f18859bc7ad Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 12 Dec 2022 17:10:39 -0700 Subject: [PATCH 11/23] architecture diagram --- .readme/veda-backend.drawio.xml | 2 ++ .readme/veda-backend.svg | 1 + README.md | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 .readme/veda-backend.drawio.xml create mode 100644 .readme/veda-backend.svg diff --git a/.readme/veda-backend.drawio.xml b/.readme/veda-backend.drawio.xml new file mode 100644 index 00000000..7baedbd9 --- /dev/null +++ b/.readme/veda-backend.drawio.xml @@ -0,0 +1,2 @@ + +7Vxbc9u6Ef41nmkfxCF4lR4l2e4507SjE2fapi8ZiIQpxBTBkpAsnV9fgAQkEoBuFiXFieMkJhYgiMvutxcseeeO56u/FTCf/YPEKL1z7Hh1597fOQ7wgoD94pR1TekDuyYkBY5Foy3hCf+JBFE2W+AYla2GlJCU4rxNjEiWoYi2aLAoyGu72TNJ20/NYYI0wlMEU536bxzTmZiFb2/pvyGczOSTgS1qpjB6SQqyyMTz7hz3ofpTV8+h7Eu0L2cwJq8Nkvtw544LQmh9NV+NUcrXVi5bfd/jjtrNuAuU0WNu+O07jbM/+qO/py9/fE8+f/rvY4R6bt3LEqYLJKcRpKy/0TNh3bL1hFFdEfxvwUc6GpNFgVHBqv6JXrdkdpWI39XtU5Ug+ivpOm13yCt6ZcUXQ9YAuPmqvk3p+R5SyOqfKCmQ7JTNtu63/SxG1p6vNXVao3HIgqY4Q+MNm9libGOSkqJq44Ih/+F0nKYN+qM/HPm8fUkL8oJkTUYy1vkohuUMxaLHJSooZtz3CU5ROiElpphkrG5KKCXzRoNhihNeQUnOqFCUIrbZbPHd0YzOU1YGYpBCroAjy2Ji/JGwzOsZPeMVH8coJ5j38rBknZWiE8adOb9hvkq4nFvwtfSs6SJ6QXy1KjbfzKGepZQWsFlNPnS02smeYMP0DEwQmSNarFkTcUMIvPoWgSOu7Vh+TXndymUoSLOGSIZ9gQYCCZJN31tpYBdCIE4QDv+QcLyFmc+XqEny9GU4PofXKwaoVssfsb8MN8b1P5/VjjnFcnwD0UQLdSLQm7FfwPQElWiihToR6M14SY66TTTRQl8fsXo3MNwNlLvZX3d0EnQw8Yoxk4l7XLDGtfBnpOAMfiSssJqBzX9+RGwx4kiBSsbjEfo94uMZsWJ9pbSKSw1qOgAWXwWWMNRhxTbAin0hWAkGmkC2J825gSQkYztJ+A5VO/EdUboWWwEXlLT3SeMQ0Hce+Jh28mHKmWS0sWEaTYQVIxcfxZoBpSw9m0q1vXvm7AhDg8IiQfSQQaLvZYFSSPGyPZDOdyZ0ut8ZtlTF+j/i/qrwlRc42Ijy/apZe79uliaowGxuXCzvjUjwXvfZv+U+OwfN3tto9s5s5S/4C075g29rJu/WW7s03c+kz1I4n8bw2/MiE3q+e93m+k5bt/muptuCUNdtktY9grrXQ9Dwp0RQN3gPCCpH+R4CB9xb6g0nv3+g4S+Ghp6nW/qBIYAgaZ1LCbB1OOQgIddYBIg6REhrg4lfm3U7APJHx0JPBoQPYSGwd3DGddBQjtOAhjFe7gC0swHyKeJxBFzS8o6HfgM4r+S9+r+m1O2LNyHdhmyYwQUnNVlMUxx1MFJF7MoXRKOZFLGPWK+G4ouyelYHMLxBPQnDtm6U+oaAi3+pgIsMELfE8wqwvB9eH+Ecp3yJ2iKymxE3vAWuD8TOlYxSceuEc20jhOe3Q3ieGpurxyXuUrhlM4wz8N25uLW7RDHslRRGPZwlqKSkOAkDFW5+nWGKnvJ6UK9MyNuMacadJtAJEdhjsr4hvN1iYBW5Lh739cHAcKAEfKAjketeylA4eKTUDSPFkMJejnPE1Vz5wUlncpKrcJKEn5vxEbCvo8EajkX4U3kW/jvxLPyTPIumdNd+QDYt3+oOHGf3b+4cMr5al7i8wqMmbOWfSTE/pVdFWBo8bnYGTkGwfUjVzpVwDNZ3hugrKV5KK0rJIt7niBwyGqVj4KoIyQsTSJkfkFUUx+4oaSNQLP2Bb1Kx7jVPV2XHv9LpanBszNi57fEq+AW3Jjxya+qI+q22pm8b9Mw7d5HRClMRFB0EovxVWivsemu68MK6UVDPjH5QZ/tcYd7hbA9cy3Nt2wd94HpqWp4fKMBdT0dzvbVuA0/x4e3r+vDAHmgs/q/JWONyY26clhen5sRp+XDtXDgtz0zNMdPyy9qZbloynJoxp6XVtTPvtOQ3NUNOS6Pbm9vWEGCTf3goSBmxMmSdFqIPNUZZt0lTmJd4unXrULQoSsbXn1F96Fc7GqZYJhfTvDLMTCdRVe23ZR7td0t1C3B/uDZFz9zULNla4Cz5VJXu3X0ebitY3IE15igi5gLdT/VMkddNw+71isl/+Wn0iud7Db3Ssy1g+ycrl7ce3l1ZKflHKqVae3WulQJ7r1ZS3YljtZIfKlppcF2t1O9fRRzenELTsRxdl2f7x9re3k1zaeQwT4/xnBM4Puf0+IT0mRNDO2dOSrwjNCnIEseoKDsYsyKdH+fIVztH9qTFLrA5cHVz5qrnyAPvsDEjTc9dXLBzXcT7jXAqe7JPXS8lGGd+gSoAhnRQR9V63b3rYMq0VpbsIHsfPG/SON0oUzsFUGXjPxcFsiIyzxd1bybeP3l/m/u4h7v03W1GUkFoDXy/3+epbgMPmPayf7kzJ//KtkrjHYnLnDltLflBKC17YRgN+jeKEu1FnsPhxmPfv7h1xtxum+fofF9nR77v+ZnEes9vsETK2ZTAImYN/lKdyS+w0dCq6hj4P+Pkr28zV25wfi8R9Zyje2krnXl0fxKWym76FvBCewukiubUtOZgYPXtBuwacPdiVod/hAqVKgzPq68dHN7hgzq36mkorUCjSSgedj+jlH+rYcin6TxGcWZbuGJotpUF16PcpK4M80dOZ0b5I2LPr657jpevekuS9vgqOR7vgCleWPaA07fyLDHheit1RLViD5negnWrwY+an3BoVZAiRo3Wl+NE3/Icv8V9AGjsBwLLdTfMF3gG9tvR5AJWwBERPcmObHUohulnbtdmCa81xosb3Cr5ryoUYnpVvmMdT+WXFQfvOqVnu0lxlowrS5Dr9l4dVjs9/6ggFAqpGFwWjezACvknR5S92/ADQysdkoDdtxqIFHiBgSn6VnipdxsC0xtAnTnL2nYx+GU/e6z4ozzmOY7j1BTK3+DePqf50FmHqtEqH7k+DmAld5fHnBckXkRUvD6TG88sYI6/JZCiV7juxrl2lJw2V6bYNn1FT2cpSeuen0wW4Qc/XYWfLvJBF5XDLvg2Fituv6NUh+a3H6tyH/4P \ No newline at end of file diff --git a/.readme/veda-backend.svg b/.readme/veda-backend.svg new file mode 100644 index 00000000..6966098d --- /dev/null +++ b/.readme/veda-backend.svg @@ -0,0 +1 @@ +
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file diff --git a/README.md b/README.md index 586d2087..31665e0e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee - [titiler](https://github.com/developmentseed/titiler) - [titiler-pgstac](https://github.com/stac-utils/titiler-pgstac) +![architecture diagram](.readme/veda-backend.svg) + ## Deployment This repo includes CDK scripts to deploy a PgStac AWS RDS database and other resources to support APIs maintained by the VEDA backend development team. From 520edee8f79bed8abcd9e36b516450fadef5f76e Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 12 Dec 2022 17:50:35 -0700 Subject: [PATCH 12/23] diagram formatting --- .readme/veda-backend.drawio.svg | 4 ++++ .readme/veda-backend.drawio.xml | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .readme/veda-backend.drawio.svg diff --git a/.readme/veda-backend.drawio.svg b/.readme/veda-backend.drawio.svg new file mode 100644 index 00000000..2ff79cf9 --- /dev/null +++ b/.readme/veda-backend.drawio.svg @@ -0,0 +1,4 @@ + + + +
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file diff --git a/.readme/veda-backend.drawio.xml b/.readme/veda-backend.drawio.xml index 7baedbd9..db85d284 100644 --- a/.readme/veda-backend.drawio.xml +++ b/.readme/veda-backend.drawio.xml @@ -1,2 +1,2 @@ -7Vxbc9u6Ef41nmkfxCF4lR4l2e4507SjE2fapi8ZiIQpxBTBkpAsnV9fgAQkEoBuFiXFieMkJhYgiMvutxcseeeO56u/FTCf/YPEKL1z7Hh1597fOQ7wgoD94pR1TekDuyYkBY5Foy3hCf+JBFE2W+AYla2GlJCU4rxNjEiWoYi2aLAoyGu72TNJ20/NYYI0wlMEU536bxzTmZiFb2/pvyGczOSTgS1qpjB6SQqyyMTz7hz3ofpTV8+h7Eu0L2cwJq8Nkvtw544LQmh9NV+NUcrXVi5bfd/jjtrNuAuU0WNu+O07jbM/+qO/py9/fE8+f/rvY4R6bt3LEqYLJKcRpKy/0TNh3bL1hFFdEfxvwUc6GpNFgVHBqv6JXrdkdpWI39XtU5Ug+ivpOm13yCt6ZcUXQ9YAuPmqvk3p+R5SyOqfKCmQ7JTNtu63/SxG1p6vNXVao3HIgqY4Q+MNm9libGOSkqJq44Ih/+F0nKYN+qM/HPm8fUkL8oJkTUYy1vkohuUMxaLHJSooZtz3CU5ROiElpphkrG5KKCXzRoNhihNeQUnOqFCUIrbZbPHd0YzOU1YGYpBCroAjy2Ji/JGwzOsZPeMVH8coJ5j38rBknZWiE8adOb9hvkq4nFvwtfSs6SJ6QXy1KjbfzKGepZQWsFlNPnS02smeYMP0DEwQmSNarFkTcUMIvPoWgSOu7Vh+TXndymUoSLOGSIZ9gQYCCZJN31tpYBdCIE4QDv+QcLyFmc+XqEny9GU4PofXKwaoVssfsb8MN8b1P5/VjjnFcnwD0UQLdSLQm7FfwPQElWiihToR6M14SY66TTTRQl8fsXo3MNwNlLvZX3d0EnQw8Yoxk4l7XLDGtfBnpOAMfiSssJqBzX9+RGwx4kiBSsbjEfo94uMZsWJ9pbSKSw1qOgAWXwWWMNRhxTbAin0hWAkGmkC2J825gSQkYztJ+A5VO/EdUboWWwEXlLT3SeMQ0Hce+Jh28mHKmWS0sWEaTYQVIxcfxZoBpSw9m0q1vXvm7AhDg8IiQfSQQaLvZYFSSPGyPZDOdyZ0ut8ZtlTF+j/i/qrwlRc42Ijy/apZe79uliaowGxuXCzvjUjwXvfZv+U+OwfN3tto9s5s5S/4C075g29rJu/WW7s03c+kz1I4n8bw2/MiE3q+e93m+k5bt/muptuCUNdtktY9grrXQ9Dwp0RQN3gPCCpH+R4CB9xb6g0nv3+g4S+Ghp6nW/qBIYAgaZ1LCbB1OOQgIddYBIg6REhrg4lfm3U7APJHx0JPBoQPYSGwd3DGddBQjtOAhjFe7gC0swHyKeJxBFzS8o6HfgM4r+S9+r+m1O2LNyHdhmyYwQUnNVlMUxx1MFJF7MoXRKOZFLGPWK+G4ouyelYHMLxBPQnDtm6U+oaAi3+pgIsMELfE8wqwvB9eH+Ecp3yJ2iKymxE3vAWuD8TOlYxSceuEc20jhOe3Q3ieGpurxyXuUrhlM4wz8N25uLW7RDHslRRGPZwlqKSkOAkDFW5+nWGKnvJ6UK9MyNuMacadJtAJEdhjsr4hvN1iYBW5Lh739cHAcKAEfKAjketeylA4eKTUDSPFkMJejnPE1Vz5wUlncpKrcJKEn5vxEbCvo8EajkX4U3kW/jvxLPyTPIumdNd+QDYt3+oOHGf3b+4cMr5al7i8wqMmbOWfSTE/pVdFWBo8bnYGTkGwfUjVzpVwDNZ3hugrKV5KK0rJIt7niBwyGqVj4KoIyQsTSJkfkFUUx+4oaSNQLP2Bb1Kx7jVPV2XHv9LpanBszNi57fEq+AW3Jjxya+qI+q22pm8b9Mw7d5HRClMRFB0EovxVWivsemu68MK6UVDPjH5QZ/tcYd7hbA9cy3Nt2wd94HpqWp4fKMBdT0dzvbVuA0/x4e3r+vDAHmgs/q/JWONyY26clhen5sRp+XDtXDgtz0zNMdPyy9qZbloynJoxp6XVtTPvtOQ3NUNOS6Pbm9vWEGCTf3goSBmxMmSdFqIPNUZZt0lTmJd4unXrULQoSsbXn1F96Fc7GqZYJhfTvDLMTCdRVe23ZR7td0t1C3B/uDZFz9zULNla4Cz5VJXu3X0ebitY3IE15igi5gLdT/VMkddNw+71isl/+Wn0iud7Db3Ssy1g+ycrl7ce3l1ZKflHKqVae3WulQJ7r1ZS3YljtZIfKlppcF2t1O9fRRzenELTsRxdl2f7x9re3k1zaeQwT4/xnBM4Puf0+IT0mRNDO2dOSrwjNCnIEseoKDsYsyKdH+fIVztH9qTFLrA5cHVz5qrnyAPvsDEjTc9dXLBzXcT7jXAqe7JPXS8lGGd+gSoAhnRQR9V63b3rYMq0VpbsIHsfPG/SON0oUzsFUGXjPxcFsiIyzxd1bybeP3l/m/u4h7v03W1GUkFoDXy/3+epbgMPmPayf7kzJ//KtkrjHYnLnDltLflBKC17YRgN+jeKEu1FnsPhxmPfv7h1xtxum+fofF9nR77v+ZnEes9vsETK2ZTAImYN/lKdyS+w0dCq6hj4P+Pkr28zV25wfi8R9Zyje2krnXl0fxKWym76FvBCewukiubUtOZgYPXtBuwacPdiVod/hAqVKgzPq68dHN7hgzq36mkorUCjSSgedj+jlH+rYcin6TxGcWZbuGJotpUF16PcpK4M80dOZ0b5I2LPr657jpevekuS9vgqOR7vgCleWPaA07fyLDHheit1RLViD5negnWrwY+an3BoVZAiRo3Wl+NE3/Icv8V9AGjsBwLLdTfMF3gG9tvR5AJWwBERPcmObHUohulnbtdmCa81xosb3Cr5ryoUYnpVvmMdT+WXFQfvOqVnu0lxlowrS5Dr9l4dVjs9/6ggFAqpGFwWjezACvknR5S92/ADQysdkoDdtxqIFHiBgSn6VnipdxsC0xtAnTnL2nYx+GU/e6z4ozzmOY7j1BTK3+DePqf50FmHqtEqH7k+DmAld5fHnBckXkRUvD6TG88sYI6/JZCiV7juxrl2lJw2V6bYNn1FT2cpSeuen0wW4Qc/XYWfLvJBF5XDLvg2Fituv6NUh+a3H6tyH/4P \ No newline at end of file +7VzbcuO4Ef0aVyUPYhG8So+SbG+2apNSraeSzNMURMIS1hTBgNDF+/UBSFAiAVAXk5LGicczY6EBgiBw+nSj0dSDO13tfqEwW/6dxCh5cOx49+A+PjgOcF2X/xKS91ISukEpWFAcy0YHwQv+E0mhLaVrHKO80ZARkjCcNYURSVMUsYYMUkq2zWavJGneNYMLpAleIpjo0n/hmC1L6dC3D/K/IbxYVncGtqyZw+htQck6lfd7cNyn4k9ZvYJVX7J9voQx2dZE7tODO6WEsPLTajdFiZjbatrK655bavfjpihl51zglBdsYLJG1YiDhF86eSW8Bz51MCorgv+sxaAmU7KmGFFe9Q+0PYj5p4X8XVw+VwWyv5y9J80ORcUgLyAw5g2Am+3Ky5SeHyGDvP6FEYqqTvmDlf0278XF2v21pk5jNA5ZswSnaLpHlC3HNiUJoUUbF4zFj5DjJKnJn/3xxBftc0bJG6pqUpLyzicxzJcolj1uEGWYA+03OEfJjOSYYZLyujlhjKxqDcYJXogKRjIuhbIU8XXlk+9OlmyV8DKQg5QqBJyqLB9M3BLmWflEr3gnxjHJCBa9PG14Z7nshAMxExesdguh0hbc5p41X0dvSMxWgej9M5RPWSkG2M+mGDratSIR7PHNeQORFWL0nTeRF4TAKy+RlOHajuWXku1BBUMpWta0LxxKxZdKv9j3fQA+/yCxb9YDt1UPYrwRfcv5P8AyQa+sHf4d0N5d5WaLl2/j6UXKsBcXj9uiIgVuikn2J/wvZ5Zp+c/ntVMhsRzfIDTJQl0I9Gb8FzDdQRWaZKEuBHozUapG3RSaZKGvj1i9GhiuBsrV/K87uYhxuFbGmKvSI6a8cckZKaFCL85kI14zssVPz5RUKEI3QjKSD0U5x32Efo3EaCa8WH5SWsW5xk89sJGvslEY6lxkG7jI7s5FnqZ4zccTq04WJOUrRsRKFHP+B2LsXU46XDPSXBENCWDoPIm7t+ItEWCY7L2ZWhPpz1TTjGLFlSpXrRRJr49BukCs4XGcsRQUJZDhTbP3LhPr9z+x/Bno+7/l9UXhuygITpDlx1299vG9XpohivlTCIP+aFTY+y2Te79lCk75pPexqr05st/wN5yIG9/Xh223Dm325KdzZDvYjQSu5jH88bpOpTXt34a4vtO0Ib6r2ZAg1G1IJeuiQ+HtqC78VFQ3/Kmobvh5tt9iSzEYz379oq3/M9ryPN31DQzb8ErWRR9GGjaEMleTKeMpPTKZteeu7/W6FiK7F2dVUdGGGz28H2tV42mLlhiZpzOTvURi/4tzlj+IoGYAV4ViFv+XkrI9/RAlmQMgV3+o2Xqe4Kj/UE3+hli0rFTkK7Sp0e06L+7VA18C223ypa27eb4hVOD3ECoAwKCJN2DQ40z4DFc4EbPR1IZ2zO1hBPrjTOcmjp68dCZgV4se+c3okaeGhcpRyauU5d4P4zwEXP8EZ4NiOMgZjAY4XaCcEXoRXylw3C4xQy9ZOagtV8gmsswcUSclieEjfuAHQqgNBKosc/Xoog9GhrMO4AOdNaqT1U6s0X7a0StmYsjgIMMZEtYn/wJNR9C4CmgqUrkJZPSg9JU98/BzuOb+T+aa+5e45nU9LB3pdJ5/1J8+z3HeXznmuHjPcX6DW834JL8SurqkVwXsNYyavelLuOYYpzTP1h2D+5oitiX0LbeihKzjY578KVes8qxdlctEYQYZd6TTQuLYPR3yB4qrPPJNds+90sEaCLR1/bQna1WopE48wR155woB53tNrWsIt9yT0k1B4k++x0M7zGQAbhTI8vfKsPPPBysvCu+1gnqOcL3dotP/krfsFkeu5bm27YMhcD01D8oPFOYrx63tHbVuA0/ZhNrX24SONIj+czbVUGrMH9Jyh9S8IS1nqJkvpOXiqHk4Wg5OMxtISxhSs4q01KNmdpKWIKRmEWmpRkfzf2oKaNr0nAqIRbwMeadU9qHGw8o2SQKzHM8PexUUrWnOEfw7Kk+CSp/aFDcTapYVPozpeKKo/bHJouN7Ld1ZOivFKOdzgdPFb0Xp0T22bWsEJntwXBxFmVygb748U5Rv37CLCXBMAff/GRPg+V7NBAxsC9j+xXbgo2c6PdkPV7cf4GYGJLCPGhDVdT7XgPihYkBGVzMg+zm9VXjh4bIUhp4VoactgMFpAXdMZqjGc3nsoUvoscux4AUJDBeGHDo+lHzXYUbJBseI5j2MWVGvrwPCmx0QevaowaOBq/sO1zogNL7YoEChcunaFrx1CuTbV3Be9WRfOjVKPMj8zkcADElyjmqMPjQ73unZOQnak+cQGn6NmtKqVio4/1xTZEVkla3L3kyIvngp60tWYkZfs3qIDoTWyPeHQ5EtNPKAaYWGPRw7GFK2r3zs4Fz52OHg9o7Cyg2WTshoeMPoh77ginPh6c5FGQW7hSfRQwq405LH2D1DUu/5A/Y9X84JpDFv8JfirHSNje5LUcd59hUv/voxJ+AO56oVo3U5Uq08kI5Hqm1cVl3hWJ4T2gcic5qWWrNFo5E1tGu0Z+C9Xsx2eNowVYYBr4qXmU+v20lLVvQ0rjwmo/skb/a4ZEy8ij0WT+Q8R3FqW7iAKV8gKqyTcD8LJ/ZZyLkD+4z4/YvPA8fLdoMNSQZiQgTRPAtzBvMBcIZWli5MxNo4qFc9vlNuqgRkMfhJ/Q3tRgWhMaq17gVfgOPLb2CqStmqW9PAct09pALPAKqWJp0wdsa5QoUx/sgMw+R34e2lC1FrjE7WIFiBqihQ+SBFelcZvRMfC1i2HZ/yJWI4XUwLp0lYzEEZ2bk8hYMSBiXUR70Rhx1YofhGAGVB9os8tICeBgjsoVUjj8ALDCs9tMIeMqkdPSSvr+6H94DaInBS5D9H3NizNoIrHMeJKRy8p6hje8FT8XLVpBRbvzKkzEtu20YwoyReR0zm5WfGuDfM8I8FZGgL3/vZMzpKso9bZRTW90Wejp5K1uld+DOizV/QuQ50rvLVCiqYrvhGBy8evrykDAQfviHGffov \ No newline at end of file diff --git a/README.md b/README.md index 31665e0e..a17a20d9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee - [titiler](https://github.com/developmentseed/titiler) - [titiler-pgstac](https://github.com/stac-utils/titiler-pgstac) -![architecture diagram](.readme/veda-backend.svg) +![architecture diagram](.readme/veda-backend.drawio.svgsvg) ## Deployment From 6683d7700460aa4190f35f391ca7bf51374e99e3 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Mon, 12 Dec 2022 17:52:09 -0700 Subject: [PATCH 13/23] diagram formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a17a20d9..3e025185 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee - [titiler](https://github.com/developmentseed/titiler) - [titiler-pgstac](https://github.com/stac-utils/titiler-pgstac) -![architecture diagram](.readme/veda-backend.drawio.svgsvg) +![architecture diagram](.readme/veda-backend.drawio.svg) ## Deployment From 37e778c726d7cb19469daff147179b19f39e0663 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 10:17:09 -0700 Subject: [PATCH 14/23] move advanced configuration to separate page in docs and provide tool for syncing aws secrets to local file --- README.md | 35 ++++++++++++++++------------------ docs/advanced_configuration.md | 24 +++++++++++++++++++++++ scripts/sync-env-local.sh | 5 +++++ 3 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 docs/advanced_configuration.md create mode 100755 scripts/sync-env-local.sh diff --git a/README.md b/README.md index 3e025185..9305a3d3 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,20 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee - [titiler](https://github.com/developmentseed/titiler) - [titiler-pgstac](https://github.com/stac-utils/titiler-pgstac) +## VEDA ecosystem ![architecture diagram](.readme/veda-backend.drawio.svg) +Veda backend is is the central index of the VEDA ecosystem. This project provides the infrastructure for a PgSTAC database, STAC API, and TiTiler. This infrastructure is used to discover, access, and visualize the Analysis Ready Cloud Optimized (ARCO) assets of the VEDA Data Store. + +| Name | Explanation | +| --- | --- | +| [**veda-config**]() | Configuration for viewing VEDA assets in dashboard UI | +| [**veda-ui**]() | Dashboard UI for viewing and analysing VEDA assets | +| [**veda-stac-ingestor**]() | Load records to PgSTAC | +| [**veda-data-pipelines**]() | Cloud optimize data assets and submit records for publication to veda-stac-ingestor | ## Deployment -This repo includes CDK scripts to deploy a PgStac AWS RDS database and other resources to support APIs maintained by the VEDA backend development team. +This repo includes CDK scripts to deploy a PgSTAC AWS RDS database and other resources to support APIs maintained by the VEDA backend development team. ### Tooling & supporting documentation @@ -22,38 +31,26 @@ This repo includes CDK scripts to deploy a PgStac AWS RDS database and other res ### Enviroment variables -An [.example.env](.example.env) template is supplied for for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `-backend/-env`, for example `veda-backend/dev-env`. +An [.example.env](.example.env) template is supplied for for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `--env`, for example `veda-backend-dev-env`. ### Fetch environment variables using AWS CLI -To retrieve the variables for a stage that has been previously deployed, the secrets manager can be used to quickly populate an .env file. -> Note: The environment variables stored as AWS secrets are manually maintained and should be reviewed before using. +To retrieve the variables for a stage that has been previously deployed, the secrets manager can be used to quickly populate an .env file using [scripts/sync-env-local.sh](scripts/sync-env-local.sh). +> **Warning** The environment variables stored as AWS secrets are manually maintained and should be reviewed before deploying updates to existing stacks. ``` -export AWS_SECRET_ID=-backend/-env - -aws secretsmanager get-secret-value --secret-id ${AWS_SECRET_ID} --query SecretString --output text | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env +# sync-env-local.sh +./scripts/sync-env-local.sh veda-backend-dev-env ``` | Name | Explanation | | --- | --- | | `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda` | | `STAGE` | **REQUIRED** Deployment stage used to name stack and resources, i.e. `dev`, `staging`, `prod` | -| `VPC_ID` | Optional resource identifier of VPC, if none a new VPC with public and private subnets will be provisioned. | -| `PERMISSIONS_BOUNDARY_POLICY_NAME` | Optional name of IAM policy to define stack permissions boundary | -| `CDK_DEFAULT_ACCOUNT` | When deploying from a local machine the AWS account id is required to deploy to an exiting VPC | -| `CDK_DEFAULT_REGION` | When deploying from a local machine the AWS region id is required to deploy to an exiting VPC | | `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.5 | | `VEDA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom veda-backend schema, i.e. 0.1.1 | | `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot | -| `VEDA_DB_PRIVATE_SUBNETS` | Optional boolean to deploy database to private subnet | -| `VEDA_DOMAIN_HOSTED_ZONE_ID` | Optional Route53 zone identifier if using a custom domain name | -| `VEDA_DOMAIN_HOSTED_ZONE_NAME` | Optional custom domain name, i.e. veda-backend.xyz | -| `VEDA_DOMAIN_ALT_HOSTED_ZONE_ID` | Optional second Route53 zone identifier if using a custom domain name | -| `VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME` | Optional second custom domain name, i.e. alt-veda-backend.xyz | -| `VEDA_DOMAIN_API_PREFIX` | Optional domain prefix override supports using a custom prefix instead of the STAGE variabe (an alternate version of the stack can be deployed with a unique STAGE=altprod and after testing prod API traffic can be cut over to the alternate version of the stack by setting the prefix to prod) | -| `VEDA_RASTER_ENABLE_MOSAIC_SEARCH` | Optional deploy the raster API with the mosaic/list endpoint TRUE/FALSE | -| `VEDA_RASTER_DATA_ACCESS_ROLE_ARN` | Optional arn of IAM Role to be assumed by raster-api for S3 bucket data access, if not provided default role for the lambda construct is used | +> **Note:** See [Advanced Configuration](docs/advanced_configuration.md) for details about custom configuration options. ### Deploying to the cloud diff --git a/docs/advanced_configuration.md b/docs/advanced_configuration.md new file mode 100644 index 00000000..9ea34125 --- /dev/null +++ b/docs/advanced_configuration.md @@ -0,0 +1,24 @@ +# Advanced Configuration +The constructs and applications in this project are configured using [pydantic](https://docs.pydantic.dev/usage/settings/). The settings are defined in config.py files stored alongside the associated construct or application--for example the settings for the RDS PostgreSQL construct are defined in [database/infrastructure/config.py](../database/infrastructure/config.py); the settings for the TiTiler API are defined in [raster_api/runtime/src/config.py](../raster_api/runtime/src/config.py). For custom configuration, use environment variables to override the pydantic defaults. + +## Selected configuration variables +Environment variables for specific VEDA backend components are prefixed, for example database configuration variables are prefixed `VEDA_DB`. See the config.py file in each construct for the appropriate prefix. +| Name | Explanation | +| --- | --- | +| `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda` | +| `STAGE` | **REQUIRED** Deployment stage used to name stack and resources, i.e. `dev`, `staging`, `prod` | +| `VPC_ID` | Optional resource identifier of VPC, if none a new VPC with public and private subnets will be provisioned. | +| `PERMISSIONS_BOUNDARY_POLICY_NAME` | Optional name of IAM policy to define stack permissions boundary | +| `CDK_DEFAULT_ACCOUNT` | When deploying from a local machine the AWS account id is required to deploy to an exiting VPC | +| `CDK_DEFAULT_REGION` | When deploying from a local machine the AWS region id is required to deploy to an exiting VPC | +| `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.5 | +| `VEDA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom veda-backend schema, i.e. 0.1.1 | +| `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot | +| `VEDA_DB_PRIVATE_SUBNETS` | Optional boolean to deploy database to private subnet | +| `VEDA_DOMAIN_HOSTED_ZONE_ID` | Optional Route53 zone identifier if using a custom domain name | +| `VEDA_DOMAIN_HOSTED_ZONE_NAME` | Optional custom domain name, i.e. veda-backend.xyz | +| `VEDA_DOMAIN_ALT_HOSTED_ZONE_ID` | Optional second Route53 zone identifier if using a custom domain name | +| `VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME` | Optional second custom domain name, i.e. alt-veda-backend.xyz | +| `VEDA_DOMAIN_API_PREFIX` | Optional domain prefix override supports using a custom prefix instead of the STAGE variabe (an alternate version of the stack can be deployed with a unique STAGE=altprod and after testing prod API traffic can be cut over to the alternate version of the stack by setting the prefix to prod) | +| `VEDA_RASTER_ENABLE_MOSAIC_SEARCH` | Optional deploy the raster API with the mosaic/list endpoint TRUE/FALSE | +| `VEDA_RASTER_DATA_ACCESS_ROLE_ARN` | Optional arn of IAM Role to be assumed by raster-api for S3 bucket data access, if not provided default role for the lambda construct is used | \ No newline at end of file diff --git a/scripts/sync-env-local.sh b/scripts/sync-env-local.sh new file mode 100755 index 00000000..98c4ecaf --- /dev/null +++ b/scripts/sync-env-local.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Use this script to load environment variables for a deployment from AWS Secrets + +echo Loading environment secrets from $1 +aws secretsmanager get-secret-value --secret-id $1 --query SecretString --output text | jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' > .env From 8da727afd656b0aa233bc84337189606fab7ae3d Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 10:34:26 -0700 Subject: [PATCH 15/23] move docs for deploying to managed environments to separate page --- README.md | 90 +++------------------- docs/deploying_to_existing_environments.md | 75 ++++++++++++++++++ 2 files changed, 85 insertions(+), 80 deletions(-) create mode 100644 docs/deploying_to_existing_environments.md diff --git a/README.md b/README.md index 9305a3d3..ee77eb9b 100644 --- a/README.md +++ b/README.md @@ -32,25 +32,24 @@ This repo includes CDK scripts to deploy a PgSTAC AWS RDS database and other res ### Enviroment variables An [.example.env](.example.env) template is supplied for for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `--env`, for example `veda-backend-dev-env`. +> **Warning** The environment variables stored as AWS secrets are manually maintained and should be reviewed before deploying updates to existing stacks. ### Fetch environment variables using AWS CLI -To retrieve the variables for a stage that has been previously deployed, the secrets manager can be used to quickly populate an .env file using [scripts/sync-env-local.sh](scripts/sync-env-local.sh). -> **Warning** The environment variables stored as AWS secrets are manually maintained and should be reviewed before deploying updates to existing stacks. +To retrieve the variables for a stage that has been previously deployed, the secrets manager can be used to quickly populate an .env file with [scripts/sync-env-local.sh](scripts/sync-env-local.sh). ``` -# sync-env-local.sh -./scripts/sync-env-local.sh veda-backend-dev-env +./scripts/sync-env-local.sh ``` - +### Basic environment variables | Name | Explanation | | --- | --- | -| `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda` | +| `APP_NAME` | Optional app name used to name stack and resources, defaults to `veda-backend` | | `STAGE` | **REQUIRED** Deployment stage used to name stack and resources, i.e. `dev`, `staging`, `prod` | | `VEDA_DB_PGSTAC_VERSION` | **REQUIRED** version of PgStac database, i.e. 0.5 | | `VEDA_DB_SCHEMA_VERSION` | **REQUIRED** The version of the custom veda-backend schema, i.e. 0.1.1 | | `VEDA_DB_SNAPSHOT_ID` | **Once used always REQUIRED** Optional RDS snapshot identifier to initialize RDS from a snapshot | -> **Note:** See [Advanced Configuration](docs/advanced_configuration.md) for details about custom configuration options. +> **Note** See [Advanced Configuration](docs/advanced_configuration.md) for details about custom configuration options. ### Deploying to the cloud @@ -86,79 +85,10 @@ If this is a development stack that is safe to delete, you can delete the stack 2. Detach the Internet Gateway (IGW) from the VPC and delete it. 3. If this stack created a new VPC, delete the VPC (this should delete a subnet and security group too). -## Deployment to MCP and/or an existing VPC - -### MCP access - - At this time, this project requires that anyone deploying to the Mission Cloud Platform (MCP) environments should have gone through a NASA credentialing process and then submitted and gotten approval for access to the VEDA project on MCP. - -### MCP and existing VPC endpoint requirements - -VPC interface endpoints must be configured to allow app components to connect to other services within the VPC and gateway endpoints need to be configured for external connections. - -| service-name | vpc-endpoint-type | comments | -| -- | -- | -- | -| secretsmanager | Interface | security group configuration recommendations below | -| logs | Interface | cloudwatch-logs, security group configuration recommendations below | -| s3 | Gateway | | -| dynamodb | Gateway | required if using DynamoDB streams | - -### Create `Interface` VPC endpoints -Create a security group for the VPC Interface endpoints ([AWS docs](https://docs.aws.amazon.com/cli/latest/userguide/cli-services-ec2-sg.html)) -```bash -aws ec2 create-security-group --vpc-id --group-name vpc-interface-endpoints --description "security group for vpc interface endpoints" -``` -Configure ingress policy for this SG (the egress is configured for 'free' when a new SG is created) -```bash -# Lookup CidrBlock -aws ec2 describe-vpcs --vpc-ids $VPC_ID | jq -r '.Vpcs[].CidrBlock' - -aws ec2 authorize-security-group-ingress --group-id --protocol tcp --port 443 --cidr -``` -Create VPC Interface endpoints -``` -# Choose private subnets (example subnet was generated by aws-cdk) -aws ec2 describe-subnets --filters Name=vpc-id,Values= Name=tag:aws-cdk:subnet-name,Values=private | jq -r '.Subnets[].SubnetId' - -# Secrets manager endpoint -aws ec2 create-vpc-endpoint \ ---vpc-id \ ---vpc-endpoint-type Interface \ ---service-name com.amazonaws.us-west-2.secretsmanager \ ---subnet-ids \ ---security-group-ids - -# Cloudwatch logs endpoint uses same security group cfg -aws ec2 create-vpc-endpoint \ ---vpc-id \ ---vpc-endpoint-type Interface \ ---service-name com.amazonaws.us-west-2.logs \ ---subnet-ids \ ---security-group-ids -``` - -### Create `Gateway` VPC endpoints -``` -# List route tables for VPC -aws ec2 describe-route-tables --filters Name=vpc-id,Values= | jq -r '.RouteTables[].RouteTableId' - -# Create Gateway endpoint for S3 -aws ec2 create-vpc-endpoint \ ---vpc-id \ ---vpc-endpoint-type Gateway \ ---service-name com.amazonaws.us-west-2.s3 \ ---route-table-ids - -# Optional create Gateway endpoint for DynamoDB -aws ec2 create-vpc-endpoint \ ---vpc-id \ ---vpc-endpoint-type Gateway \ ---service-name com.amazonaws.us-west-2.dynamodb \ ---route-table-ids -``` - -## [OPTIONAL] Deploy standalone base infrastructure -For convenience, [standalone base infrastructure](standalone_base_infrastructure/README.md#standalone-base-infrastructure) scripts are provided to deploy base infrastructure to simulate deployment in a controlled environment. +## Custom deployments +The default settings for this project generate a complete AWS environment including a VPC and gateways for the stack. See this guidance for adjusting the veda-backend stack for existing managed and/or shared AWS environments. +- [Deploy to an existing managed AWS environment](docs/deploying_to_existing_environments.md) +- [Creating a shared base VPC and AWS environment](docs/deploying_to_existing_environments.md#optional-deploy-standalone-base-infrastructure) ## Local Docker deployment diff --git a/docs/deploying_to_existing_environments.md b/docs/deploying_to_existing_environments.md new file mode 100644 index 00000000..2f167cb5 --- /dev/null +++ b/docs/deploying_to_existing_environments.md @@ -0,0 +1,75 @@ +# Deploying to Existing Environments + +## Deployment to MCP and/or an existing VPC + +### MCP access + + At this time, this project requires that anyone deploying to the Mission Cloud Platform (MCP) environments should have gone through a NASA credentialing process and then submitted and gotten approval for access to the VEDA project on MCP. + +### MCP and existing VPC endpoint requirements + +VPC interface endpoints must be configured to allow app components to connect to other services within the VPC and gateway endpoints need to be configured for external connections. + +| service-name | vpc-endpoint-type | comments | +| -- | -- | -- | +| secretsmanager | Interface | security group configuration recommendations below | +| logs | Interface | cloudwatch-logs, security group configuration recommendations below | +| s3 | Gateway | | +| dynamodb | Gateway | required if using DynamoDB streams | + +### Create `Interface` VPC endpoints +Create a security group for the VPC Interface endpoints ([AWS docs](https://docs.aws.amazon.com/cli/latest/userguide/cli-services-ec2-sg.html)) +```bash +aws ec2 create-security-group --vpc-id --group-name vpc-interface-endpoints --description "security group for vpc interface endpoints" +``` +Configure ingress policy for this SG (the egress is configured for 'free' when a new SG is created) +```bash +# Lookup CidrBlock +aws ec2 describe-vpcs --vpc-ids $VPC_ID | jq -r '.Vpcs[].CidrBlock' + +aws ec2 authorize-security-group-ingress --group-id --protocol tcp --port 443 --cidr +``` +Create VPC Interface endpoints +``` +# Choose private subnets (example subnet was generated by aws-cdk) +aws ec2 describe-subnets --filters Name=vpc-id,Values= Name=tag:aws-cdk:subnet-name,Values=private | jq -r '.Subnets[].SubnetId' + +# Secrets manager endpoint +aws ec2 create-vpc-endpoint \ +--vpc-id \ +--vpc-endpoint-type Interface \ +--service-name com.amazonaws.us-west-2.secretsmanager \ +--subnet-ids \ +--security-group-ids + +# Cloudwatch logs endpoint uses same security group cfg +aws ec2 create-vpc-endpoint \ +--vpc-id \ +--vpc-endpoint-type Interface \ +--service-name com.amazonaws.us-west-2.logs \ +--subnet-ids \ +--security-group-ids +``` + +### Create `Gateway` VPC endpoints +``` +# List route tables for VPC +aws ec2 describe-route-tables --filters Name=vpc-id,Values= | jq -r '.RouteTables[].RouteTableId' + +# Create Gateway endpoint for S3 +aws ec2 create-vpc-endpoint \ +--vpc-id \ +--vpc-endpoint-type Gateway \ +--service-name com.amazonaws.us-west-2.s3 \ +--route-table-ids + +# Optional create Gateway endpoint for DynamoDB +aws ec2 create-vpc-endpoint \ +--vpc-id \ +--vpc-endpoint-type Gateway \ +--service-name com.amazonaws.us-west-2.dynamodb \ +--route-table-ids +``` + +## [OPTIONAL] Deploy standalone base infrastructure +For convenience, [standalone base infrastructure](standalone_base_infrastructure/README.md#standalone-base-infrastructure) scripts are provided to deploy base infrastructure to simulate deployment in a controlled environment. From 04a2018d8337dada604bad05ea42948481e7af38 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 10:55:12 -0700 Subject: [PATCH 16/23] add veda ecosystem context --- .readme/veda-backend.svg | 1 - README.md | 25 +++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) delete mode 100644 .readme/veda-backend.svg diff --git a/.readme/veda-backend.svg b/.readme/veda-backend.svg deleted file mode 100644 index 6966098d..00000000 --- a/.readme/veda-backend.svg +++ /dev/null @@ -1 +0,0 @@ -
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file diff --git a/README.md b/README.md index ee77eb9b..2c52ad14 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,9 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee - [titiler](https://github.com/developmentseed/titiler) - [titiler-pgstac](https://github.com/stac-utils/titiler-pgstac) -## VEDA ecosystem +## VEDA backend context ![architecture diagram](.readme/veda-backend.drawio.svg) -Veda backend is is the central index of the VEDA ecosystem. This project provides the infrastructure for a PgSTAC database, STAC API, and TiTiler. This infrastructure is used to discover, access, and visualize the Analysis Ready Cloud Optimized (ARCO) assets of the VEDA Data Store. - -| Name | Explanation | -| --- | --- | -| [**veda-config**]() | Configuration for viewing VEDA assets in dashboard UI | -| [**veda-ui**]() | Dashboard UI for viewing and analysing VEDA assets | -| [**veda-stac-ingestor**]() | Load records to PgSTAC | -| [**veda-data-pipelines**]() | Cloud optimize data assets and submit records for publication to veda-stac-ingestor | +Veda backend is is the central index of the [VEDA ecosystem](#veda-ecosystem). This project provides the infrastructure for a PgSTAC database, STAC API, and TiTiler. This infrastructure is used to discover, access, and visualize the Analysis Ready Cloud Optimized (ARCO) assets of the VEDA Data Store. ## Deployment @@ -109,9 +102,21 @@ STAC records should be loaded using [pypgstac](https://github.com/stac-utils/pgs ## Support scripts Support scripts are provided for manual system operations. - [Rotate pgstac password](support_scripts/README.md#rotate-pgstac-password) -## Usage examples: + +## VEDA ecosystem + +| Name | Explanation | +| --- | --- | +| **veda-backend** | Central index (database) and APIs for recording, discovering, viewing, and using VEDA assets | +| [**veda-config**](https://github.com/NASA-IMPACT/veda-config) | Configuration for viewing VEDA assets in dashboard UI | +| [**veda-ui**](https://github.com/NASA-IMPACT/veda-ui) | Dashboard UI for viewing and analysing VEDA assets | +| [**veda-stac-ingestor**](https://github.com/NASA-IMPACT/veda-stac-ingestor) | Entry-point for users/services to add new records to database | +| [**veda-data-pipelines**](https://github.com/NASA-IMPACT/veda-data-pipelines) | Cloud optimize data assets and submit records for publication to veda-stac-ingestor | + +## Usage examples https://github.com/NASA-IMPACT/veda-documentation + # STAC community resources ## STAC browser From 25a196b8a5f893057a22721f4c546513d75abe65 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 10:56:00 -0700 Subject: [PATCH 17/23] add veda ecosystem context --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2c52ad14..e28209cb 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,9 @@ STAC records should be loaded using [pypgstac](https://github.com/stac-utils/pgs Support scripts are provided for manual system operations. - [Rotate pgstac password](support_scripts/README.md#rotate-pgstac-password) -## VEDA ecosystem +# VEDA ecosystem +## Projects | Name | Explanation | | --- | --- | | **veda-backend** | Central index (database) and APIs for recording, discovering, viewing, and using VEDA assets | @@ -112,10 +113,12 @@ Support scripts are provided for manual system operations. | [**veda-ui**](https://github.com/NASA-IMPACT/veda-ui) | Dashboard UI for viewing and analysing VEDA assets | | [**veda-stac-ingestor**](https://github.com/NASA-IMPACT/veda-stac-ingestor) | Entry-point for users/services to add new records to database | | [**veda-data-pipelines**](https://github.com/NASA-IMPACT/veda-data-pipelines) | Cloud optimize data assets and submit records for publication to veda-stac-ingestor | +| [**veda-documentation**](https://github.com/NASA-IMPACT/veda-documentation) | Documentation repository for end users of VEDA ecosystem data and tools | -## Usage examples +## VEDA usage examples -https://github.com/NASA-IMPACT/veda-documentation +[VEDA documentation](https://nasa-impact.github.io/veda-documentation/) +[VEDA dashboard](https://www.earthdata.nasa.gov/dashboard) # STAC community resources From c0250ab0c65693d43f02d94ea47d28161272004b Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 10:57:39 -0700 Subject: [PATCH 18/23] emphasize veda usage --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e28209cb..5c17085a 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,9 @@ Support scripts are provided for manual system operations. ## VEDA usage examples -[VEDA documentation](https://nasa-impact.github.io/veda-documentation/) -[VEDA dashboard](https://www.earthdata.nasa.gov/dashboard) +### [VEDA documentation](https://nasa-impact.github.io/veda-documentation/) + +### [VEDA dashboard](https://www.earthdata.nasa.gov/dashboard) # STAC community resources From cd3af3fc4b0d987a616640e8d44ea3deaae033cb Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 11:27:18 -0700 Subject: [PATCH 19/23] update references to cloud-optimized-pipelines --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5c17085a..986a125c 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,12 @@ The primary tools employed in the [eoAPI demo](https://github.com/developmentsee ## VEDA backend context ![architecture diagram](.readme/veda-backend.drawio.svg) + Veda backend is is the central index of the [VEDA ecosystem](#veda-ecosystem). This project provides the infrastructure for a PgSTAC database, STAC API, and TiTiler. This infrastructure is used to discover, access, and visualize the Analysis Ready Cloud Optimized (ARCO) assets of the VEDA Data Store. ## Deployment -This repo includes CDK scripts to deploy a PgSTAC AWS RDS database and other resources to support APIs maintained by the VEDA backend development team. +This project uses an AWS CDK [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html) stack to deploy a full AWS virtual private cloud environment with a database and supporting lambda function APIs. The deployment constructs, database, and API services are highly configurable. This section provices basic deployment instructions as well as support for customization. ### Tooling & supporting documentation @@ -62,13 +63,9 @@ python3 -m pip install -e ".[dev,deploy,test]" ``` # Review what infrastructure changes your deployment will cause cdk diff -# Execute deployment, security changes will require approval for deployment +# Execute deployment and standby--security changes will require approval for deployment cdk deploy ``` - -#### Check CloudFormation deployment status - -After logging in to the console at https://.signin.aws.amazon.com/console the status of the CloudFormation stack can be viewed here: https://.console.aws.amazon.com/cloudformation/home. ## Deleting the CloudFormation stack @@ -79,6 +76,7 @@ If this is a development stack that is safe to delete, you can delete the stack 3. If this stack created a new VPC, delete the VPC (this should delete a subnet and security group too). ## Custom deployments + The default settings for this project generate a complete AWS environment including a VPC and gateways for the stack. See this guidance for adjusting the veda-backend stack for existing managed and/or shared AWS environments. - [Deploy to an existing managed AWS environment](docs/deploying_to_existing_environments.md) - [Creating a shared base VPC and AWS environment](docs/deploying_to_existing_environments.md#optional-deploy-standalone-base-infrastructure) @@ -96,8 +94,11 @@ docker compose down # Operations -## Ingesting metadata -STAC records should be loaded using [pypgstac](https://github.com/stac-utils/pgstac#pypgstac). The [cloud-optimized-data-pipelines](https://github.com/NASA-IMPACT/cloud-optimized-data-pipelines) project provides examples of cloud pipelines that use pypgstac to load data into a STAC catalog, as well as examples of transforming data to cloud optimized formats. +## Adding new data to veda-backend + +> **Warning** PgSTAC records should be loaded in the database using [pypgstac](https://github.com/stac-utils/pgstac#pypgstac) for proper indexing and partitioning. + +The VEDA ecosystem includes tools specifially created for loading PgSTAC records and optimizing data assets. The [veda-data-pipelines](https://github.com/NASA-IMPACT/veda-data-pipelines) project provides examples of cloud pipelines that transform data to cloud optimized formats, generate STAC metadata, and submit records for publication to the veda-backend database using the [veda-stac-ingestor](https://github.com/NASA-IMPACT/veda-stac-ingestor). ## Support scripts Support scripts are provided for manual system operations. From 3656a54ab0e7b3bf8b4268b20dd619e1691e792d Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 12:25:48 -0700 Subject: [PATCH 20/23] update info flow --- .readme/veda-backend.drawio.svg | 2 +- .readme/veda-backend.drawio.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.readme/veda-backend.drawio.svg b/.readme/veda-backend.drawio.svg index 2ff79cf9..9a85b369 100644 --- a/.readme/veda-backend.drawio.svg +++ b/.readme/veda-backend.drawio.svg @@ -1,4 +1,4 @@ -
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file +
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file diff --git a/.readme/veda-backend.drawio.xml b/.readme/veda-backend.drawio.xml index db85d284..43d3f2aa 100644 --- a/.readme/veda-backend.drawio.xml +++ b/.readme/veda-backend.drawio.xml @@ -1,2 +1,2 @@ -7VzbcuO4Ef0aVyUPYhG8So+SbG+2apNSraeSzNMURMIS1hTBgNDF+/UBSFAiAVAXk5LGicczY6EBgiBw+nSj0dSDO13tfqEwW/6dxCh5cOx49+A+PjgOcF2X/xKS91ISukEpWFAcy0YHwQv+E0mhLaVrHKO80ZARkjCcNYURSVMUsYYMUkq2zWavJGneNYMLpAleIpjo0n/hmC1L6dC3D/K/IbxYVncGtqyZw+htQck6lfd7cNyn4k9ZvYJVX7J9voQx2dZE7tODO6WEsPLTajdFiZjbatrK655bavfjpihl51zglBdsYLJG1YiDhF86eSW8Bz51MCorgv+sxaAmU7KmGFFe9Q+0PYj5p4X8XVw+VwWyv5y9J80ORcUgLyAw5g2Am+3Ky5SeHyGDvP6FEYqqTvmDlf0278XF2v21pk5jNA5ZswSnaLpHlC3HNiUJoUUbF4zFj5DjJKnJn/3xxBftc0bJG6pqUpLyzicxzJcolj1uEGWYA+03OEfJjOSYYZLyujlhjKxqDcYJXogKRjIuhbIU8XXlk+9OlmyV8DKQg5QqBJyqLB9M3BLmWflEr3gnxjHJCBa9PG14Z7nshAMxExesdguh0hbc5p41X0dvSMxWgej9M5RPWSkG2M+mGDratSIR7PHNeQORFWL0nTeRF4TAKy+RlOHajuWXku1BBUMpWta0LxxKxZdKv9j3fQA+/yCxb9YDt1UPYrwRfcv5P8AyQa+sHf4d0N5d5WaLl2/j6UXKsBcXj9uiIgVuikn2J/wvZ5Zp+c/ntVMhsRzfIDTJQl0I9Gb8FzDdQRWaZKEuBHozUapG3RSaZKGvj1i9GhiuBsrV/K87uYhxuFbGmKvSI6a8cckZKaFCL85kI14zssVPz5RUKEI3QjKSD0U5x32Efo3EaCa8WH5SWsW5xk89sJGvslEY6lxkG7jI7s5FnqZ4zccTq04WJOUrRsRKFHP+B2LsXU46XDPSXBENCWDoPIm7t+ItEWCY7L2ZWhPpz1TTjGLFlSpXrRRJr49BukCs4XGcsRQUJZDhTbP3LhPr9z+x/Bno+7/l9UXhuygITpDlx1299vG9XpohivlTCIP+aFTY+y2Te79lCk75pPexqr05st/wN5yIG9/Xh223Dm325KdzZDvYjQSu5jH88bpOpTXt34a4vtO0Ib6r2ZAg1G1IJeuiQ+HtqC78VFQ3/Kmobvh5tt9iSzEYz379oq3/M9ryPN31DQzb8ErWRR9GGjaEMleTKeMpPTKZteeu7/W6FiK7F2dVUdGGGz28H2tV42mLlhiZpzOTvURi/4tzlj+IoGYAV4ViFv+XkrI9/RAlmQMgV3+o2Xqe4Kj/UE3+hli0rFTkK7Sp0e06L+7VA18C223ypa27eb4hVOD3ECoAwKCJN2DQ40z4DFc4EbPR1IZ2zO1hBPrjTOcmjp68dCZgV4se+c3okaeGhcpRyauU5d4P4zwEXP8EZ4NiOMgZjAY4XaCcEXoRXylw3C4xQy9ZOagtV8gmsswcUSclieEjfuAHQqgNBKosc/Xoog9GhrMO4AOdNaqT1U6s0X7a0StmYsjgIMMZEtYn/wJNR9C4CmgqUrkJZPSg9JU98/BzuOb+T+aa+5e45nU9LB3pdJ5/1J8+z3HeXznmuHjPcX6DW834JL8SurqkVwXsNYyavelLuOYYpzTP1h2D+5oitiX0LbeihKzjY578KVes8qxdlctEYQYZd6TTQuLYPR3yB4qrPPJNds+90sEaCLR1/bQna1WopE48wR155woB53tNrWsIt9yT0k1B4k++x0M7zGQAbhTI8vfKsPPPBysvCu+1gnqOcL3dotP/krfsFkeu5bm27YMhcD01D8oPFOYrx63tHbVuA0/ZhNrX24SONIj+czbVUGrMH9Jyh9S8IS1nqJkvpOXiqHk4Wg5OMxtISxhSs4q01KNmdpKWIKRmEWmpRkfzf2oKaNr0nAqIRbwMeadU9qHGw8o2SQKzHM8PexUUrWnOEfw7Kk+CSp/aFDcTapYVPozpeKKo/bHJouN7Ld1ZOivFKOdzgdPFb0Xp0T22bWsEJntwXBxFmVygb748U5Rv37CLCXBMAff/GRPg+V7NBAxsC9j+xXbgo2c6PdkPV7cf4GYGJLCPGhDVdT7XgPihYkBGVzMg+zm9VXjh4bIUhp4VoactgMFpAXdMZqjGc3nsoUvoscux4AUJDBeGHDo+lHzXYUbJBseI5j2MWVGvrwPCmx0QevaowaOBq/sO1zogNL7YoEChcunaFrx1CuTbV3Be9WRfOjVKPMj8zkcADElyjmqMPjQ73unZOQnak+cQGn6NmtKqVio4/1xTZEVkla3L3kyIvngp60tWYkZfs3qIDoTWyPeHQ5EtNPKAaYWGPRw7GFK2r3zs4Fz52OHg9o7Cyg2WTshoeMPoh77ginPh6c5FGQW7hSfRQwq405LH2D1DUu/5A/Y9X84JpDFv8JfirHSNje5LUcd59hUv/voxJ+AO56oVo3U5Uq08kI5Hqm1cVl3hWJ4T2gcic5qWWrNFo5E1tGu0Z+C9Xsx2eNowVYYBr4qXmU+v20lLVvQ0rjwmo/skb/a4ZEy8ij0WT+Q8R3FqW7iAKV8gKqyTcD8LJ/ZZyLkD+4z4/YvPA8fLdoMNSQZiQgTRPAtzBvMBcIZWli5MxNo4qFc9vlNuqgRkMfhJ/Q3tRgWhMaq17gVfgOPLb2CqStmqW9PAct09pALPAKqWJp0wdsa5QoUx/sgMw+R34e2lC1FrjE7WIFiBqihQ+SBFelcZvRMfC1i2HZ/yJWI4XUwLp0lYzEEZ2bk8hYMSBiXUR70Rhx1YofhGAGVB9os8tICeBgjsoVUjj8ALDCs9tMIeMqkdPSSvr+6H94DaInBS5D9H3NizNoIrHMeJKRy8p6hje8FT8XLVpBRbvzKkzEtu20YwoyReR0zm5WfGuDfM8I8FZGgL3/vZMzpKso9bZRTW90Wejp5K1uld+DOizV/QuQ50rvLVCiqYrvhGBy8evrykDAQfviHGffov \ No newline at end of file +7Vxbc+MoFv41qdp5sEp32Y+2k8xO1eyWa9I1M/3UhSVis5GFFuFL5tcPSGBLgHyJZKcznU53xxwQQvCd7xwOR77zpqvdzwTky//gBKZ3rp3s7rz7O9d1HHfIfnHJayUJXa8SLAhKRKOD4An9BYXQFtI1SmDRaEgxTinKm8IYZxmMaUMGCMHbZrNnnDbvmoMF1ARPMUh16R8ooctKOgzsg/zfEC2W8s6OLWrmIH5ZELzOxP3uXO+h/FNVr4DsS7QvliDB25rIe7jzpgRjWn1a7aYw5XMrp6267rGldj9uAjN6zgVudcEGpGsoRxym7NLJM2Y9sKkDcVUR/n/NBzWZ4jVBkLCq/8LtQcw+LcTv8vK5KhD9FfQ1bXbIKwZFCYExa+B4+a66TOn5HlDA6p8oJlB2yh6s6rd5LybW7q81dRujcfGapiiD0z2ibDG2KU4xKdt4zpj/cDlK05r8MRhPAt6+oAS/QFmT4Yx1PklAsYSJ6HEDCUUMaL+COUxnuEAU4YzVzTGleFVrME7RgldQnDMpEKWYrSubfG+ypKuUlR0xSKFCjivL4sH4LUGRV0/0jHZ8HJMcI97Lw4Z1VohOGBBzfsFqt+AqbYFt4VvzdfwC+WyViN4/Q/WUUjGc/WzyocNdKxKdPb4Zb0C8gpS8sibigsjxq0sEZXi2awWVZHtQwUiIljXtiwTZAKH0i33fB+CzDwL7Zj3wWvUgQRvet5j/AyxT+Ezb4d8B7d1VbrZ4+jKeXqQMe3H5uC0qUuKmnORgwv4yZplW/wJWO+USyw0MQpMs0oWO3oz9ckx3UIUmWaQLHb0ZL8lRN4UmWRToI1avdgxXO8rV7K83uYhxmFYmiKnSPSKsccUZGSZcL85kI1YzsvlPz5RUKkI3QjKSD4EFw30Mf4n5aCasWH1SWiWFxk89sFGgslEU6VxkG7jI7s5FvqZ4zcfjq44XOGMrhvlKlHP+P0jpq5h0sKa4uSIaEpyh+8Dv3oq3lINhsvdmak2EP8M7BYSOubfFDVMKigLFUvyIUnlvmCWykTCITCLqD6sFE8UjqxZfuI/CBQRkAWnDcTljRQlMAUWbZu9d1ifof33YM5DXP8X1ZeErL3BqEeX7Xb32/rVemkGC2FNwv+DeqPcffrW991vt8JSH/D42vje3+gv6glJ+4/f1qNttVZt1++7c6g5WLAWreQK+Pa8zYdv7t2he4DYtWuBpFi2MdIsmZV10KLodY0Y/ImMOvyvGHH6cmALfJw3Gs18+2e8HYz/f1/350BBbkLIu+jDSsMGVWU6m0PkeCdHaU+DXel0LH35w6pMR48beYPh+5CfH0xZJMhJYZ0J8inlsABW0uOMB3xCsSv0u/68kVXvyJmYzB4eu/lCz9TxlsOk9jFW8QBovpaZ9hn011l4X5b16oF3H9pq0a+tOZ2AIowQ9hFEcx6CJNyDi44T6CFYo5bPR1IZ2zO1h5JxHvedwpnsTf1FcOuOwq0XWgmZkzVdDZtWoxFXKcu+HcR4Crn+6tYEJGDCzFg9QtoAFxeQivlLguF0iCp/yalBbppBNZJk5ok5KAsNH3Mk3hJcbCFRZ5uqR18AZGc6BnMDRWcPzemCN9pOgXjGTAAoGOcohtz7FJ2g6gsZTQCNJ5SaQ0QP2V3bwox/Kww++Mw8/uMTDr6tz5Y9n8+Ktbvl5/vf+yjGD12uBihvcasYm+RmT1SW9KjpTg7rZKb+Eso5RUzN9wTV4wRmkW0xeCitO8To5tiE45dFJB91TKZEXZoAyfzwrJa7dUx5FqHjco8BkPr0rnV06obauP/rhpYz/1PkrfEf6ukIw/oOvkGeIIb2ngTEF0D/4xhXuEBXByVEoyl+lt8I+H1wXXnitFdSjmuttgd3+l7xlCzzyLN+z7cAZOp6vJr4FocLD1bi1DbHWbegrO2v7ejvrkQbR32dTDaXGhDEtWUxNFNOSxJoJYlrylZp4pSVdNdO/tAwxNY1MyzVrpqNpGWFq2piWW3Y04aumgKad3KkoX8zKgHVKRB9qkK9qk6YgL9D8sAGD8ZoUDMG/weqUrNoomIKBXM3y0qMyHd2Utd82eXx8A6m7bmfllBVsLlC2+LUs3XvH9qKNaGsPbpSrKJPn6DtK3xS63DfsYgJc0ynCP8YE+IFfMwED23Ls4GI78Nbzrp7sh6fbD+dmBiS0jxoQ1ZE/14AEkWJARlczIPs5vVXMpA6RM7JEelaEfkAnSaABundM9JDjuTwS0iWe2uWs84LkjgsDIB0fSrzcMiN4gxJIih7GrKjX56nnzU49fXvU4NHQ032Ha516Gt9kUaAgXbq2BW+dAvG6HZjLnuxLp0aJTplf8gkdQx6iqxqjN82Of3p2ToL25OGKhl+jprSqlQrOv9YEWjFe5euqNxOiL17K+pJVmNHXrB4wdCJrFATDIc+kGvmOaYWGPZylGJLrr3yW4l75LOXg9o4i6QYLJ2Q0vEb047pxMx03io/i6z5KFUy7hUPSQ7K+25Iq2j0JVe/5DW5CsZxjQBLW4F/lOfIaGb2gso7R9TNa/PQ2X+IdzpwlMXY5bpaOTMfj5jZKlFe4lu9G9oEP3abB10zaaGQN7Rp7GuizF+sfnbZv0r6gVfkS/Ol1O2kQy57G0vEyemHiZvdLSvkr/GP+RO5jnGS2hUqYsgUi3MhxL7b0hR+5nPnBj5Ddv/w8cP18N9jgdMAnhBPNI7eKoBg47tDKs4WJnxtJDKrjeMrbFYAsBz+pv9nfqMAkgbXWveDLYfgKGpiS6Wx1oxxanreHVOgbQNXSpBPGzjiekBhjj0wRSH/jTmO24LXGIGcNghJUZYGIBylT36ogIP9YwrLtTJgtEUXZYlr6XtzwDqoA0eXpLQRTIKA+6o047NCK+DdJKAuyX+Sh5egpko49tGrkEfqhYaWHVtRDsrqrR/b11X3zVlJbBEaK7OeIN3zWfnKFkiQ1RZX3FHVsS3kq7K6alHIHWUWmWclr20/mBCfrmIpXH3Jj+Bzk6NsCULgFr/1sPV0lEcqT2Zb17ZWvo0fKOn2HwhlB60/oXAc6V/lKDhVMV3xphhUPX3pTxZMP3yzkPfwN \ No newline at end of file From 802d8b56172651fa22282af92ad2a207618509aa Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 13 Dec 2022 16:15:47 -0700 Subject: [PATCH 21/23] diagram update --- .readme/veda-backend.drawio.svg | 2 +- .readme/veda-backend.drawio.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.readme/veda-backend.drawio.svg b/.readme/veda-backend.drawio.svg index 9a85b369..53c7f05f 100644 --- a/.readme/veda-backend.drawio.svg +++ b/.readme/veda-backend.drawio.svg @@ -1,4 +1,4 @@ -
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file +
Data Store
Data Store
PgSTAC
PgSTAC
TiTiler
TiTiler
STAC-API
STAC-API
Scientists &
Public
Scientis...
veda-stac-ingestor
veda-stac-ingestor
veda-data-pipelines
veda-data-pipelines
 
Analysis
Platform
...
VPC
VPC
Scientists &
Data Providers
Scientis...
Dashboard (veda-ui & veda-config)
Dashboard (veda-...Text is not SVG - cannot display \ No newline at end of file diff --git a/.readme/veda-backend.drawio.xml b/.readme/veda-backend.drawio.xml index 43d3f2aa..0310b577 100644 --- a/.readme/veda-backend.drawio.xml +++ b/.readme/veda-backend.drawio.xml @@ -1,2 +1,2 @@ -7Vxbc+MoFv41qdp5sEp32Y+2k8xO1eyWa9I1M/3UhSVis5GFFuFL5tcPSGBLgHyJZKcznU53xxwQQvCd7xwOR77zpqvdzwTky//gBKZ3rp3s7rz7O9d1HHfIfnHJayUJXa8SLAhKRKOD4An9BYXQFtI1SmDRaEgxTinKm8IYZxmMaUMGCMHbZrNnnDbvmoMF1ARPMUh16R8ooctKOgzsg/zfEC2W8s6OLWrmIH5ZELzOxP3uXO+h/FNVr4DsS7QvliDB25rIe7jzpgRjWn1a7aYw5XMrp6267rGldj9uAjN6zgVudcEGpGsoRxym7NLJM2Y9sKkDcVUR/n/NBzWZ4jVBkLCq/8LtQcw+LcTv8vK5KhD9FfQ1bXbIKwZFCYExa+B4+a66TOn5HlDA6p8oJlB2yh6s6rd5LybW7q81dRujcfGapiiD0z2ibDG2KU4xKdt4zpj/cDlK05r8MRhPAt6+oAS/QFmT4Yx1PklAsYSJ6HEDCUUMaL+COUxnuEAU4YzVzTGleFVrME7RgldQnDMpEKWYrSubfG+ypKuUlR0xSKFCjivL4sH4LUGRV0/0jHZ8HJMcI97Lw4Z1VohOGBBzfsFqt+AqbYFt4VvzdfwC+WyViN4/Q/WUUjGc/WzyocNdKxKdPb4Zb0C8gpS8sibigsjxq0sEZXi2awWVZHtQwUiIljXtiwTZAKH0i33fB+CzDwL7Zj3wWvUgQRvet5j/AyxT+Ezb4d8B7d1VbrZ4+jKeXqQMe3H5uC0qUuKmnORgwv4yZplW/wJWO+USyw0MQpMs0oWO3oz9ckx3UIUmWaQLHb0ZL8lRN4UmWRToI1avdgxXO8rV7K83uYhxmFYmiKnSPSKsccUZGSZcL85kI1YzsvlPz5RUKkI3QjKSD4EFw30Mf4n5aCasWH1SWiWFxk89sFGgslEU6VxkG7jI7s5FvqZ4zcfjq44XOGMrhvlKlHP+P0jpq5h0sKa4uSIaEpyh+8Dv3oq3lINhsvdmak2EP8M7BYSOubfFDVMKigLFUvyIUnlvmCWykTCITCLqD6sFE8UjqxZfuI/CBQRkAWnDcTljRQlMAUWbZu9d1ifof33YM5DXP8X1ZeErL3BqEeX7Xb32/rVemkGC2FNwv+DeqPcffrW991vt8JSH/D42vje3+gv6glJ+4/f1qNttVZt1++7c6g5WLAWreQK+Pa8zYdv7t2he4DYtWuBpFi2MdIsmZV10KLodY0Y/ImMOvyvGHH6cmALfJw3Gs18+2e8HYz/f1/350BBbkLIu+jDSsMGVWU6m0PkeCdHaU+DXel0LH35w6pMR48beYPh+5CfH0xZJMhJYZ0J8inlsABW0uOMB3xCsSv0u/68kVXvyJmYzB4eu/lCz9TxlsOk9jFW8QBovpaZ9hn011l4X5b16oF3H9pq0a+tOZ2AIowQ9hFEcx6CJNyDi44T6CFYo5bPR1IZ2zO1h5JxHvedwpnsTf1FcOuOwq0XWgmZkzVdDZtWoxFXKcu+HcR4Crn+6tYEJGDCzFg9QtoAFxeQivlLguF0iCp/yalBbppBNZJk5ok5KAsNH3Mk3hJcbCFRZ5uqR18AZGc6BnMDRWcPzemCN9pOgXjGTAAoGOcohtz7FJ2g6gsZTQCNJ5SaQ0QP2V3bwox/Kww++Mw8/uMTDr6tz5Y9n8+Ktbvl5/vf+yjGD12uBihvcasYm+RmT1SW9KjpTg7rZKb+Eso5RUzN9wTV4wRmkW0xeCitO8To5tiE45dFJB91TKZEXZoAyfzwrJa7dUx5FqHjco8BkPr0rnV06obauP/rhpYz/1PkrfEf6ukIw/oOvkGeIIb2ngTEF0D/4xhXuEBXByVEoyl+lt8I+H1wXXnitFdSjmuttgd3+l7xlCzzyLN+z7cAZOp6vJr4FocLD1bi1DbHWbegrO2v7ejvrkQbR32dTDaXGhDEtWUxNFNOSxJoJYlrylZp4pSVdNdO/tAwxNY1MyzVrpqNpGWFq2piWW3Y04aumgKad3KkoX8zKgHVKRB9qkK9qk6YgL9D8sAGD8ZoUDMG/weqUrNoomIKBXM3y0qMyHd2Utd82eXx8A6m7bmfllBVsLlC2+LUs3XvH9qKNaGsPbpSrKJPn6DtK3xS63DfsYgJc0ynCP8YE+IFfMwED23Ls4GI78Nbzrp7sh6fbD+dmBiS0jxoQ1ZE/14AEkWJARlczIPs5vVXMpA6RM7JEelaEfkAnSaABundM9JDjuTwS0iWe2uWs84LkjgsDIB0fSrzcMiN4gxJIih7GrKjX56nnzU49fXvU4NHQ032Ha516Gt9kUaAgXbq2BW+dAvG6HZjLnuxLp0aJTplf8gkdQx6iqxqjN82Of3p2ToL25OGKhl+jprSqlQrOv9YEWjFe5euqNxOiL17K+pJVmNHXrB4wdCJrFATDIc+kGvmOaYWGPZylGJLrr3yW4l75LOXg9o4i6QYLJ2Q0vEb047pxMx03io/i6z5KFUy7hUPSQ7K+25Iq2j0JVe/5DW5CsZxjQBLW4F/lOfIaGb2gso7R9TNa/PQ2X+IdzpwlMXY5bpaOTMfj5jZKlFe4lu9G9oEP3abB10zaaGQN7Rp7GuizF+sfnbZv0r6gVfkS/Ol1O2kQy57G0vEyemHiZvdLSvkr/GP+RO5jnGS2hUqYsgUi3MhxL7b0hR+5nPnBj5Ddv/w8cP18N9jgdMAnhBPNI7eKoBg47tDKs4WJnxtJDKrjeMrbFYAsBz+pv9nfqMAkgbXWveDLYfgKGpiS6Wx1oxxanreHVOgbQNXSpBPGzjiekBhjj0wRSH/jTmO24LXGIGcNghJUZYGIBylT36ogIP9YwrLtTJgtEUXZYlr6XtzwDqoA0eXpLQRTIKA+6o047NCK+DdJKAuyX+Sh5egpko49tGrkEfqhYaWHVtRDsrqrR/b11X3zVlJbBEaK7OeIN3zWfnKFkiQ1RZX3FHVsS3kq7K6alHIHWUWmWclr20/mBCfrmIpXH3Jj+Bzk6NsCULgFr/1sPV0lEcqT2Zb17ZWvo0fKOn2HwhlB60/oXAc6V/lKDhVMV3xphhUPX3pTxZMP3yzkPfwN \ No newline at end of file +7Vxtc+MoEv41qbr7YJXeZX+0neRuq/auXJupvdtPU1giNhdZ6BCOnf31BwJsCZBfYtmZ3GQykzENQgiefrppWr4Lpqvt3wgol//AGczvfDfb3gX3d74/ShL2mwvehCAKXSFYEJQJkbcXPKE/oRSqZmuUwarVkGKcU1S2hSkuCpjSlgwQgjftZs84b9+1BAtoCJ5SkJvSf6GMLoV0GLl7+d8hWizVnT1X1sxB+rIgeF3I+935wUP9R1SvgOpLtq+WIMObhih4uAumBGMqPq22U5jzqVXTJq577KjdjZvAgp5ygS8ueAX5GqoRxzm7dPKMWQ9s6kAqKuL/rvmgJlO8JggSVvVPuNmL2aeF/L++fK4LZH8VfcvbHfKKQVVDYMwaeEG5FZdpPd8DClj9E8UEqk7Zg4l+2/diYuP+RlO/NRofr2mOCjjdIcqVY5viHJO6TeCN+Q+XozxvyB+j8STi7StK8AtUNQUuWOeTDFRLmMkeXyGhiAHtVzCH+QxXiCJcsLo5phSvGg3GOVrwCopLJgWylLJ1ZZMfTJZ0lbOyJwcpVcjzVVk+GL8lqErxRM9oy8cxKTHivTy8ss4q2QkDYskvWG0XXKMdsKlCZ75OXyCfrRrRu2cQT6kUw9vNJh863HYi0dvhm9EGxCtIyRtrIi9IvFBcIikjcH0nEpLNXgUTKVo2tC8ZSsWXSr/Y9b0HPvsgsW/Xg6BTDzL0yvuW87+HZQ6faTf8L0D75So3Wzx9G0/PUoaduH7cDhWpcVNPcjRhfxmzTMW/iNVOucTxI4vQJktMoWc2Y/95tjvoQpssMYWe2YyX1KjbQpssicwR61d7lqs97Wr2N5icxThMKzPEVOkeEdZYcEaBCdeLE9mI1Yxc/tMzJdWKcBkhWcmHwIrhPoW/pHw0E1YUn7RWWWXwUw9sFOlspDyaJhe5Fi5yL+ei0FC89uPxVccLXLAVw3wl6jn/D6T0TU46WFPcXhEDCd7Qf+B378RbzsEw2XkzjSbSn+GdAkLH3NvihikHVYVSJX5Eubo3LDLVSBpEJpH1+9WCmeaRicUXoli6gIAsIG05LiesKIE5oOi13fsl6xP1vz7sGcjbv+X1deEPXuDUIsv322bt/VuzNIMEsafgfsG9Ve8//WoHH7fa8TEP+WNsfG9u9Tf0DeX8xh/rUXfbqi7r9sO51RdYsRys5hn4/rwupG3v36IFkd+2aFFgWLQ4MS2akl2iQ8ntGDP5GRlz+EMx5vDzxBT4Pmkwnv3yxX4/GfuFoenPx5bYgpJdog8jAxtcmdVkSp3vkRCdHQX+0azr4MNPTn0qYtzaGww/jvzUeLoiSVYCu5gQn1IeG0AVre54wDcGq1q/699CItqTdzGbPTh09Yearec5g03vYazqBdJ0qTTtK+xrsPa6qu/VA+16btCmXdd0OiNLGCXqIYzieRZNvAERHybUR7BCOZ+NtjZ0Y24HI+806j2FM/2b+Ivy0hmHXSOyFrUja6EeMhOjkldpy70bxmkIuP7p1ivMwICZtXSAigWsKCZn8ZUGx80SUfhUikFtmEK2kWXniCYpSQwfcCffEV5uIVBnmatHXiNvZDkH8iLPZI0g6IE1uk+CesVMBigYlKiE3PpUX6C5EDSBBhpFKjeBjBmwv7KDn/xUHn70g3n40TkeflOdhT9ezKv3uuWn+d+7K8cMXm8Vqm5wqxmb5GdMVuf0qulMA+p2p/wcyjpETe30Bd/iBReQbjB5qZw0x+vs0IbgmEenHPRAp0RemAHK/PGilvhuT3kUseZxjyKb+QyudHbpxca6/uyHlyr+0+Sv+APp6wrB+E++QoElhvSRBsYWQP/kG1e4RVT4LqNwKMvcdxm4jhurBnv/hRfeGgX9vKanffDtNr2jwAkD1428oReEeqpbFGvMKwZpbIGNbhOFXEX23vX20iMDlL/PpgYurSliRnqYnhpmpIW1U8KMdCs91cpIs2onfBk5YXrimJFd1k5AM3LA9EQxI5vsYIpXQ+Vse7djcb2UlQHrlMg+9LCeaJPnoKzQfL/lgumaVAzBv0FxLia2BrbwH9epsvahbIc1de331zI9vGU0nbWTssgqNheoWPxal+6DQ7vPVny1B8fJD7WUL8/cQ4a2YGWga917SN+3nRv835B+GIVt0vfc6GzSf+8JV7/GoukkqADj9Q1I7B40ILrrfqoBiRItGDu6mgHZzemtoiRNiJyQF9KzIvQDOkUCLdB9YGqHGs/5sY9LIqiXnG6ekc5xZsjjwoeSr7PMCH5FGSRVD2PW1OvrnPNm55yhO2rxaByYvsO1zjmt765oUFAuXdeCd06BfMEOzFVP7rlTo8Wj7K/1xJ4l89DXjdG7Zic8PjtHQXv0OMXAr1VTOtVKB+efawKdFK/KtejNhuizl7K5ZAIz5po1Q4Re4oyiaDjkuVOj0LOt0LCH0xNLOv2VT0/8K5+eNGIdiXKDpRMyUrGPXkMd142UmbjRfJTQ9FFE+OwWDkkP6fl+R3Lo5WmnZs/vcBOq5RwDkrEGf6lPjtfI6gXVdYyun9Hir+/zJT7glFkR4yUHzMqRufCAuYsS1RW+E/qJu+dDv23wDZM2GjlDt8GeFvrsxfonx+2bsi9oVb/2fnzdjhrEuqexcrysXpi82f2SUv7S/pg/kf+YZoXroBqmbIEIN3Lci6194UcuZ37wI2T3rz8P/LDcDl5xPuATwonmkVtFUA08f+iUxcLGz620Bd1xPObtSkDWg5803+VvVWCSwUbrXvDlMXxFLUypBLamUY6dINhBKg4toOpochHGTjiQUBhjj0wRyH/jTmOx4LXWIGcDggpUdYHIB6mT3UQQkH+sYdl1CsyWiKJiMa19L254ByJAdH5CC8EUSKiPeiMON3YS/t0R2oLsFnnoeGZSpOcOnQZ5xGFsWemhk/SQnu6bkX1zdd+9lTQWgZEi+zngDZ+0n1yhLMttUeUdRR3aUh4Lu+smpd5Bisg0KwVd+8mS4GydUvmyQ2kNn4MSfV8ACjfgrZ+tp6+lPgUqv7K5vQpN9CjZRd+acELQ+gs614HOVb6EQwfTFV+TYcX919yIePL+q4SCh/8B \ No newline at end of file From 00c4bc093518826ef671dda12caee60cf03f1789 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 11 Jan 2023 13:16:59 -0400 Subject: [PATCH 22/23] Update deprecated actions --- .github/workflows/develop.yml | 20 ++++++++++---------- .github/workflows/main.yml | 20 ++++++++++---------- .github/workflows/pr.yml | 20 ++++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 0533716f..3e35cd68 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -9,13 +9,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} @@ -32,13 +32,13 @@ jobs: needs: [lint] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} @@ -72,9 +72,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' @@ -90,7 +90,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-west-2 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ~/.npm key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} @@ -98,7 +98,7 @@ jobs: - name: Install CDK run: npm install -g aws-cdk@2 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37e753ba..9ca4662d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,13 +9,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} @@ -32,13 +32,13 @@ jobs: needs: [lint] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} @@ -72,9 +72,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' @@ -90,7 +90,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-west-2 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ~/.npm key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} @@ -98,7 +98,7 @@ jobs: - name: Install CDK run: npm install -g aws-cdk@2 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 38fa3832..dfa43fa5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -6,13 +6,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} @@ -29,13 +29,13 @@ jobs: needs: [lint] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} @@ -68,9 +68,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' @@ -86,7 +86,7 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-west-2 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ~/.npm key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} @@ -94,7 +94,7 @@ jobs: - name: Install CDK run: npm install -g aws-cdk@2 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} From e26eb1e3638e98bbd66a5250b5a0da1af244067b Mon Sep 17 00:00:00 2001 From: smohiudd Date: Sat, 14 Jan 2023 20:13:10 -0700 Subject: [PATCH 23/23] cog validate env session --- raster_api/runtime/src/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raster_api/runtime/src/app.py b/raster_api/runtime/src/app.py index f3b9f945..e5d0c23a 100644 --- a/raster_api/runtime/src/app.py +++ b/raster_api/runtime/src/app.py @@ -80,7 +80,7 @@ def cog_validate( strict: bool = Query(False, description="Treat warnings as errors"), ): """Validate a COG""" - return rio_cogeo_info(src_path, strict=strict) + return rio_cogeo_info(src_path, strict=strict, config=settings.get_gdal_config()) @cog.router.get("/viewer", response_class=HTMLResponse)