From 695a21635d070932452a6c2da24f352646d6e804 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 27 Feb 2023 18:10:34 -0500 Subject: [PATCH] add discovery metadata process workflows (#25) * add discovery metadata process workflows * update to ES 8 --- Dockerfile | 5 +- docker/entrypoint.sh | 54 ++++++++++++--------- docker/pygeoapi-config.yml | 20 ++++++++ requirements.txt | 3 +- wis2box_api/plugins/process/station_info.py | 2 +- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c237d0..e49c0c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ # ############################################################################### -FROM geopython/pygeoapi:0.14.0 +FROM geopython/pygeoapi:latest -RUN apt-get update -y && apt-get install curl -y +RUN apt-get update -y && apt-get install curl python3-pip git unzip -y ENV PYGEOAPI_CONFIG=/data/wis2box/config/pygeoapi/local.config.yml ENV PYGEOAPI_OPENAPI=/data/wis2box/config/pygeoapi/local.openapi.yml @@ -32,6 +32,7 @@ COPY ./docker/pygeoapi-config.yml $PYGEOAPI_CONFIG RUN cd /app \ && python3 setup.py install \ + && pip3 install https://github.com/wmo-im/pywcmp/archive/master.zip \ && chmod +x /app/docker/es-entrypoint.sh /app/docker/wait-for-elasticsearch.sh ENTRYPOINT [ "/app/docker/es-entrypoint.sh" ] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 255743b..9d86e19 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -40,43 +40,51 @@ entry_cmd=${1:-run} # Shorthand function error() { - echo "ERROR: $@" - exit -1 + echo "ERROR: $@" + exit -1 } # Workdir cd /pygeoapi -# Lock all python files (for gunicorn hot reload) +# Lock all Python files (for gunicorn hot reload) find . -type f -name "*.py" | xargs chmod -R 0444 -echo "Trying to generate openapi.yml" +echo "Caching topic hierarchy JSON" +mkdir -p ~/.pywcmp/wis2-topic-hierarchy +curl --output-dir /tmp -O https://wmo-im.github.io/wis2-topic-hierarchy/all.json.zip +cd ~/.pywcmp/wis2-topic-hierarchy && unzip -j /tmp/all.json.zip + +echo "Caching WCMP2 schema" +mkdir ~/.pywcmp/wcmp-2 && curl --output-dir ~/.pywcmp/wcmp-2 -O https://raw.githubusercontent.com/wmo-im/wcmp2/main/schemas/wcmp2-bundled.json + +echo "Trying to generate OpenAPI document" pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} # pygeoapi openapi validate ${PYGEOAPI_OPENAPI} -[[ $? -ne 0 ]] && error "openapi.yml could not be generated ERROR" +[[ $? -ne 0 ]] && error "ERROR: OpenAPI document could not be generated" echo "openapi.yml generated continue to pygeoapi" case ${entry_cmd} in - # Run pygeoapi server - run) - # SCRIPT_NAME should not have value '/' - [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" - - echo "Start gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" - exec gunicorn --workers ${WSGI_WORKERS} \ - --worker-class=${WSGI_WORKER_CLASS} \ - --timeout ${WSGI_WORKER_TIMEOUT} \ - --name=${CONTAINER_NAME} \ - --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ - --reload \ - --reload-extra-file ${PYGEOAPI_CONFIG} \ - wis2box_api.app:app - ;; - *) - error "unknown command arg: must be run (default) or test" - ;; + # Run pygeoapi server + run) + # SCRIPT_NAME should not have value '/' + [[ "${SCRIPT_NAME}" = '/' ]] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" + + echo "Start gunicorn name=${CONTAINER_NAME} on ${CONTAINER_HOST}:${CONTAINER_PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" + exec gunicorn --workers ${WSGI_WORKERS} \ + --worker-class=${WSGI_WORKER_CLASS} \ + --timeout ${WSGI_WORKER_TIMEOUT} \ + --name=${CONTAINER_NAME} \ + --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ + --reload \ + --reload-extra-file ${PYGEOAPI_CONFIG} \ + wis2box_api.app:app + ;; + *) + error "unknown command arg: must be run (default)" + ;; esac echo "END /entrypoint.sh" diff --git a/docker/pygeoapi-config.yml b/docker/pygeoapi-config.yml index a1215be..3b12621 100644 --- a/docker/pygeoapi-config.yml +++ b/docker/pygeoapi-config.yml @@ -90,3 +90,23 @@ resources: type: process processor: name: wis2box_api.plugins.process.station_info.StationInfoProcessor + + pygeometa-metadata-generate: + type: process + processor: + name: pygeometa.pygeoapi_plugin.PygeometaMetadataGenerateProcessor + + pywcmp-wis2-topics-list: + type: process + processor: + name: pywcmp.pygeoapi_plugin.WIS2TopicHierarchyListTopicsProcessor + + pywcmp-wis2-topics-validate: + type: process + processor: + name: pywcmp.pygeoapi_plugin.WIS2TopicHierarchyValidateTopicProcessor + + pywcmp-wis2-wcmp2-ets: + type: process + processor: + name: pywcmp.pygeoapi_plugin.WCMP2ETSProcessor diff --git a/requirements.txt b/requirements.txt index b74ea6e..77ce450 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -elasticsearch<8 +elasticsearch flask-cors json-merge-patch +pygeometa diff --git a/wis2box_api/plugins/process/station_info.py b/wis2box_api/plugins/process/station_info.py index 562c33d..92b5319 100644 --- a/wis2box_api/plugins/process/station_info.py +++ b/wis2box_api/plugins/process/station_info.py @@ -192,7 +192,7 @@ def execute(self, data): } query = {'size': 0, 'query': query_core, 'aggs': query_agg} - response = self.es.search(index=index, body=query) + response = self.es.search(index=index, **query) response_buckets = response['aggregations']['each']['buckets'] hits = {b['key']: len(b['count']['buckets']) for b in response_buckets}