From 0bb34f576c4e2d2c5b73cc783370af51b22eea48 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 11:29:35 +0100 Subject: [PATCH 01/71] Move api to subdirectory --- pygeoapi/{api.py => api/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pygeoapi/{api.py => api/__init__.py} (100%) diff --git a/pygeoapi/api.py b/pygeoapi/api/__init__.py similarity index 100% rename from pygeoapi/api.py rename to pygeoapi/api/__init__.py From adf90ebd1c16b71d891238ad2cbf8e06463e16bf Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 11:33:08 +0100 Subject: [PATCH 02/71] Move processes api to own file --- pygeoapi/api/__init__.py | 480 --------------------------------- pygeoapi/api/processes.py | 541 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 541 insertions(+), 480 deletions(-) create mode 100644 pygeoapi/api/processes.py diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 13bdfe1d5..f92eaf3aa 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -3175,486 +3175,6 @@ def get_collection_map_legend( return headers, HTTPStatus.BAD_REQUEST, to_json( data, self.pretty_print) - @gzip - @pre_process - @jsonldify - def describe_processes(self, request: Union[APIRequest, Any], - process=None) -> Tuple[dict, int, str]: - """ - Provide processes metadata - - :param request: A request object - :param process: process identifier, defaults to None to obtain - information about all processes - - :returns: tuple of headers, status code, content - """ - - processes = [] - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if process is not None: - if process not in self.manager.processes.keys(): - msg = 'Identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - if len(self.manager.processes) > 0: - if process is not None: - relevant_processes = [process] - else: - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - relevant_processes = list(self.manager.processes)[:limit] - except TypeError: - LOGGER.debug('returning all processes') - relevant_processes = self.manager.processes.keys() - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - for key in relevant_processes: - p = self.manager.get_processor(key) - p2 = l10n.translate_struct(deepcopy(p.metadata), - request.locale) - p2['id'] = key - - if process is None: - p2.pop('inputs') - p2.pop('outputs') - p2.pop('example', None) - - p2['jobControlOptions'] = ['sync-execute'] - if self.manager.is_async: - p2['jobControlOptions'].append('async-execute') - - p2['outputTransmission'] = ['value'] - p2['links'] = p2.get('links', []) - - jobs_url = f"{self.base_url}/jobs" - process_url = f"{self.base_url}/processes/{key}" - - # TODO translation support - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'href': f'{process_url}?f={F_JSON}', - 'title': 'Process description as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'href': f'{process_url}?f={F_HTML}', - 'title': 'Process description as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_HTML}', - 'title': 'jobs for this process as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_JSON}', - 'title': 'jobs for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', - 'href': f'{process_url}/execution?f={F_JSON}', - 'title': 'Execution for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - processes.append(p2) - - if process is not None: - response = processes[0] - else: - process_url = f"{self.base_url}/processes" - response = { - 'processes': processes, - 'links': [{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{process_url}?f={F_JSON}' - }, { - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{process_url}?f={F_JSONLD}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{process_url}?f={F_HTML}' - }] - } - - if request.format == F_HTML: # render - if process is not None: - response = render_j2_template(self.tpl_config, - 'processes/process.html', - response, request.locale) - else: - response = render_j2_template(self.tpl_config, - 'processes/index.html', response, - request.locale) - - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(response, self.pretty_print) - - @gzip - @pre_process - def get_jobs(self, request: Union[APIRequest, Any], - job_id=None) -> Tuple[dict, int, str]: - """ - Get process jobs - - :param request: A request object - :param job_id: id of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if job_id is None: - jobs = sorted(self.manager.get_jobs(), - key=lambda k: k['job_start_datetime'], - reverse=True) - else: - try: - jobs = [self.manager.get_job(job_id)] - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, - 'InvalidParameterValue', job_id) - - serialized_jobs = { - 'jobs': [], - 'links': [{ - 'href': f"{self.base_url}/jobs?f={F_HTML}", - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'Jobs list as HTML' - }, { - 'href': f"{self.base_url}/jobs?f={F_JSON}", - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs list as JSON' - }] - } - for job_ in jobs: - job2 = { - 'type': 'process', - 'processID': job_['process_id'], - 'jobID': job_['identifier'], - 'status': job_['status'], - 'message': job_['message'], - 'progress': job_['progress'], - 'parameters': job_.get('parameters'), - 'job_start_datetime': job_['job_start_datetime'], - 'job_end_datetime': job_['job_end_datetime'] - } - - # TODO: translate - if JobStatus[job_['status']] in ( - JobStatus.successful, JobStatus.running, JobStatus.accepted): - - job_result_url = f"{self.base_url}/jobs/{job_['identifier']}/results" # noqa - - job2['links'] = [{ - 'href': f'{job_result_url}?f={F_HTML}', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', - 'type': FORMAT_TYPES[F_HTML], - 'title': f'results of job {job_id} as HTML' - }, { - 'href': f'{job_result_url}?f={F_JSON}', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', - 'type': FORMAT_TYPES[F_JSON], - 'title': f'results of job {job_id} as JSON' - }] - - if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], - FORMAT_TYPES[F_HTML]): - - job2['links'].append({ - 'href': job_result_url, - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa - 'type': job_['mimetype'], - 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa - }) - - serialized_jobs['jobs'].append(job2) - - if job_id is None: - j2_template = 'jobs/index.html' - else: - serialized_jobs = serialized_jobs['jobs'][0] - j2_template = 'jobs/job.html' - - if request.format == F_HTML: - data = { - 'jobs': serialized_jobs, - 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) - } - response = render_j2_template(self.tpl_config, j2_template, data, - request.locale) - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(serialized_jobs, - self.pretty_print) - - @gzip - @pre_process - def execute_process(self, request: Union[APIRequest, Any], - process_id) -> Tuple[dict, int, str]: - """ - Execute process - - :param request: A request object - :param process_id: id of process - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Responses are always in US English only - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if process_id not in self.manager.processes: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - data = request.data - if not data: - # TODO not all processes require input, e.g. time-dependent or - # random value generators - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - try: - # Parse bytes data, if applicable - data = data.decode() - LOGGER.debug(data) - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input does not appear to be valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - data_dict = data.get('inputs', {}) - LOGGER.debug(data_dict) - - try: - execution_mode = RequestedProcessExecutionMode( - request.headers.get('Prefer', request.headers.get('prefer')) - ) - except ValueError: - execution_mode = None - try: - LOGGER.debug('Executing process') - result = self.manager.execute_process( - process_id, data_dict, execution_mode=execution_mode) - job_id, mime_type, outputs, status, additional_headers = result - headers.update(additional_headers or {}) - headers['Location'] = f'{self.base_url}/jobs/{job_id}' - except ProcessorExecuteError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, - request.format, err.ogc_exception_code, err.message) - - response = {} - if status == JobStatus.failed: - response = outputs - - if data.get('response', 'raw') == 'raw': - headers['Content-Type'] = mime_type - response = outputs - elif status not in (JobStatus.failed, JobStatus.accepted): - response['outputs'] = [outputs] - - if status == JobStatus.accepted: - http_status = HTTPStatus.CREATED - elif status == JobStatus.failed: - http_status = HTTPStatus.BAD_REQUEST - else: - http_status = HTTPStatus.OK - - if mime_type == 'application/json': - response2 = to_json(response, self.pretty_print) - else: - response2 = response - - return headers, http_status, response2 - - @gzip - @pre_process - def get_job_result(self, request: Union[APIRequest, Any], - job_id) -> Tuple[dict, int, str]: - """ - Get result of job (instance of a process) - - :param request: A request object - :param job_id: ID of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - try: - job = self.manager.get_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchJob', job_id - ) - - status = JobStatus[job['status']] - - if status == JobStatus.running: - msg = 'job still running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.accepted: - # NOTE: this case is not mentioned in the specification - msg = 'job accepted but not yet running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.failed: - msg = 'job failed' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - mimetype, job_output = self.manager.get_job_result(job_id) - except JobResultNotFoundError: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'JobResultNotFound', job_id - ) - - if mimetype not in (None, FORMAT_TYPES[F_JSON]): - headers['Content-Type'] = mimetype - content = job_output - else: - if request.format == F_JSON: - content = json.dumps(job_output, sort_keys=True, indent=4, - default=json_serial) - else: - # HTML - headers['Content-Type'] = "text/html" - data = { - 'job': {'id': job_id}, - 'result': job_output - } - content = render_j2_template( - self.config, 'jobs/results/index.html', - data, request.locale) - - return headers, HTTPStatus.OK, content - - @pre_process - def delete_job( - self, request: Union[APIRequest, Any], job_id - ) -> Tuple[dict, int, str]: - """ - Delete a process job - - :param job_id: job identifier - - :returns: tuple of headers, status code, content - """ - response_headers = request.get_response_headers( - SYSTEM_LOCALE, **self.api_headers) - try: - success = self.manager.delete_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, response_headers, request.format, - 'NoSuchJob', job_id - ) - else: - if success: - http_status = HTTPStatus.OK - jobs_url = f"{self.base_url}/jobs" - - response = { - 'jobID': job_id, - 'status': JobStatus.dismissed.value, - 'message': 'Job dismissed', - 'progress': 100, - 'links': [{ - 'href': jobs_url, - 'rel': 'up', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The job list for the current process' - }] - } - else: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, - request.format, 'InternalError', job_id - ) - LOGGER.info(response) - # TODO: this response does not have any headers - return {}, http_status, response - @gzip @pre_process def get_collection_edr_query( diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py new file mode 100644 index 000000000..4428bd4a4 --- /dev/null +++ b/pygeoapi/api/processes.py @@ -0,0 +1,541 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +import json +from http import HTTPStatus +from typing import Tuple + +from pygeoapi.util import ( + json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, + to_json) +from pygeoapi.process.base import ( + JobNotFoundError, + JobResultNotFoundError, + ProcessorExecuteError, +) +from pygeoapi.process.manager.base import get_manager +from pygeoapi.openapi import OPENAPI_YAML + +from . import APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES + +LOGGER = logging.getLogger(__name__) + + +@gzip +@pre_process +@jsonldify +def describe_processes(self, request: Union[APIRequest, Any], + process=None) -> Tuple[dict, int, str]: + """ + Provide processes metadata + + :param request: A request object + :param process: process identifier, defaults to None to obtain + information about all processes + + :returns: tuple of headers, status code, content + """ + + processes = [] + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(**self.api_headers) + + if process is not None: + if process not in self.manager.processes.keys(): + msg = 'Identifier not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + if len(self.manager.processes) > 0: + if process is not None: + relevant_processes = [process] + else: + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + + if limit <= 0: + msg = 'limit value should be strictly positive' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + relevant_processes = list(self.manager.processes)[:limit] + except TypeError: + LOGGER.debug('returning all processes') + relevant_processes = self.manager.processes.keys() + except ValueError: + msg = 'limit value should be an integer' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + for key in relevant_processes: + p = self.manager.get_processor(key) + p2 = l10n.translate_struct(deepcopy(p.metadata), + request.locale) + p2['id'] = key + + if process is None: + p2.pop('inputs') + p2.pop('outputs') + p2.pop('example', None) + + p2['jobControlOptions'] = ['sync-execute'] + if self.manager.is_async: + p2['jobControlOptions'].append('async-execute') + + p2['outputTransmission'] = ['value'] + p2['links'] = p2.get('links', []) + + jobs_url = f"{self.base_url}/jobs" + process_url = f"{self.base_url}/processes/{key}" + + # TODO translation support + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'href': f'{process_url}?f={F_JSON}', + 'title': 'Process description as JSON', + 'hreflang': self.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'href': f'{process_url}?f={F_HTML}', + 'title': 'Process description as HTML', + 'hreflang': self.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_HTML}', + 'title': 'jobs for this process as HTML', + 'hreflang': self.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_JSON}', + 'title': 'jobs for this process as JSON', + 'hreflang': self.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', + 'href': f'{process_url}/execution?f={F_JSON}', + 'title': 'Execution for this process as JSON', + 'hreflang': self.default_locale + } + p2['links'].append(link) + + processes.append(p2) + + if process is not None: + response = processes[0] + else: + process_url = f"{self.base_url}/processes" + response = { + 'processes': processes, + 'links': [{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{process_url}?f={F_JSON}' + }, { + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{process_url}?f={F_JSONLD}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{process_url}?f={F_HTML}' + }] + } + + if request.format == F_HTML: # render + if process is not None: + response = render_j2_template(self.tpl_config, + 'processes/process.html', + response, request.locale) + else: + response = render_j2_template(self.tpl_config, + 'processes/index.html', response, + request.locale) + + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(response, self.pretty_print) + +@gzip +@pre_process +def get_jobs(self, request: Union[APIRequest, Any], + job_id=None) -> Tuple[dict, int, str]: + """ + Get process jobs + + :param request: A request object + :param job_id: id of job + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + if job_id is None: + jobs = sorted(self.manager.get_jobs(), + key=lambda k: k['job_start_datetime'], + reverse=True) + else: + try: + jobs = [self.manager.get_job(job_id)] + except JobNotFoundError: + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, + 'InvalidParameterValue', job_id) + + serialized_jobs = { + 'jobs': [], + 'links': [{ + 'href': f"{self.base_url}/jobs?f={F_HTML}", + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'Jobs list as HTML' + }, { + 'href': f"{self.base_url}/jobs?f={F_JSON}", + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Jobs list as JSON' + }] + } + for job_ in jobs: + job2 = { + 'type': 'process', + 'processID': job_['process_id'], + 'jobID': job_['identifier'], + 'status': job_['status'], + 'message': job_['message'], + 'progress': job_['progress'], + 'parameters': job_.get('parameters'), + 'job_start_datetime': job_['job_start_datetime'], + 'job_end_datetime': job_['job_end_datetime'] + } + + # TODO: translate + if JobStatus[job_['status']] in ( + JobStatus.successful, JobStatus.running, JobStatus.accepted): + + job_result_url = f"{self.base_url}/jobs/{job_['identifier']}/results" # noqa + + job2['links'] = [{ + 'href': f'{job_result_url}?f={F_HTML}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_HTML], + 'title': f'results of job {job_id} as HTML' + }, { + 'href': f'{job_result_url}?f={F_JSON}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_JSON], + 'title': f'results of job {job_id} as JSON' + }] + + if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_HTML]): + + job2['links'].append({ + 'href': job_result_url, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa + 'type': job_['mimetype'], + 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa + }) + + serialized_jobs['jobs'].append(job2) + + if job_id is None: + j2_template = 'jobs/index.html' + else: + serialized_jobs = serialized_jobs['jobs'][0] + j2_template = 'jobs/job.html' + + if request.format == F_HTML: + data = { + 'jobs': serialized_jobs, + 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) + } + response = render_j2_template(self.tpl_config, j2_template, data, + request.locale) + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(serialized_jobs, + self.pretty_print) + +@gzip +@pre_process +def execute_process(self, request: Union[APIRequest, Any], + process_id) -> Tuple[dict, int, str]: + """ + Execute process + + :param request: A request object + :param process_id: id of process + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + # Responses are always in US English only + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + if process_id not in self.manager.processes: + msg = 'identifier not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + data = request.data + if not data: + # TODO not all processes require input, e.g. time-dependent or + # random value generators + msg = 'missing request data' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + try: + # Parse bytes data, if applicable + data = data.decode() + LOGGER.debug(data) + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input does not appear to be valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + data_dict = data.get('inputs', {}) + LOGGER.debug(data_dict) + + try: + execution_mode = RequestedProcessExecutionMode( + request.headers.get('Prefer', request.headers.get('prefer')) + ) + except ValueError: + execution_mode = None + try: + LOGGER.debug('Executing process') + result = self.manager.execute_process( + process_id, data_dict, execution_mode=execution_mode) + job_id, mime_type, outputs, status, additional_headers = result + headers.update(additional_headers or {}) + headers['Location'] = f'{self.base_url}/jobs/{job_id}' + except ProcessorExecuteError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, + request.format, err.ogc_exception_code, err.message) + + response = {} + if status == JobStatus.failed: + response = outputs + + if data.get('response', 'raw') == 'raw': + headers['Content-Type'] = mime_type + response = outputs + elif status not in (JobStatus.failed, JobStatus.accepted): + response['outputs'] = [outputs] + + if status == JobStatus.accepted: + http_status = HTTPStatus.CREATED + elif status == JobStatus.failed: + http_status = HTTPStatus.BAD_REQUEST + else: + http_status = HTTPStatus.OK + + if mime_type == 'application/json': + response2 = to_json(response, self.pretty_print) + else: + response2 = response + + return headers, http_status, response2 + +@gzip +@pre_process +def get_job_result(self, request: Union[APIRequest, Any], + job_id) -> Tuple[dict, int, str]: + """ + Get result of job (instance of a process) + + :param request: A request object + :param job_id: ID of job + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + try: + job = self.manager.get_job(job_id) + except JobNotFoundError: + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchJob', job_id + ) + + status = JobStatus[job['status']] + + if status == JobStatus.running: + msg = 'job still running' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.accepted: + # NOTE: this case is not mentioned in the specification + msg = 'job accepted but not yet running' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.failed: + msg = 'job failed' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + mimetype, job_output = self.manager.get_job_result(job_id) + except JobResultNotFoundError: + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'JobResultNotFound', job_id + ) + + if mimetype not in (None, FORMAT_TYPES[F_JSON]): + headers['Content-Type'] = mimetype + content = job_output + else: + if request.format == F_JSON: + content = json.dumps(job_output, sort_keys=True, indent=4, + default=json_serial) + else: + # HTML + headers['Content-Type'] = "text/html" + data = { + 'job': {'id': job_id}, + 'result': job_output + } + content = render_j2_template( + self.config, 'jobs/results/index.html', + data, request.locale) + + return headers, HTTPStatus.OK, content + +@pre_process +def delete_job( + self, request: Union[APIRequest, Any], job_id +) -> Tuple[dict, int, str]: + """ + Delete a process job + + :param job_id: job identifier + + :returns: tuple of headers, status code, content + """ + response_headers = request.get_response_headers( + SYSTEM_LOCALE, **self.api_headers) + try: + success = self.manager.delete_job(job_id) + except JobNotFoundError: + return self.get_exception( + HTTPStatus.NOT_FOUND, response_headers, request.format, + 'NoSuchJob', job_id + ) + else: + if success: + http_status = HTTPStatus.OK + jobs_url = f"{self.base_url}/jobs" + + response = { + 'jobID': job_id, + 'status': JobStatus.dismissed.value, + 'message': 'Job dismissed', + 'progress': 100, + 'links': [{ + 'href': jobs_url, + 'rel': 'up', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'The job list for the current process' + }] + } + else: + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, + request.format, 'InternalError', job_id + ) + LOGGER.info(response) + # TODO: this response does not have any headers + return {}, http_status, response + From cdb6176059f9dbcfbc8ed33ed093dcf6bcff56e5 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 11:41:17 +0100 Subject: [PATCH 03/71] Adapt processes view methods --- pygeoapi/api/__init__.py | 11 +++ pygeoapi/api/processes.py | 143 +++++++++++++++++--------------------- tests/test_api.py | 124 ++++++++++++++++----------------- tests/util.py | 20 ++++++ 4 files changed, 157 insertions(+), 141 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index f92eaf3aa..c69039190 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -52,6 +52,7 @@ import urllib.parse from dateutil.parser import parse as dateparse +import flask from pygeofilter.parsers.ecql import parse as parse_ecql_text from pygeofilter.parsers.cql_json import parse as parse_cql_json from pyproj.exceptions import CRSError @@ -377,6 +378,16 @@ def with_data(cls, request, supported_locales) -> 'APIRequest': request.body(), loop).result(1) return api_req + @classmethod + def from_flask(cls, request: flask.Request, supported_locales + ) -> 'APIRequest': + """ + Factory class similar to with_data, but only for flask requests + """ + api_req = cls(request, supported_locales) + api_req._data = request.data + return api_req + @staticmethod def _get_params(request): """ diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 4428bd4a4..d6747fe09 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -38,11 +38,14 @@ # ================================================================= +from copy import deepcopy +from datetime import datetime, timezone import logging -import json from http import HTTPStatus +import json from typing import Tuple +from pygeoapi import l10n from pygeoapi.util import ( json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, to_json) @@ -54,15 +57,12 @@ from pygeoapi.process.manager.base import get_manager from pygeoapi.openapi import OPENAPI_YAML -from . import APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES +from . import APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, DATETIME_FORMAT, F_JSONLD LOGGER = logging.getLogger(__name__) -@gzip -@pre_process -@jsonldify -def describe_processes(self, request: Union[APIRequest, Any], +def describe_processes(api: API, request: APIRequest, process=None) -> Tuple[dict, int, str]: """ Provide processes metadata @@ -76,18 +76,16 @@ def describe_processes(self, request: Union[APIRequest, Any], processes = [] - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) + headers = request.get_response_headers(**api.api_headers) if process is not None: - if process not in self.manager.processes.keys(): + if process not in api.manager.processes.keys(): msg = 'Identifier not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NoSuchProcess', msg) - if len(self.manager.processes) > 0: + if len(api.manager.processes) > 0: if process is not None: relevant_processes = [process] else: @@ -97,22 +95,22 @@ def describe_processes(self, request: Union[APIRequest, Any], if limit <= 0: msg = 'limit value should be strictly positive' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) - relevant_processes = list(self.manager.processes)[:limit] + relevant_processes = list(api.manager.processes)[:limit] except TypeError: LOGGER.debug('returning all processes') - relevant_processes = self.manager.processes.keys() + relevant_processes = api.manager.processes.keys() except ValueError: msg = 'limit value should be an integer' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) for key in relevant_processes: - p = self.manager.get_processor(key) + p = api.manager.get_processor(key) p2 = l10n.translate_struct(deepcopy(p.metadata), request.locale) p2['id'] = key @@ -123,14 +121,14 @@ def describe_processes(self, request: Union[APIRequest, Any], p2.pop('example', None) p2['jobControlOptions'] = ['sync-execute'] - if self.manager.is_async: + if api.manager.is_async: p2['jobControlOptions'].append('async-execute') p2['outputTransmission'] = ['value'] p2['links'] = p2.get('links', []) - jobs_url = f"{self.base_url}/jobs" - process_url = f"{self.base_url}/processes/{key}" + jobs_url = f"{api.base_url}/jobs" + process_url = f"{api.base_url}/processes/{key}" # TODO translation support link = { @@ -138,7 +136,7 @@ def describe_processes(self, request: Union[APIRequest, Any], 'rel': request.get_linkrel(F_JSON), 'href': f'{process_url}?f={F_JSON}', 'title': 'Process description as JSON', - 'hreflang': self.default_locale + 'hreflang': api.default_locale } p2['links'].append(link) @@ -147,7 +145,7 @@ def describe_processes(self, request: Union[APIRequest, Any], 'rel': request.get_linkrel(F_HTML), 'href': f'{process_url}?f={F_HTML}', 'title': 'Process description as HTML', - 'hreflang': self.default_locale + 'hreflang': api.default_locale } p2['links'].append(link) @@ -156,7 +154,7 @@ def describe_processes(self, request: Union[APIRequest, Any], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', 'href': f'{jobs_url}?f={F_HTML}', 'title': 'jobs for this process as HTML', - 'hreflang': self.default_locale + 'hreflang': api.default_locale } p2['links'].append(link) @@ -165,7 +163,7 @@ def describe_processes(self, request: Union[APIRequest, Any], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', 'href': f'{jobs_url}?f={F_JSON}', 'title': 'jobs for this process as JSON', - 'hreflang': self.default_locale + 'hreflang': api.default_locale } p2['links'].append(link) @@ -174,7 +172,7 @@ def describe_processes(self, request: Union[APIRequest, Any], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', 'href': f'{process_url}/execution?f={F_JSON}', 'title': 'Execution for this process as JSON', - 'hreflang': self.default_locale + 'hreflang': api.default_locale } p2['links'].append(link) @@ -183,7 +181,7 @@ def describe_processes(self, request: Union[APIRequest, Any], if process is not None: response = processes[0] else: - process_url = f"{self.base_url}/processes" + process_url = f"{api.base_url}/processes" response = { 'processes': processes, 'links': [{ @@ -206,21 +204,20 @@ def describe_processes(self, request: Union[APIRequest, Any], if request.format == F_HTML: # render if process is not None: - response = render_j2_template(self.tpl_config, + response = render_j2_template(api.tpl_config, 'processes/process.html', response, request.locale) else: - response = render_j2_template(self.tpl_config, + response = render_j2_template(api.tpl_config, 'processes/index.html', response, request.locale) return headers, HTTPStatus.OK, response - return headers, HTTPStatus.OK, to_json(response, self.pretty_print) + return headers, HTTPStatus.OK, to_json(response, api.pretty_print) -@gzip -@pre_process -def get_jobs(self, request: Union[APIRequest, Any], +# TODO: get_jobs doesn't have tests +def get_jobs(api: API, request: APIRequest, job_id=None) -> Tuple[dict, int, str]: """ Get process jobs @@ -231,31 +228,29 @@ def get_jobs(self, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) + **api.api_headers) if job_id is None: - jobs = sorted(self.manager.get_jobs(), + jobs = sorted(api.manager.get_jobs(), key=lambda k: k['job_start_datetime'], reverse=True) else: try: - jobs = [self.manager.get_job(job_id)] + jobs = [api.manager.get_job(job_id)] except JobNotFoundError: - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'InvalidParameterValue', job_id) serialized_jobs = { 'jobs': [], 'links': [{ - 'href': f"{self.base_url}/jobs?f={F_HTML}", + 'href': f"{api.base_url}/jobs?f={F_HTML}", 'rel': request.get_linkrel(F_HTML), 'type': FORMAT_TYPES[F_HTML], 'title': 'Jobs list as HTML' }, { - 'href': f"{self.base_url}/jobs?f={F_JSON}", + 'href': f"{api.base_url}/jobs?f={F_JSON}", 'rel': request.get_linkrel(F_JSON), 'type': FORMAT_TYPES[F_JSON], 'title': 'Jobs list as JSON' @@ -278,7 +273,7 @@ def get_jobs(self, request: Union[APIRequest, Any], if JobStatus[job_['status']] in ( JobStatus.successful, JobStatus.running, JobStatus.accepted): - job_result_url = f"{self.base_url}/jobs/{job_['identifier']}/results" # noqa + job_result_url = f"{api.base_url}/jobs/{job_['identifier']}/results" # noqa job2['links'] = [{ 'href': f'{job_result_url}?f={F_HTML}', @@ -315,16 +310,14 @@ def get_jobs(self, request: Union[APIRequest, Any], 'jobs': serialized_jobs, 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) } - response = render_j2_template(self.tpl_config, j2_template, data, + response = render_j2_template(api.tpl_config, j2_template, data, request.locale) return headers, HTTPStatus.OK, response return headers, HTTPStatus.OK, to_json(serialized_jobs, - self.pretty_print) + api.pretty_print) -@gzip -@pre_process -def execute_process(self, request: Union[APIRequest, Any], +def execute_process(api: API, request: APIRequest, process_id) -> Tuple[dict, int, str]: """ Execute process @@ -335,15 +328,12 @@ def execute_process(self, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) - # Responses are always in US English only headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if process_id not in self.manager.processes: + **api.api_headers) + if process_id not in api.manager.processes: msg = 'identifier not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NoSuchProcess', msg) @@ -352,7 +342,7 @@ def execute_process(self, request: Union[APIRequest, Any], # TODO not all processes require input, e.g. time-dependent or # random value generators msg = 'missing request data' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'MissingParameterValue', msg) @@ -369,7 +359,7 @@ def execute_process(self, request: Union[APIRequest, Any], # Input does not appear to be valid JSON LOGGER.error(err) msg = 'invalid request data' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) @@ -384,14 +374,14 @@ def execute_process(self, request: Union[APIRequest, Any], execution_mode = None try: LOGGER.debug('Executing process') - result = self.manager.execute_process( + result = api.manager.execute_process( process_id, data_dict, execution_mode=execution_mode) job_id, mime_type, outputs, status, additional_headers = result headers.update(additional_headers or {}) - headers['Location'] = f'{self.base_url}/jobs/{job_id}' + headers['Location'] = f'{api.base_url}/jobs/{job_id}' except ProcessorExecuteError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) @@ -413,15 +403,13 @@ def execute_process(self, request: Union[APIRequest, Any], http_status = HTTPStatus.OK if mime_type == 'application/json': - response2 = to_json(response, self.pretty_print) + response2 = to_json(response, api.pretty_print) else: response2 = response return headers, http_status, response2 -@gzip -@pre_process -def get_job_result(self, request: Union[APIRequest, Any], +def get_job_result(api: API, request: APIRequest, job_id) -> Tuple[dict, int, str]: """ Get result of job (instance of a process) @@ -432,14 +420,12 @@ def get_job_result(self, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) + **api.api_headers) try: - job = self.manager.get_job(job_id) + job = api.manager.get_job(job_id) except JobNotFoundError: - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NoSuchJob', job_id ) @@ -448,27 +434,27 @@ def get_job_result(self, request: Union[APIRequest, Any], if status == JobStatus.running: msg = 'job still running' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'ResultNotReady', msg) elif status == JobStatus.accepted: # NOTE: this case is not mentioned in the specification msg = 'job accepted but not yet running' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'ResultNotReady', msg) elif status == JobStatus.failed: msg = 'job failed' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) try: - mimetype, job_output = self.manager.get_job_result(job_id) + mimetype, job_output = api.manager.get_job_result(job_id) except JobResultNotFoundError: - return self.get_exception( + return api.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'JobResultNotFound', job_id ) @@ -488,14 +474,13 @@ def get_job_result(self, request: Union[APIRequest, Any], 'result': job_output } content = render_j2_template( - self.config, 'jobs/results/index.html', + api.config, 'jobs/results/index.html', data, request.locale) return headers, HTTPStatus.OK, content -@pre_process def delete_job( - self, request: Union[APIRequest, Any], job_id + api: API, request: APIRequest, job_id ) -> Tuple[dict, int, str]: """ Delete a process job @@ -505,18 +490,18 @@ def delete_job( :returns: tuple of headers, status code, content """ response_headers = request.get_response_headers( - SYSTEM_LOCALE, **self.api_headers) + SYSTEM_LOCALE, **api.api_headers) try: - success = self.manager.delete_job(job_id) + success = api.manager.delete_job(job_id) except JobNotFoundError: - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, response_headers, request.format, 'NoSuchJob', job_id ) else: if success: http_status = HTTPStatus.OK - jobs_url = f"{self.base_url}/jobs" + jobs_url = f"{api.base_url}/jobs" response = { 'jobID': job_id, @@ -531,7 +516,7 @@ def delete_job( }] } else: - return self.get_exception( + return api.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, request.format, 'InternalError', job_id ) diff --git a/tests/test_api.py b/tests/test_api.py index d7cd57696..a102d8b2d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -46,11 +46,14 @@ API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ ) +from pygeoapi.api.processes import ( + describe_processes, execute_process, delete_job, get_job_result, +) from pygeoapi.util import (yaml_load, get_crs_from_uri, get_api_rules, get_base_url) from .util import (get_test_file_path, mock_request, - mock_flask, mock_starlette) + mock_flask, mock_starlette, mock_api_request) from pygeoapi.models.provider.base import TileMatrixSetEnum @@ -1597,31 +1600,31 @@ def test_get_collection_tiles(config, api_): def test_describe_processes(config, api_): - req = mock_request({'limit': 1}) + req = mock_api_request({'limit': 1}) # Test for description of single processes - rsp_headers, code, response = api_.describe_processes(req) + rsp_headers, code, response = describe_processes(api_, req) data = json.loads(response) assert code == HTTPStatus.OK assert len(data['processes']) == 1 assert len(data['links']) == 3 - req = mock_request() + req = mock_api_request() # Test for undefined process - rsp_headers, code, response = api_.describe_processes(req, 'foo') + rsp_headers, code, response = describe_processes(api_, req, 'foo') data = json.loads(response) assert code == HTTPStatus.NOT_FOUND assert data['code'] == 'NoSuchProcess' # Test for description of all processes - rsp_headers, code, response = api_.describe_processes(req) + rsp_headers, code, response = describe_processes(api_, req) data = json.loads(response) assert code == HTTPStatus.OK assert len(data['processes']) == 2 assert len(data['links']) == 3 # Test for particular, defined process - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') process = json.loads(response) assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] @@ -1638,38 +1641,38 @@ def test_describe_processes(config, api_): assert 'async-execute' in process['jobControlOptions'] # Check HTML response when requested in headers - req = mock_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] # No language requested: return default from YAML assert rsp_headers['Content-Language'] == 'en-US' # Check JSON response when requested in headers - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + req = mock_api_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] assert rsp_headers['Content-Language'] == 'en-US' # Check HTML response when requested with query parameter - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] # No language requested: return default from YAML assert rsp_headers['Content-Language'] == 'en-US' # Check JSON response when requested with query parameter - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] assert rsp_headers['Content-Language'] == 'en-US' # Check JSON response when requested with French language parameter - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] assert rsp_headers['Content-Language'] == 'fr-CA' @@ -1677,25 +1680,25 @@ def test_describe_processes(config, api_): assert process['title'] == 'Bonjour le Monde' # Check JSON response when language requested in headers - req = mock_request(HTTP_ACCEPT_LANGUAGE='fr') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') + req = mock_api_request(HTTP_ACCEPT_LANGUAGE='fr') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] assert rsp_headers['Content-Language'] == 'fr-CA' # Test for undefined process - req = mock_request() - rsp_headers, code, response = api_.describe_processes(req, 'goodbye-world') + req = mock_api_request() + rsp_headers, code, response = describe_processes(api_, req, 'goodbye-world') data = json.loads(response) assert code == HTTPStatus.NOT_FOUND assert data['code'] == 'NoSuchProcess' assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] # Test describe doesn't crash if example is missing - req = mock_request() + req = mock_api_request() processor = api_.manager.get_processor("hello-world") example = processor.metadata.pop("example") - rsp_headers, code, response = api_.describe_processes(req) + rsp_headers, code, response = describe_processes(api_, req) processor.metadata['example'] = example data = json.loads(response) assert code == HTTPStatus.OK @@ -1742,8 +1745,8 @@ def test_execute_process(config, api_): cleanup_jobs = set() # Test posting empty payload to existing process - req = mock_request(data='') - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data='') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') assert rsp_headers['Content-Language'] == 'en-US' data = json.loads(response) @@ -1751,15 +1754,15 @@ def test_execute_process(config, api_): assert 'Location' not in rsp_headers assert data['code'] == 'MissingParameterValue' - req = mock_request(data=req_body_0) - rsp_headers, code, response = api_.execute_process(req, 'foo') + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'foo') data = json.loads(response) assert code == HTTPStatus.NOT_FOUND assert 'Location' not in rsp_headers assert data['code'] == 'NoSuchProcess' - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.OK @@ -1772,8 +1775,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_1) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_1) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.OK @@ -1786,8 +1789,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_2) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_2) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.OK @@ -1797,8 +1800,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_3) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_3) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.OK @@ -1808,8 +1811,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_4) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_4) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.BAD_REQUEST @@ -1818,8 +1821,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_5) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_5) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.BAD_REQUEST assert 'Location' in rsp_headers @@ -1829,8 +1832,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_6) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_6) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.BAD_REQUEST @@ -1841,15 +1844,15 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_0) - rsp_headers, code, response = api_.execute_process(req, 'goodbye-world') + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'goodbye-world') response = json.loads(response) assert code == HTTPStatus.NOT_FOUND assert 'Location' not in rsp_headers assert response['code'] == 'NoSuchProcess' - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') response = json.loads(response) assert code == HTTPStatus.OK @@ -1857,8 +1860,8 @@ def test_execute_process(config, api_): cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) - req = mock_request(data=req_body_1, HTTP_Prefer='respond-async') - rsp_headers, code, response = api_.execute_process(req, 'hello-world') + req = mock_api_request(data=req_body_1, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') assert 'Location' in rsp_headers response = json.loads(response) @@ -1871,7 +1874,7 @@ def test_execute_process(config, api_): # Cleanup time.sleep(2) # Allow time for any outstanding async jobs for _, job_id in cleanup_jobs: - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) assert code == HTTPStatus.OK @@ -1882,9 +1885,8 @@ def _execute_a_job(api_): } } - req = mock_request(data=req_body_sync) - rsp_headers, code, response = api_.execute_process( - req, 'hello-world') + req = mock_api_request(data=req_body_sync) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') data = json.loads(response) assert code == HTTPStatus.OK @@ -1896,8 +1898,7 @@ def _execute_a_job(api_): def test_delete_job(api_): - rsp_headers, code, response = api_.delete_job( - mock_request(), 'does-not-exist') + rsp_headers, code, response = delete_job(api_, mock_api_request(), 'does-not-exist') assert code == HTTPStatus.NOT_FOUND req_body_async = { @@ -1906,43 +1907,42 @@ def test_delete_job(api_): } } job_id = _execute_a_job(api_) - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) assert code == HTTPStatus.OK - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) assert code == HTTPStatus.NOT_FOUND - req = mock_request(data=req_body_async, HTTP_Prefer='respond-async') - rsp_headers, code, response = api_.execute_process( - req, 'hello-world') + req = mock_api_request(data=req_body_async, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') assert code == HTTPStatus.CREATED assert 'Location' in rsp_headers time.sleep(2) # Allow time for async execution to complete job_id = rsp_headers['Location'].split('/')[-1] - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) assert code == HTTPStatus.OK - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) assert code == HTTPStatus.NOT_FOUND def test_get_job_result(api_): - rsp_headers, code, response = api_.get_job_result(mock_request(), + rsp_headers, code, response = get_job_result(api_, mock_api_request(), 'not-exist') assert code == HTTPStatus.NOT_FOUND job_id = _execute_a_job(api_) - rsp_headers, code, response = api_.get_job_result(mock_request(), job_id) + rsp_headers, code, response = get_job_result(api_, mock_api_request(), job_id) # default response is html assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == 'text/html' assert 'Hello Sync Test!' in response - rsp_headers, code, response = api_.get_job_result( - mock_request({'f': 'json'}), job_id, + rsp_headers, code, response = get_job_result(api_, + mock_api_request({'f': 'json'}), job_id, ) assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == 'application/json' diff --git a/tests/util.py b/tests/util.py index 5de974531..e952d61ea 100644 --- a/tests/util.py +++ b/tests/util.py @@ -40,6 +40,8 @@ from werkzeug.wrappers import Request from werkzeug.datastructures import ImmutableMultiDict +from pygeoapi.api import APIRequest + LOGGER = logging.getLogger(__name__) @@ -77,6 +79,24 @@ def mock_request(params: dict = None, data=None, **headers) -> Request: return request +def mock_api_request(params: dict | None = None, data=None, **headers + ) -> APIRequest: + """ + Mocks an APIRequest + + :param params: Optional query parameter dict for the request. + Will be set to {} if omitted. + :param data: Optional data/body to send with the request. + Can be text/bytes or a JSON dictionary. + :param headers: Optional request HTTP headers to set. + :returns: APIRequest instance + """ + return APIRequest.from_flask( + mock_request(params=params, data=data, **headers), + # NOTE: could also read supported_locales from test config + supported_locales=['en-US', 'fr-CA'], + ) + @contextmanager def mock_flask(config_file: str = 'pygeoapi-test-config.yml', openapi_file: str = 'pygeoapi-test-openapi.yml', From b8941f3fe9a3da0c3d5009c4a03d749b1f4c30aa Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 17:14:42 +0100 Subject: [PATCH 04/71] Move openapi definition to processes api --- pygeoapi/api/__init__.py | 7 ++ pygeoapi/api/processes.py | 184 +++++++++++++++++++++++++++++++++++++- pygeoapi/openapi.py | 176 ++---------------------------------- 3 files changed, 196 insertions(+), 171 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index c69039190..bd0655467 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -3825,3 +3825,10 @@ def validate_subset(value: str) -> dict: subsets[subset_name] = list(map(get_typed_value, values)) return subsets + + +# NOTE: this needs to be at the end to avoid import problems +from . import processes + + +ALL_APIS = [processes] diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index d6747fe09..3acb750dc 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -55,7 +55,6 @@ ProcessorExecuteError, ) from pygeoapi.process.manager.base import get_manager -from pygeoapi.openapi import OPENAPI_YAML from . import APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, DATETIME_FORMAT, F_JSONLD @@ -524,3 +523,186 @@ def delete_job( # TODO: this response does not have any headers return {}, http_status, response + +def get_oas_30(cfg: dict, locale_: str): + from pygeoapi.openapi import OPENAPI_YAML + + oas = {'tags': []} + + paths = {} + + process_manager = get_manager(cfg) + + if len(process_manager.processes) > 0: + paths['/processes'] = { + 'get': { + 'summary': 'Processes', + 'description': 'Processes', + 'tags': ['server'], + 'operationId': 'getProcesses', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + LOGGER.debug('setting up processes') + + for k, v in process_manager.processes.items(): + if k.startswith('_'): + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + name = l10n.translate(k, locale_) + p = process_manager.get_processor(k) + md_desc = l10n.translate(p.metadata['description'], locale_) + process_name_path = f'/processes/{name}' + tag = { + 'name': name, + 'description': md_desc, # noqa + 'externalDocs': {} + } + for link in p.metadata.get('links', []): + if link['type'] == 'information': + translated_link = l10n.translate(link, locale_) + tag['externalDocs']['description'] = translated_link[ + 'type'] + tag['externalDocs']['url'] = translated_link['url'] + break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] + + oas['tags'].append(tag) + + paths[process_name_path] = { + 'get': { + 'summary': 'Get process metadata', + 'description': md_desc, + 'tags': [name], + 'operationId': f'describe{name.capitalize()}Process', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths[f'{process_name_path}/execution'] = { + 'post': { + 'summary': f"Process {l10n.translate(p.metadata['title'], locale_)} execution", # noqa + 'description': md_desc, + 'tags': [name], + 'operationId': f'execute{name.capitalize()}Job', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + }, + 'requestBody': { + 'description': 'Mandatory execute request JSON', + 'required': True, + 'content': { + 'application/json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa + } + } + } + } + } + } + if 'example' in p.metadata: + paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa + + name_in_path = { + 'name': 'jobId', + 'in': 'path', + 'description': 'job identifier', + 'required': True, + 'schema': { + 'type': 'string' + } + } + + paths['/jobs'] = { + 'get': { + 'summary': 'Retrieve jobs list', + 'description': 'Retrieve a list of jobs', + 'tags': ['jobs'], + 'operationId': 'getJobs', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths['/jobs/{jobId}'] = { + 'get': { + 'summary': 'Retrieve job details', + 'description': 'Retrieve job details', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJob', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa + } + }, + 'delete': { + 'summary': 'Cancel / delete job', + 'description': 'Cancel / delete job', + 'tags': ['jobs'], + 'parameters': [ + name_in_path + ], + 'operationId': 'deleteJob', + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa + } + }, + } + + paths['/jobs/{jobId}/results'] = { + 'get': { + 'summary': 'Retrieve job results', + 'description': 'Retrive job resiults', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJobResults', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa + } + } + } + + oas['paths'] = paths + + tag = { + 'name': 'jobs', + 'description': 'Process jobs', + } + oas['tags'].insert(1, tag) + + return oas + diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 098a83f79..6829123aa 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -44,6 +44,7 @@ import yaml from pygeoapi import l10n +from pygeoapi.api import ALL_APIS from pygeoapi.models.openapi import OAPIFormat from pygeoapi.plugin import load_plugin from pygeoapi.process.manager.base import get_manager @@ -1170,178 +1171,13 @@ def get_oas_30(cfg): } } - process_manager = get_manager(cfg) - - if len(process_manager.processes) > 0: - paths['/processes'] = { - 'get': { - 'summary': 'Processes', - 'description': 'Processes', - 'tags': ['server'], - 'operationId': 'getProcesses', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } - } - } - LOGGER.debug('setting up processes') - - for k, v in process_manager.processes.items(): - if k.startswith('_'): - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - name = l10n.translate(k, locale_) - p = process_manager.get_processor(k) - md_desc = l10n.translate(p.metadata['description'], locale_) - process_name_path = f'/processes/{name}' - tag = { - 'name': name, - 'description': md_desc, # noqa - 'externalDocs': {} - } - for link in p.metadata.get('links', []): - if link['type'] == 'information': - translated_link = l10n.translate(link, locale_) - tag['externalDocs']['description'] = translated_link[ - 'type'] - tag['externalDocs']['url'] = translated_link['url'] - break - if len(tag['externalDocs']) == 0: - del tag['externalDocs'] - - oas['tags'].append(tag) - - paths[process_name_path] = { - 'get': { - 'summary': 'Get process metadata', - 'description': md_desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Process', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - paths[f'{process_name_path}/execution'] = { - 'post': { - 'summary': f"Process {l10n.translate(p.metadata['title'], locale_)} execution", # noqa - 'description': md_desc, - 'tags': [name], - 'operationId': f'execute{name.capitalize()}Job', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - }, - 'requestBody': { - 'description': 'Mandatory execute request JSON', - 'required': True, - 'content': { - 'application/json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa - } - } - } - } - } - } - if 'example' in p.metadata: - paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa - - name_in_path = { - 'name': 'jobId', - 'in': 'path', - 'description': 'job identifier', - 'required': True, - 'schema': { - 'type': 'string' - } - } - - paths['/jobs'] = { - 'get': { - 'summary': 'Retrieve jobs list', - 'description': 'Retrieve a list of jobs', - 'tags': ['jobs'], - 'operationId': 'getJobs', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - paths['/jobs/{jobId}'] = { - 'get': { - 'summary': 'Retrieve job details', - 'description': 'Retrieve job details', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJob', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - 'delete': { - 'summary': 'Cancel / delete job', - 'description': 'Cancel / delete job', - 'tags': ['jobs'], - 'parameters': [ - name_in_path - ], - 'operationId': 'deleteJob', - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - } - - paths['/jobs/{jobId}/results'] = { - 'get': { - 'summary': 'Retrieve job results', - 'description': 'Retrive job resiults', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJobResults', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - } - } - - tag = { - 'name': 'jobs', - 'description': 'Process jobs', - } - oas['tags'].insert(1, tag) - oas['paths'] = paths + for api in ALL_APIS: + sub_oas = api.get_oas_30(cfg, locale_) + oas['paths'].update(sub_oas['paths']) + oas['tags'].extend(sub_oas['tags']) + if cfg['server'].get('admin', False): schema_dict = get_config_schema() oas['definitions'] = schema_dict['definitions'] From f446b2546676e173e0eea01d38febdbced3a483e Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 17:44:07 +0100 Subject: [PATCH 05/71] Use processes api in flask --- pygeoapi/api/__init__.py | 21 +++++++++++++++++++++ pygeoapi/flask_app.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index bd0655467..fb12ef515 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -214,6 +214,7 @@ def inner(*args): return inner +# TODO: remove this when all functions have been refactored def gzip(func): """ Decorator that compresses the content of an outgoing API result @@ -245,6 +246,26 @@ def inner(*args, **kwargs): return inner +def apply_gzip(headers: dict, content: str | bytes) -> str | bytes: + """ + Compress content if requested in header. + """ + charset = CHARSET[0] + if F_GZIP in headers.get('Content-Encoding', []): + try: + if isinstance(content, bytes): + # bytes means Content-Type needs to be set upstream + content = compress(content) + else: + headers['Content-Type'] = \ + f"{headers['Content-Type']}; charset={charset}" + content = compress(content.encode(charset)) + except TypeError as err: + headers.pop('Content-Encoding') + LOGGER.error(f'Error in compression: {err}') + return content + + class APIRequest: """ Transforms an incoming server-specific Request into an object diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index dc829783d..0cb0ad85c 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -34,9 +34,11 @@ import click -from flask import Flask, Blueprint, make_response, request, send_from_directory +from flask import (Flask, Blueprint, make_response, request, send_from_directory, + Response, Request) -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.processes as processes_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -107,6 +109,7 @@ def schemas(path): mimetype=get_mimetype(basename_)) +# TODO: inline in execute_from_flask when all views have been refactored def get_response(result: tuple): """ Creates a Flask Response object and updates matching headers. @@ -125,6 +128,21 @@ def get_response(result: tuple): return response +def execute_from_flask(api_function, request: Request, *args + ) -> Response: + api_request = APIRequest.from_flask(request, api_.locales) + content: str | bytes + if not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) + # handle jsonld too? + + return get_response((headers, status, content)) + + @BLUEPRINT.route('/') def landing_page(): """ @@ -355,7 +373,7 @@ def get_processes(process_id=None): :returns: HTTP response """ - return get_response(api_.describe_processes(request, process_id)) + return execute_from_flask(processes_api.describe_processes, request, process_id) @BLUEPRINT.route('/jobs') @@ -371,12 +389,12 @@ def get_jobs(job_id=None): """ if job_id is None: - return get_response(api_.get_jobs(request)) + return execute_from_flask(processes_api.get_jobs, request) else: if request.method == 'DELETE': # dismiss job - return get_response(api_.delete_job(request, job_id)) + return execute_from_flask(processes_api.delete_jobs, request) else: # Return status of a specific job - return get_response(api_.get_jobs(request, job_id)) + return execute_from_flask(processes_api.get_jobs, request, job_id) @BLUEPRINT.route('/processes//execution', methods=['POST']) @@ -389,7 +407,7 @@ def execute_process_jobs(process_id): :returns: HTTP response """ - return get_response(api_.execute_process(request, process_id)) + return execute_from_flask(processes_api.execute_process, request, process_id) @BLUEPRINT.route('/jobs//results', @@ -402,7 +420,7 @@ def get_job_result(job_id=None): :returns: HTTP response """ - return get_response(api_.get_job_result(request, job_id)) + return execute_from_flask(processes_api.get_job_result, request, job_id) @BLUEPRINT.route('/jobs//results/', @@ -416,6 +434,7 @@ def get_job_result_resource(job_id, resource): :returns: HTTP response """ + # TODO: this does not seem to exist? return get_response(api_.get_job_result_resource( request, job_id, resource)) From 14d132c625883c454e5fc00c1ec6b3a9ae8775c7 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 17:48:53 +0100 Subject: [PATCH 06/71] Linter --- pygeoapi/api/__init__.py | 18 ++++++------------ pygeoapi/api/processes.py | 10 ++++++++-- pygeoapi/flask_app.py | 10 ++++++---- pygeoapi/openapi.py | 1 - tests/test_api.py | 23 ++++++++++++++--------- tests/util.py | 1 + 6 files changed, 35 insertions(+), 28 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index fb12ef515..5e16dc6e6 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -41,11 +41,10 @@ import asyncio from collections import OrderedDict from copy import deepcopy -from datetime import datetime, timezone +from datetime import datetime from functools import partial from gzip import compress from http import HTTPStatus -import json import logging import re from typing import Any, Tuple, Union, Optional @@ -65,11 +64,6 @@ from pygeoapi.linked_data import (geojson2jsonld, jsonldify, jsonldify_collection) from pygeoapi.log import setup_logger -from pygeoapi.process.base import ( - JobNotFoundError, - JobResultNotFoundError, - ProcessorExecuteError, -) from pygeoapi.process.manager.base import get_manager from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ( @@ -79,11 +73,11 @@ TileMatrixSetEnum) from pygeoapi.models.cql import CQLModel -from pygeoapi.util import (dategetter, RequestedProcessExecutionMode, - DATETIME_FORMAT, UrlPrefetcher, +from pygeoapi.util import (dategetter, + UrlPrefetcher, filter_dict_by_key_value, get_provider_by_type, - get_provider_default, get_typed_value, JobStatus, - json_serial, render_j2_template, str2bool, + get_provider_default, get_typed_value, + render_j2_template, str2bool, TEMPLATES, to_json, get_api_rules, get_base_url, get_crs_from_uri, get_supported_crs_list, modify_pygeofilter, CrsTransformSpec, @@ -3849,7 +3843,7 @@ def validate_subset(value: str) -> dict: # NOTE: this needs to be at the end to avoid import problems -from . import processes +from . import processes # noqa ALL_APIS = [processes] diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 3acb750dc..a470b9f6f 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -56,7 +56,10 @@ ) from pygeoapi.process.manager.base import get_manager -from . import APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, DATETIME_FORMAT, F_JSONLD +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, + DATETIME_FORMAT, F_JSONLD +) LOGGER = logging.getLogger(__name__) @@ -215,6 +218,7 @@ def describe_processes(api: API, request: APIRequest, return headers, HTTPStatus.OK, to_json(response, api.pretty_print) + # TODO: get_jobs doesn't have tests def get_jobs(api: API, request: APIRequest, job_id=None) -> Tuple[dict, int, str]: @@ -316,6 +320,7 @@ def get_jobs(api: API, request: APIRequest, return headers, HTTPStatus.OK, to_json(serialized_jobs, api.pretty_print) + def execute_process(api: API, request: APIRequest, process_id) -> Tuple[dict, int, str]: """ @@ -408,6 +413,7 @@ def execute_process(api: API, request: APIRequest, return headers, http_status, response2 + def get_job_result(api: API, request: APIRequest, job_id) -> Tuple[dict, int, str]: """ @@ -478,6 +484,7 @@ def get_job_result(api: API, request: APIRequest, return headers, HTTPStatus.OK, content + def delete_job( api: API, request: APIRequest, job_id ) -> Tuple[dict, int, str]: @@ -705,4 +712,3 @@ def get_oas_30(cfg: dict, locale_: str): oas['tags'].insert(1, tag) return oas - diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 0cb0ad85c..389c4d2fe 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -34,8 +34,8 @@ import click -from flask import (Flask, Blueprint, make_response, request, send_from_directory, - Response, Request) +from flask import (Flask, Blueprint, make_response, request, + send_from_directory, Response, Request) from pygeoapi.api import API, APIRequest, apply_gzip import pygeoapi.api.processes as processes_api @@ -373,7 +373,8 @@ def get_processes(process_id=None): :returns: HTTP response """ - return execute_from_flask(processes_api.describe_processes, request, process_id) + return execute_from_flask(processes_api.describe_processes, request, + process_id) @BLUEPRINT.route('/jobs') @@ -407,7 +408,8 @@ def execute_process_jobs(process_id): :returns: HTTP response """ - return execute_from_flask(processes_api.execute_process, request, process_id) + return execute_from_flask(processes_api.execute_process, request, + process_id) @BLUEPRINT.route('/jobs//results', diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 6829123aa..e32141f8c 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -47,7 +47,6 @@ from pygeoapi.api import ALL_APIS from pygeoapi.models.openapi import OAPIFormat from pygeoapi.plugin import load_plugin -from pygeoapi.process.manager.base import get_manager from pygeoapi.provider.base import ProviderTypeError, SchemaType from pygeoapi.util import (filter_dict_by_key_value, get_provider_by_type, filter_providers_by_type, to_json, yaml_load, diff --git a/tests/test_api.py b/tests/test_api.py index a102d8b2d..a24557fa7 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1688,7 +1688,8 @@ def test_describe_processes(config, api_): # Test for undefined process req = mock_api_request() - rsp_headers, code, response = describe_processes(api_, req, 'goodbye-world') + rsp_headers, code, response = describe_processes(api_, req, + 'goodbye-world') data = json.loads(response) assert code == HTTPStatus.NOT_FOUND assert data['code'] == 'NoSuchProcess' @@ -1874,7 +1875,8 @@ def test_execute_process(config, api_): # Cleanup time.sleep(2) # Allow time for any outstanding async jobs for _, job_id in cleanup_jobs: - rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + rsp_headers, code, response = delete_job(api_, mock_api_request(), + job_id) assert code == HTTPStatus.OK @@ -1898,7 +1900,8 @@ def _execute_a_job(api_): def test_delete_job(api_): - rsp_headers, code, response = delete_job(api_, mock_api_request(), 'does-not-exist') + rsp_headers, code, response = delete_job(api_, mock_api_request(), + 'does-not-exist') assert code == HTTPStatus.NOT_FOUND req_body_async = { @@ -1930,20 +1933,22 @@ def test_delete_job(api_): def test_get_job_result(api_): - rsp_headers, code, response = get_job_result(api_, mock_api_request(), - 'not-exist') + rsp_headers, code, response = get_job_result( + api_, mock_api_request(), 'not-exist', + ) assert code == HTTPStatus.NOT_FOUND job_id = _execute_a_job(api_) - rsp_headers, code, response = get_job_result(api_, mock_api_request(), job_id) + rsp_headers, code, response = get_job_result(api_, mock_api_request(), + job_id) # default response is html assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == 'text/html' assert 'Hello Sync Test!' in response - rsp_headers, code, response = get_job_result(api_, - mock_api_request({'f': 'json'}), job_id, - ) + rsp_headers, code, response = get_job_result( + api_, mock_api_request({'f': 'json'}), job_id, + ) assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == 'application/json' assert json.loads(response)['value'] == "Hello Sync Test!" diff --git a/tests/util.py b/tests/util.py index e952d61ea..64b0c37ae 100644 --- a/tests/util.py +++ b/tests/util.py @@ -97,6 +97,7 @@ def mock_api_request(params: dict | None = None, data=None, **headers supported_locales=['en-US', 'fr-CA'], ) + @contextmanager def mock_flask(config_file: str = 'pygeoapi-test-config.yml', openapi_file: str = 'pygeoapi-test-openapi.yml', From 15cbd46e7200c775ebc3c24106c09f926c4e5d76 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 17:57:03 +0100 Subject: [PATCH 07/71] Fix import issues --- pygeoapi/api/__init__.py | 14 +++++++------- pygeoapi/api/processes.py | 6 +++--- pygeoapi/openapi.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 5e16dc6e6..2b05f351c 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -186,6 +186,13 @@ DEFAULT_STORAGE_CRS = DEFAULT_CRS +def all_apis(): + # NOTE: this is a function and not a constant to avoid import loops + from . import processes + + return [processes] + + def pre_process(func): """ Decorator that transforms an incoming Request instance specific to the @@ -3840,10 +3847,3 @@ def validate_subset(value: str) -> dict: subsets[subset_name] = list(map(get_typed_value, values)) return subsets - - -# NOTE: this needs to be at the end to avoid import problems -from . import processes # noqa - - -ALL_APIS = [processes] diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index a470b9f6f..0b07e47bd 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -48,7 +48,7 @@ from pygeoapi import l10n from pygeoapi.util import ( json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, - to_json) + to_json, DATETIME_FORMAT) from pygeoapi.process.base import ( JobNotFoundError, JobResultNotFoundError, @@ -57,10 +57,10 @@ from pygeoapi.process.manager.base import get_manager from . import ( - APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, - DATETIME_FORMAT, F_JSONLD + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, ) + LOGGER = logging.getLogger(__name__) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index e32141f8c..899d0e9ee 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -44,7 +44,7 @@ import yaml from pygeoapi import l10n -from pygeoapi.api import ALL_APIS +from pygeoapi.api import all_apis from pygeoapi.models.openapi import OAPIFormat from pygeoapi.plugin import load_plugin from pygeoapi.provider.base import ProviderTypeError, SchemaType @@ -1172,7 +1172,7 @@ def get_oas_30(cfg): oas['paths'] = paths - for api in ALL_APIS: + for api in all_apis(): sub_oas = api.get_oas_30(cfg, locale_) oas['paths'].update(sub_oas['paths']) oas['tags'].extend(sub_oas['tags']) From 8083243f714478b56b9d21ab09cff8c35d0326ea Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 11 Mar 2024 19:22:04 +0100 Subject: [PATCH 08/71] Allow calling refactored views from starlette --- pygeoapi/api/__init__.py | 14 ++++++++++-- pygeoapi/starlette_app.py | 48 +++++++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 2b05f351c..af7570703 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -51,7 +51,6 @@ import urllib.parse from dateutil.parser import parse as dateparse -import flask from pygeofilter.parsers.ecql import parse as parse_ecql_text from pygeofilter.parsers.cql_json import parse as parse_cql_json from pyproj.exceptions import CRSError @@ -364,6 +363,8 @@ def __init__(self, request, supported_locales): # Get received headers self._headers = self.get_request_headers(request.headers) + # TODO: remove this after all views have been refactored (only used + # in pre_process) @classmethod def with_data(cls, request, supported_locales) -> 'APIRequest': """ @@ -401,7 +402,7 @@ def with_data(cls, request, supported_locales) -> 'APIRequest': return api_req @classmethod - def from_flask(cls, request: flask.Request, supported_locales + def from_flask(cls, request, supported_locales ) -> 'APIRequest': """ Factory class similar to with_data, but only for flask requests @@ -410,6 +411,15 @@ def from_flask(cls, request: flask.Request, supported_locales api_req._data = request.data return api_req + @classmethod + async def from_starlette(cls, request, supported_locales) -> 'APIRequest': + """ + Factory class similar to with_data, but only for flask requests + """ + api_req = cls(request, supported_locales) + api_req._data = await request.body() + return api_req + @staticmethod def _get_params(request): """ diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 8264566dd..d82340763 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -50,7 +50,8 @@ ) import uvicorn -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.processes as processes_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -113,10 +114,12 @@ async def get_response( """ loop = asyncio.get_running_loop() - result = await loop.run_in_executor( + headers, status, content = await loop.run_in_executor( None, call_api_threadsafe, loop, api_call, *args) + return _to_response(headers, status, content) - headers, status, content = result + +def _to_response(headers, status, content): if headers['Content-Type'] == 'text/html': response = HTMLResponse(content=content, status_code=status) else: @@ -130,6 +133,27 @@ async def get_response( return response +async def execute_from_starlette(api_function, request: Request, *args + ) -> Response: + api_request = await APIRequest.from_starlette(request, api_.locales) + content: str | bytes + if not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + loop = asyncio.get_running_loop() + headers, status, content = await loop.run_in_executor( + None, call_api_threadsafe, loop, api_function, + api_, api_request, *args) + # NOTE: that gzip currently doesn't work in starlette + # https://github.com/geopython/pygeoapi/issues/1591 + content = apply_gzip(headers, content) + + response = _to_response(headers, status, content) + + return response + + async def landing_page(request: Request): """ OGC API landing page endpoint @@ -389,7 +413,8 @@ async def get_processes(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.describe_processes, request, process_id) + return await execute_from_starlette(processes_api.describe_processes, + request, process_id) async def get_jobs(request: Request, job_id=None): @@ -406,12 +431,14 @@ async def get_jobs(request: Request, job_id=None): job_id = request.path_params['job_id'] if job_id is None: # list of submit job - return await get_response(api_.get_jobs, request) + return await execute_from_starlette(processes_api.get_jobs, request) else: # get or delete job if request.method == 'DELETE': - return await get_response(api_.delete_job, job_id) + return await execute_from_starlette(processes_api.delete_job, + request, job_id) else: # Return status of a specific job - return await get_response(api_.get_jobs, request, job_id) + return await execute_from_starlette(processes_api.get_jobs, + request, job_id) async def execute_process_jobs(request: Request, process_id=None): @@ -427,7 +454,8 @@ async def execute_process_jobs(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.execute_process, request, process_id) + return await execute_from_starlette(processes_api.execute_process, + request, process_id) async def get_job_result(request: Request, job_id=None): @@ -443,7 +471,8 @@ async def get_job_result(request: Request, job_id=None): if 'job_id' in request.path_params: job_id = request.path_params['job_id'] - return await get_response(api_.get_job_result, request, job_id) + return await execute_from_starlette(processes_api.get_job_result, + request, job_id) async def get_job_result_resource(request: Request, @@ -463,6 +492,7 @@ async def get_job_result_resource(request: Request, if 'resource' in request.path_params: resource = request.path_params['resource'] + # TODO: this api function currently doesn't exist return await get_response( api_.get_job_result_resource, request, job_id, resource) From eb5760d0e355300e989a59f1dac747781a1119fc Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 10:08:32 +0100 Subject: [PATCH 09/71] Allow calling refactored views from django --- pygeoapi/api/__init__.py | 14 +++++++----- pygeoapi/django_/views.py | 46 ++++++++++++++++++++++++++------------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index af7570703..642e27add 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -404,22 +404,26 @@ def with_data(cls, request, supported_locales) -> 'APIRequest': @classmethod def from_flask(cls, request, supported_locales ) -> 'APIRequest': - """ - Factory class similar to with_data, but only for flask requests - """ + """Factory class similar to with_data, but only for flask requests""" api_req = cls(request, supported_locales) api_req._data = request.data return api_req @classmethod async def from_starlette(cls, request, supported_locales) -> 'APIRequest': - """ - Factory class similar to with_data, but only for flask requests + """Factory class similar to with_data, but only for starlette requests """ api_req = cls(request, supported_locales) api_req._data = await request.body() return api_req + @classmethod + def from_django(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for django requests""" + api_req = cls(request, supported_locales) + api_req._data = request.body + return api_req + @staticmethod def _get_params(request): """ diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 3c235f172..6088515b4 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -40,7 +40,8 @@ from django.conf import settings from django.http import HttpRequest, HttpResponse -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.processes as processes_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -368,10 +369,7 @@ def processes(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'describe_processes', process_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.describe_processes, request, process_id) def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: @@ -384,11 +382,7 @@ def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: :returns: Django HTTP response """ - - response_ = _feed_response(request, 'get_jobs', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_jobs, request, job_id) def job_results(request: HttpRequest, @@ -402,10 +396,7 @@ def job_results(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_job_result', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_job_result, request, job_id) def job_results_resource(request: HttpRequest, process_id: str, job_id: str, @@ -420,6 +411,7 @@ def job_results_resource(request: HttpRequest, process_id: str, job_id: str, :returns: Django HTTP response """ + # TODO: this api method does not exist response_ = _feed_response( request, 'get_job_result_resource', @@ -551,6 +543,7 @@ def admin_config_resource(request: HttpRequest, resource_id) +# TODO: remove this when all views have been refactored def _feed_response(request: HttpRequest, api_definition: str, *args, **kwargs) -> Tuple[Dict, int, str]: """Use pygeoapi api to process the input request""" @@ -566,8 +559,31 @@ def _feed_response(request: HttpRequest, api_definition: str, return api(request, *args, **kwargs) +def execute_from_django(api_function, request: HttpRequest, *args + ) -> HttpResponse: + + api_: API | "Admin" + if settings.PYGEOAPI_CONFIG['server'].get('admin'): # noqa + from pygeoapi.admin import Admin + api_ = Admin(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) + else: + api_ = API(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) + + api_request = APIRequest.from_django(request, api_.locales) + content: str | bytes + if not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) + + return _to_django_response(headers, status, content) + + +# TODO: inline this to execute_from_django after refactoring def _to_django_response(headers: Mapping, status_code: int, - content: str) -> HttpResponse: + content: str | bytes) -> HttpResponse: """Convert API payload to a django response""" response = HttpResponse(content, status=status_code) From 58b1f3c108b90f1fb12ae25932c3afebe97b63a4 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 10:19:24 +0100 Subject: [PATCH 10/71] Linter --- pygeoapi/django_/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 6088515b4..0adc010e8 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -369,7 +369,8 @@ def processes(request: HttpRequest, :returns: Django HTTP response """ - return execute_from_django(processes_api.describe_processes, request, process_id) + return execute_from_django(processes_api.describe_processes, request, + process_id) def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: From 8bd674a0bfe0d7d6e9f0c4f8d06f92510624b4be Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 10:30:50 +0100 Subject: [PATCH 11/71] Move edr api to own file --- pygeoapi/api/__init__.py | 149 ------------- pygeoapi/api/environmental_data_retrieval.py | 207 +++++++++++++++++++ 2 files changed, 207 insertions(+), 149 deletions(-) create mode 100644 pygeoapi/api/environmental_data_retrieval.py diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 642e27add..adaf68e9b 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -55,8 +55,6 @@ from pygeofilter.parsers.cql_json import parse as parse_cql_json from pyproj.exceptions import CRSError import pytz -from shapely.errors import WKTReadingError -from shapely.wkt import loads as shapely_loads from pygeoapi import __version__, l10n from pygeoapi.formatter.base import FormatterSerializationError @@ -3222,153 +3220,6 @@ def get_collection_map_legend( return headers, HTTPStatus.BAD_REQUEST, to_json( data, self.pretty_print) - @gzip - @pre_process - def get_collection_edr_query( - self, request: Union[APIRequest, Any], - dataset, instance, query_type, - location_id=None) -> Tuple[dict, int, str]: - """ - Queries collection EDR - - :param request: APIRequest instance with query params - :param dataset: dataset name - :param instance: instance name - :param query_type: EDR query type - :param location_id: location id of a /location/ query - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - headers = request.get_response_headers(self.default_locale, - **self.api_headers) - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing parameter_names parameter') - parameternames = request.params.get('parameter_names') or [] - if isinstance(parameternames, str): - parameternames = parameternames.split(',') - - bbox = None - if query_type in ['cube', 'locations']: - LOGGER.debug('Processing cube bbox') - try: - bbox = validate_bbox(request.params.get('bbox')) - if not bbox and query_type == 'cube': - raise ValueError('bbox parameter required by cube queries') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - LOGGER.debug('Processing coords parameter') - wkt = request.params.get('coords') - - if wkt: - try: - wkt = shapely_loads(wkt) - except WKTReadingError: - msg = 'invalid coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif query_type not in ['cube', 'locations']: - msg = 'missing coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - within = within_units = None - if query_type == 'radius': - LOGGER.debug('Processing within / within-units parameters') - within = request.params.get('within') - within_units = request.params.get('within-units') - - LOGGER.debug('Processing z parameter') - z = request.params.get('z') - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - collections[dataset]['providers'], 'edr')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if instance is not None and not p.get_instance(instance): - msg = 'Invalid instance identifier' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - if query_type not in p.get_query_types(): - msg = 'Unsupported query type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if parameternames and not any((fld in parameternames) - for fld in p.get_fields().keys()): - msg = 'Invalid parameter_names' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - query_args = dict( - query_type=query_type, - instance=instance, - format_=request.format, - datetime_=datetime_, - select_properties=parameternames, - wkt=wkt, - z=z, - bbox=bbox, - within=within, - within_units=within_units, - limit=int(self.config['server']['limit']), - location_id=location_id, - ) - - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/edr/query.html', data, - self.default_locale) - else: - content = to_json(data, self.pretty_print) - - return headers, HTTPStatus.OK, content - @gzip @pre_process @jsonldify diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py new file mode 100644 index 000000000..a59788089 --- /dev/null +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -0,0 +1,207 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime, timezone +import logging +from http import HTTPStatus +import json +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.util import ( + json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, + to_json, DATETIME_FORMAT) + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, +) + + +LOGGER = logging.getLogger(__name__) + + +@gzip +@pre_process +def get_collection_edr_query( + self, request: Union[APIRequest, Any], + dataset, instance, query_type, + location_id=None) -> Tuple[dict, int, str]: + """ + Queries collection EDR + + :param request: APIRequest instance with query params + :param dataset: dataset name + :param instance: instance name + :param query_type: EDR query type + :param location_id: location id of a /location/ query + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return self.get_format_exception(request) + headers = request.get_response_headers(self.default_locale, + **self.api_headers) + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing parameter_names parameter') + parameternames = request.params.get('parameter_names') or [] + if isinstance(parameternames, str): + parameternames = parameternames.split(',') + + bbox = None + if query_type in ['cube', 'locations']: + LOGGER.debug('Processing cube bbox') + try: + bbox = validate_bbox(request.params.get('bbox')) + if not bbox and query_type == 'cube': + raise ValueError('bbox parameter required by cube queries') + except ValueError as err: + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + LOGGER.debug('Processing coords parameter') + wkt = request.params.get('coords') + + if wkt: + try: + wkt = shapely_loads(wkt) + except WKTReadingError: + msg = 'invalid coords parameter' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif query_type not in ['cube', 'locations']: + msg = 'missing coords parameter' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + within = within_units = None + if query_type == 'radius': + LOGGER.debug('Processing within / within-units parameters') + within = request.params.get('within') + within_units = request.params.get('within-units') + + LOGGER.debug('Processing z parameter') + z = request.params.get('z') + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + collections[dataset]['providers'], 'edr')) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if instance is not None and not p.get_instance(instance): + msg = 'Invalid instance identifier' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + if query_type not in p.get_query_types(): + msg = 'Unsupported query type' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if parameternames and not any((fld in parameternames) + for fld in p.get_fields().keys()): + msg = 'Invalid parameter_names' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + query_args = dict( + query_type=query_type, + instance=instance, + format_=request.format, + datetime_=datetime_, + select_properties=parameternames, + wkt=wkt, + z=z, + bbox=bbox, + within=within, + within_units=within_units, + limit=int(self.config['server']['limit']), + location_id=location_id, + ) + + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, + 'collections/edr/query.html', data, + self.default_locale) + else: + content = to_json(data, self.pretty_print) + + return headers, HTTPStatus.OK, content + From ec3a49e9c9b859ff5dcb73e6316653029be5b3f8 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 10:46:58 +0100 Subject: [PATCH 12/71] Adapt edr api to new style --- pygeoapi/api/environmental_data_retrieval.py | 61 ++++++----- pygeoapi/django_/views.py | 8 +- pygeoapi/flask_app.py | 8 +- pygeoapi/starlette_app.py | 5 +- tests/test_api.py | 103 ++++++++++--------- 5 files changed, 93 insertions(+), 92 deletions(-) diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index a59788089..4ec4e21cf 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -38,32 +38,32 @@ # ================================================================= -from copy import deepcopy -from datetime import datetime, timezone import logging from http import HTTPStatus -import json from typing import Tuple -from pygeoapi import l10n +from shapely.errors import WKTReadingError +from shapely.wkt import loads as shapely_loads + +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ProviderGenericError from pygeoapi.util import ( - json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, - to_json, DATETIME_FORMAT) + get_provider_by_type, render_j2_template, to_json, + filter_dict_by_key_value, +) from . import ( - APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, + APIRequest, API, F_HTML, validate_datetime, validate_bbox ) LOGGER = logging.getLogger(__name__) -@gzip -@pre_process def get_collection_edr_query( - self, request: Union[APIRequest, Any], - dataset, instance, query_type, - location_id=None) -> Tuple[dict, int, str]: + api: API, request: APIRequest, dataset, instance, query_type, + location_id=None +) -> Tuple[dict, int, str]: """ Queries collection EDR @@ -77,15 +77,15 @@ def get_collection_edr_query( """ if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - headers = request.get_response_headers(self.default_locale, - **self.api_headers) - collections = filter_dict_by_key_value(self.config['resources'], + return api.get_format_exception(request) + headers = request.get_response_headers(api.default_locale, + **api.api_headers) + collections = filter_dict_by_key_value(api.config['resources'], 'type', 'collection') if dataset not in collections.keys(): msg = 'Collection not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) LOGGER.debug('Processing query parameters') @@ -97,7 +97,7 @@ def get_collection_edr_query( datetime_) except ValueError as err: msg = str(err) - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) @@ -114,7 +114,7 @@ def get_collection_edr_query( if not bbox and query_type == 'cube': raise ValueError('bbox parameter required by cube queries') except ValueError as err: - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', str(err)) @@ -126,12 +126,12 @@ def get_collection_edr_query( wkt = shapely_loads(wkt) except WKTReadingError: msg = 'invalid coords parameter' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) elif query_type not in ['cube', 'locations']: msg = 'missing coords parameter' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) @@ -150,26 +150,26 @@ def get_collection_edr_query( collections[dataset]['providers'], 'edr')) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) if instance is not None and not p.get_instance(instance): msg = 'Invalid instance identifier' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) if query_type not in p.get_query_types(): msg = 'Unsupported query type' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) if parameternames and not any((fld in parameternames) for fld in p.get_fields().keys()): msg = 'Invalid parameter_names' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) @@ -184,7 +184,7 @@ def get_collection_edr_query( bbox=bbox, within=within, within_units=within_units, - limit=int(self.config['server']['limit']), + limit=int(api.config['server']['limit']), location_id=location_id, ) @@ -192,16 +192,15 @@ def get_collection_edr_query( data = p.query(**query_args) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'collections/edr/query.html', data, - self.default_locale) + api.default_locale) else: - content = to_json(data, self.pretty_print) + content = to_json(data, api.pretty_print) return headers, HTTPStatus.OK, content - diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 0adc010e8..c5a3aeabc 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -42,6 +42,7 @@ from pygeoapi.api import API, APIRequest, apply_gzip import pygeoapi.api.processes as processes_api +import pygeoapi.api.environmental_data_retrieval as edr_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -444,17 +445,14 @@ def get_collection_edr_query( query_type = 'locations' else: query_type = request.path.split('/')[-1] - response_ = _feed_response( + return execute_from_django( + edr_api.get.get_collection_edr_query, request, - 'get_collection_edr_query', collection_id, instance_id, query_type, location_id ) - response = _to_django_response(*response_) - - return response def stac_catalog_root(request: HttpRequest) -> HttpResponse: diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 389c4d2fe..4339efac9 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -39,6 +39,7 @@ from pygeoapi.api import API, APIRequest, apply_gzip import pygeoapi.api.processes as processes_api +import pygeoapi.api.environmental_data_retrieval as edr_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -473,9 +474,10 @@ def get_collection_edr_query(collection_id, instance_id=None, else: query_type = request.path.split('/')[-1] - return get_response(api_.get_collection_edr_query(request, collection_id, - instance_id, query_type, - location_id)) + return execute_from_flask( + edr_api.get_collection_edr_query, request, collection_id, instance_id, + query_type, location_id, + ) @BLUEPRINT.route('/stac') diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index d82340763..bd2665da7 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -52,6 +52,7 @@ from pygeoapi.api import API, APIRequest, apply_gzip import pygeoapi.api.processes as processes_api +import pygeoapi.api.environmental_data_retrieval as edr_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -514,8 +515,8 @@ async def get_collection_edr_query(request: Request, collection_id=None, instanc instance_id = request.path_params['instance_id'] query_type = request["path"].split('/')[-1] # noqa - return await get_response( - api_.get_collection_edr_query, request, collection_id, + return await execute_from_starlette( + edr_api.get_collection_edr_query, request, collection_id, instance_id, query_type ) diff --git a/tests/test_api.py b/tests/test_api.py index a24557fa7..c11c4b178 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -46,6 +46,7 @@ API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ ) +from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query from pygeoapi.api.processes import ( describe_processes, execute_process, delete_job, get_job_result, ) @@ -1956,7 +1957,7 @@ def test_get_job_result(api_): def test_get_collection_edr_query(config, api_): # edr resource - req = mock_request() + req = mock_api_request() rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') collection = json.loads(response) parameter_names = list(collection['parameter_names'].keys()) @@ -1965,34 +1966,34 @@ def test_get_collection_edr_query(config, api_): assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] # no coords parameter - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.BAD_REQUEST # bad query type - req = mock_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'corridor') + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'corridor') assert code == HTTPStatus.BAD_REQUEST # bad coords parameter - req = mock_request({'coords': 'gah'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + req = mock_api_request({'coords': 'gah'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.BAD_REQUEST # bad parameter_names parameter - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'parameter_names': 'bad' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.BAD_REQUEST # all parameters - req = mock_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK data = json.loads(response) @@ -2013,11 +2014,11 @@ def test_get_collection_edr_query(config, api_): assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] # single parameter - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'parameter_names': 'SST' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK data = json.loads(response) @@ -2026,21 +2027,21 @@ def test_get_collection_edr_query(config, api_): assert list(data['parameters'].keys())[0] == 'SST' # Zulu time zone - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK # bounded date range - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'datetime': '2000-01-17/2000-06-16' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK data = json.loads(response) @@ -2051,12 +2052,12 @@ def test_get_collection_edr_query(config, api_): assert time_dict['num'] == 5 # unbounded date range - start - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'datetime': '../2000-06-16' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK data = json.loads(response) @@ -2067,12 +2068,12 @@ def test_get_collection_edr_query(config, api_): assert time_dict['num'] == 6 # unbounded date range - end - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'datetime': '2000-06-16/..' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK data = json.loads(response) @@ -2083,60 +2084,60 @@ def test_get_collection_edr_query(config, api_): assert time_dict['num'] == 7 # some data - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.OK # no data - req = mock_request({ + req = mock_api_request({ 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.NO_CONTENT # position no coords - req = mock_request({ + req = mock_api_request({ 'datetime': '2000-01-17' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') assert code == HTTPStatus.BAD_REQUEST # cube bbox parameter 4 dimensional - req = mock_request({ + req = mock_api_request({ 'bbox': '0,0,10,10' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') assert code == HTTPStatus.OK # cube bad bbox parameter - req = mock_request({ + req = mock_api_request({ 'bbox': '0,0,10' }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') assert code == HTTPStatus.BAD_REQUEST # cube no bbox parameter - req = mock_request({}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') + req = mock_api_request({}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') assert code == HTTPStatus.BAD_REQUEST # cube decreasing latitude coords and S3 - req = mock_request({ + req = mock_api_request({ 'bbox': '-100,40,-99,45', 'parameter_names': 'tmn', 'datetime': '1994-01-01/1994-12-31', }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'usgs-prism', None, 'cube') + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'usgs-prism', None, 'cube') assert code == HTTPStatus.OK From 91610d09195b0f487074b8c7e713da7e790524ad Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 10:57:31 +0100 Subject: [PATCH 13/71] Fix typo in django views --- pygeoapi/django_/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index c5a3aeabc..973746cd6 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -446,7 +446,7 @@ def get_collection_edr_query( else: query_type = request.path.split('/')[-1] return execute_from_django( - edr_api.get.get_collection_edr_query, + edr_api.get_collection_edr_query, request, collection_id, instance_id, From e4bdddd5a04149fe34b8390b2d9d5fddae1e7179 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 11:18:29 +0100 Subject: [PATCH 14/71] Move maps api to own file --- pygeoapi/api/__init__.py | 127 -------------------------- pygeoapi/api/maps.py | 190 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 127 deletions(-) create mode 100644 pygeoapi/api/maps.py diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index adaf68e9b..f6238b4fc 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -3032,133 +3032,6 @@ def get_collection_tiles_metadata( else: return headers, HTTPStatus.OK, tiles_metadata - @gzip - @pre_process - def get_collection_map(self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - query_args = { - 'crs': 'CRS84' - } - - format_ = request.format or 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - query_args['format_'] = request.params.get('f', 'png') - query_args['style'] = style - query_args['crs'] = request.params.get('bbox-crs', 4326) - query_args['transparent'] = request.params.get('transparent', True) - - try: - query_args['width'] = int(request.params.get('width', 500)) - query_args['height'] = int(request.params.get('height', 300)) - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid width/height' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing bbox parameter') - try: - bbox = request.params.get('bbox').split(',') - if len(bbox) != 4: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values should be minx,miny,maxx,maxy' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - except AttributeError: - bbox = self.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa - try: - query_args['bbox'] = [float(c) for c in bbox] - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values must be numbers' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - query_args['datetime_'] = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Generating map') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ in [None, 'html']: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - @gzip def get_collection_map_legend( self, request: Union[APIRequest, Any], diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py new file mode 100644 index 000000000..95eb18bbf --- /dev/null +++ b/pygeoapi/api/maps.py @@ -0,0 +1,190 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from shapely.errors import WKTReadingError +from shapely.wkt import loads as shapely_loads + +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + get_provider_by_type, render_j2_template, to_json, + filter_dict_by_key_value, +) + +from . import ( + APIRequest, API, F_HTML, validate_datetime, validate_bbox +) + + +LOGGER = logging.getLogger(__name__) + + +@gzip +@pre_process +def get_collection_map(self, request: Union[APIRequest, Any], + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + query_args = { + 'crs': 'CRS84' + } + + format_ = request.format or 'png' + headers = request.get_response_headers(**self.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + self.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, self.pretty_print) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + query_args['format_'] = request.params.get('f', 'png') + query_args['style'] = style + query_args['crs'] = request.params.get('bbox-crs', 4326) + query_args['transparent'] = request.params.get('transparent', True) + + try: + query_args['width'] = int(request.params.get('width', 500)) + query_args['height'] = int(request.params.get('height', 300)) + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid width/height' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, self.pretty_print) + + LOGGER.debug('Processing bbox parameter') + try: + bbox = request.params.get('bbox').split(',') + if len(bbox) != 4: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values should be minx,miny,maxx,maxy' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, self.pretty_print) + except AttributeError: + bbox = self.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa + try: + query_args['bbox'] = [float(c) for c in bbox] + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values must be numbers' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, self.pretty_print) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + query_args['datetime_'] = validate_datetime( + self.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Generating map') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ in [None, 'html']: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, self.pretty_print) + + From 855b2ac2299db54bd3418ae378c40d5bf712221e Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Tue, 12 Mar 2024 12:33:39 +0100 Subject: [PATCH 15/71] Adapt maps api to new style --- pygeoapi/api/maps.py | 46 ++++++++++++++------------------------- pygeoapi/django_/views.py | 12 +++++----- pygeoapi/flask_app.py | 15 +++++-------- pygeoapi/starlette_app.py | 8 ++++--- tests/test_api.py | 11 +++++----- 5 files changed, 37 insertions(+), 55 deletions(-) diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index 95eb18bbf..2c9626bdb 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -42,27 +42,18 @@ from http import HTTPStatus from typing import Tuple -from shapely.errors import WKTReadingError -from shapely.wkt import loads as shapely_loads -from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.plugin import load_plugin from pygeoapi.provider.base import ProviderGenericError -from pygeoapi.util import ( - get_provider_by_type, render_j2_template, to_json, - filter_dict_by_key_value, -) +from pygeoapi.util import get_provider_by_type, to_json -from . import ( - APIRequest, API, F_HTML, validate_datetime, validate_bbox -) +from . import APIRequest, API, validate_datetime LOGGER = logging.getLogger(__name__) -@gzip -@pre_process -def get_collection_map(self, request: Union[APIRequest, Any], +def get_collection_map(api: API, request: APIRequest, dataset, style=None) -> Tuple[dict, int, str]: """ Returns a subset of a collection map @@ -74,21 +65,18 @@ def get_collection_map(self, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) - query_args = { 'crs': 'CRS84' } format_ = request.format or 'png' - headers = request.get_response_headers(**self.api_headers) + headers = request.get_response_headers(**api.api_headers) LOGGER.debug('Processing query parameters') LOGGER.debug('Loading provider') try: collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') + api.config['resources'][dataset]['providers'], 'map') p = load_plugin('provider', collection_def) except KeyError: @@ -99,10 +87,10 @@ def get_collection_map(self, request: Union[APIRequest, Any], headers['Content-type'] = 'application/json' LOGGER.error(exception) return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) + exception, api.pretty_print) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) @@ -122,7 +110,7 @@ def get_collection_map(self, request: Union[APIRequest, Any], headers['Content-type'] = 'application/json' LOGGER.error(exception) return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) + exception, api.pretty_print) LOGGER.debug('Processing bbox parameter') try: @@ -135,9 +123,9 @@ def get_collection_map(self, request: Union[APIRequest, Any], headers['Content-type'] = 'application/json' LOGGER.error(exception) return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) + exception, api.pretty_print) except AttributeError: - bbox = self.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa + bbox = api.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa try: query_args['bbox'] = [float(c) for c in bbox] except ValueError: @@ -148,16 +136,16 @@ def get_collection_map(self, request: Union[APIRequest, Any], headers['Content-type'] = 'application/json' LOGGER.error(exception) return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) + exception, api.pretty_print) LOGGER.debug('Processing datetime parameter') datetime_ = request.params.get('datetime') try: query_args['datetime_'] = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) + api.config['resources'][dataset]['extents'], datetime_) except ValueError as err: msg = str(err) - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) @@ -166,7 +154,7 @@ def get_collection_map(self, request: Union[APIRequest, Any], data = p.query(**query_args) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) @@ -185,6 +173,4 @@ def get_collection_map(self, request: Union[APIRequest, Any], } LOGGER.error(exception) return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - + data, api.pretty_print) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 973746cd6..7c272b92f 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -41,8 +41,9 @@ from django.http import HttpRequest, HttpResponse from pygeoapi.api import API, APIRequest, apply_gzip -import pygeoapi.api.processes as processes_api import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -208,12 +209,9 @@ def collection_map(request: HttpRequest, collection_id: str): :returns: HTTP response """ - - response_ = _feed_response(request, 'get_collection_map', collection_id) - - response = _to_django_response(*response_) - - return response + return execute_from_django( + maps_api.get_collection_map, request, collection_id + ) def collection_style_map(request: HttpRequest, collection_id: str, diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 4339efac9..ca17eb659 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -38,8 +38,9 @@ send_from_directory, Response, Request) from pygeoapi.api import API, APIRequest, apply_gzip -import pygeoapi.api.processes as processes_api import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -353,15 +354,9 @@ def collection_map(collection_id, style_id=None): :returns: HTTP response """ - headers, status_code, content = api_.get_collection_map( - request, collection_id, style_id) - - response = make_response(content, status_code) - - if headers: - response.headers = headers - - return response + return execute_from_flask( + maps_api.get_collection_map, request, collection_id, style_id + ) @BLUEPRINT.route('/processes') diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index bd2665da7..757ceb343 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -51,8 +51,9 @@ import uvicorn from pygeoapi.api import API, APIRequest, apply_gzip -import pygeoapi.api.processes as processes_api import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -398,8 +399,9 @@ async def collection_map(request: Request, collection_id, style_id=None): if 'style_id' in request.path_params: style_id = request.path_params['style_id'] - return await get_response( - api_.get_collection_map, request, collection_id, style_id) + return await execute_from_starlette( + maps_api.get_collection_map, request, collection_id, style_id, + ) async def get_processes(request: Request, process_id=None): diff --git a/tests/test_api.py b/tests/test_api.py index c11c4b178..a83a5d99e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -47,6 +47,7 @@ validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ ) from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query +from pygeoapi.api.maps import get_collection_map from pygeoapi.api.processes import ( describe_processes, execute_process, delete_job, get_job_result, ) @@ -1569,13 +1570,13 @@ def test_get_collection_coverage(config, api_): def test_get_collection_map(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_map(req, 'notfound') + req = mock_api_request() + rsp_headers, code, response = get_collection_map(api_, req, 'notfound') assert code == HTTPStatus.NOT_FOUND - req = mock_request() - rsp_headers, code, response = api_.get_collection_map( - req, 'mapserver_world_map') + req = mock_api_request() + rsp_headers, code, response = get_collection_map( + api_, req, 'mapserver_world_map') assert code == HTTPStatus.OK assert isinstance(response, bytes) assert response[1:4] == b'PNG' From d4e8f1779960efee9b0f9ea80b70a65481d14834 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 11:54:00 +0100 Subject: [PATCH 16/71] Move edr openapi to edr api file --- pygeoapi/api/__init__.py | 4 +- pygeoapi/api/environmental_data_retrieval.py | 120 ++++++++++++++++++- pygeoapi/openapi.py | 103 ---------------- 3 files changed, 120 insertions(+), 107 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index f6238b4fc..99e2f8c9f 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -185,9 +185,9 @@ def all_apis(): # NOTE: this is a function and not a constant to avoid import loops - from . import processes + from . import processes, environmental_data_retrieval - return [processes] + return [processes, environmental_data_retrieval] def pre_process(func): diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index 4ec4e21cf..d708cc0b2 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -48,8 +48,8 @@ from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ProviderGenericError from pygeoapi.util import ( - get_provider_by_type, render_j2_template, to_json, - filter_dict_by_key_value, + filter_providers_by_type, get_provider_by_type, render_j2_template, + to_json, filter_dict_by_key_value, ) from . import ( @@ -204,3 +204,119 @@ def get_collection_edr_query( content = to_json(data, api.pretty_print) return headers, HTTPStatus.OK, content + + +def get_oas_30(cfg: dict, locale: str) -> dict: + from pygeoapi.openapi import OPENAPI_YAML + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in collections.items(): + LOGGER.debug('setting up edr endpoints') + edr_extension = filter_providers_by_type( + collections[k]['providers'], 'edr') + + collection_name_path = f'/collections/{k}' + + if edr_extension: + ep = load_plugin('provider', edr_extension) + + edr_query_endpoints = [] + + for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/{qt}', + 'qt': qt, + 'op_id': f'query{qt.capitalize()}{k.capitalize()}' + }) + if ep.instances: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa + 'qt': qt, + 'op_id': f'query{qt.capitalize()}Instance{k.capitalize()}' # noqa + }) + + for eqe in edr_query_endpoints: + if eqe['qt'] == 'cube': + spatial_parameter = 'bbox' + else: + spatial_parameter = f"{eqe['qt']}Coords" + paths[eqe['path']] = { + 'get': { + 'summary': f"query {v['description']} by {eqe['qt']}", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': eqe['op_id'], + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + if 'locations' in ep.get_query_types(): + paths[f'{collection_name_path}/locations'] = { + 'get': { + 'summary': f"Get pre-defined locations of {v['description']}", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': f'queryLOCATIONS{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + paths[f'{collection_name_path}/locations/{{locId}}'] = { + 'get': { + 'summary': f"query {v['description']} by location", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': f'queryLOCATIONSBYID{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + + return {'tags': [], 'paths': paths} diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 899d0e9ee..3df37916d 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -975,109 +975,6 @@ def get_oas_30(cfg): } } } - - LOGGER.debug('setting up edr endpoints') - edr_extension = filter_providers_by_type( - collections[k]['providers'], 'edr') - - if edr_extension: - ep = load_plugin('provider', edr_extension) - - edr_query_endpoints = [] - - for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/{qt}', - 'qt': qt, - 'op_id': f'query{qt.capitalize()}{k.capitalize()}' - }) - if ep.instances: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa - 'qt': qt, - 'op_id': f'query{qt.capitalize()}Instance{k.capitalize()}' # noqa - }) - - for eqe in edr_query_endpoints: - if eqe['qt'] == 'cube': - spatial_parameter = 'bbox' - else: - spatial_parameter = f"{eqe['qt']}Coords" - paths[eqe['path']] = { - 'get': { - 'summary': f"query {v['description']} by {eqe['qt']}", # noqa - 'description': v['description'], - 'tags': [k], - 'operationId': eqe['op_id'], - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - if 'locations' in ep.get_query_types(): - paths[f'{collection_name_path}/locations'] = { - 'get': { - 'summary': f"Get pre-defined locations of {v['description']}", # noqa - 'description': v['description'], - 'tags': [k], - 'operationId': f'queryLOCATIONS{k.capitalize()}', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - paths[f'{collection_name_path}/locations/{{locId}}'] = { - 'get': { - 'summary': f"query {v['description']} by location", # noqa - 'description': v['description'], - 'tags': [k], - 'operationId': f'queryLOCATIONSBYID{k.capitalize()}', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - LOGGER.debug('setting up maps endpoints') map_extension = filter_providers_by_type( collections[k]['providers'], 'map') From f482ab24c0ceba8bdab8c4ef860e7031abcc6842 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 14:58:37 +0100 Subject: [PATCH 17/71] Move maps openapi to maps api file --- pygeoapi/api/__init__.py | 4 +- pygeoapi/api/maps.py | 96 +++++++++- pygeoapi/openapi.py | 367 ++++++++++++++++----------------------- 3 files changed, 245 insertions(+), 222 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 99e2f8c9f..d452932b7 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -185,9 +185,9 @@ def all_apis(): # NOTE: this is a function and not a constant to avoid import loops - from . import processes, environmental_data_retrieval + from . import environmental_data_retrieval, maps, processes - return [processes, environmental_data_retrieval] + return [environmental_data_retrieval, maps, processes] def pre_process(func): diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index 2c9626bdb..7f5db7dae 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -38,14 +38,18 @@ # ================================================================= +from copy import deepcopy import logging from http import HTTPStatus from typing import Tuple - +from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin from pygeoapi.provider.base import ProviderGenericError -from pygeoapi.util import get_provider_by_type, to_json +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_providers_by_type, + filter_dict_by_key_value, +) from . import APIRequest, API, validate_datetime @@ -174,3 +178,91 @@ def get_collection_map(api: API, request: APIRequest, LOGGER.error(exception) return headers, HTTPStatus.BAD_REQUEST, to_json( data, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> dict: + from pygeoapi.openapi import OPENAPI_YAML + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + parameters = get_oas_30_parameters(cfg, locale) + for k, v in collections.items(): + + LOGGER.debug('setting up maps endpoints') + map_extension = filter_providers_by_type( + collections[k]['providers'], 'map') + + if map_extension: + mp = load_plugin('provider', map_extension) + + map_f = deepcopy(parameters['f']) + map_f['schema']['enum'] = [map_extension['format']['name']] + map_f['schema']['default'] = map_extension['format']['name'] + + pth = f'/collections/{k}/map' + paths[pth] = { + 'get': { + 'summary': 'Get map', + 'description': f"{v['description']} map", + 'tags': [k], + 'operationId': 'getMap', + 'parameters': [ + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + { + 'name': 'width', + 'in': 'query', + 'description': 'Response image width', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'height', + 'in': 'query', + 'description': 'Response image height', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'transparent', + 'in': 'query', + 'description': 'Background transparency of map (default=true).', # noqa + 'required': False, + 'schema': { + 'type': 'boolean', + 'default': True, + }, + 'style': 'form', + 'explode': False + }, + {'$ref': '#/components/parameters/bbox-crs-epsg'}, + map_f + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/json': {} + } + }, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + if mp.time_field is not None: + paths[pth]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + return {'tags': [], 'paths': paths} diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 3df37916d..b4e653c97 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -141,8 +141,7 @@ def get_oas_30(cfg): paths = {} # TODO: make openapi multilingual (default language only for now) - server_locales = l10n.get_locales(cfg) - locale_ = server_locales[0] + locale_ = l10n.get_locales(cfg)[0] api_rules = get_api_rules(cfg) @@ -299,149 +298,7 @@ def get_oas_30(cfg): } } }, - 'parameters': { - 'f': { - 'name': 'f', - 'in': 'query', - 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document. The default format is GeoJSON.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': ['json', 'html', 'jsonld'], - 'default': 'json' - }, - 'style': 'form', - 'explode': False - }, - 'lang': { - 'name': 'lang', - 'in': 'query', - 'description': 'The optional lang parameter instructs the server return a response in a certain language, if supported. If the language is not among the available values, the Accept-Language header language will be used if it is supported. If the header is missing, the default server language is used. Note that providers may only support a single language (or often no language at all), that can be different from the server language. Language strings can be written in a complex (e.g. "fr-CA,fr;q=0.9,en-US;q=0.8,en;q=0.7"), simple (e.g. "de") or locale-like (e.g. "de-CH" or "fr_BE") fashion.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': [l10n.locale2str(sl) for sl in server_locales], - 'default': l10n.locale2str(locale_) - } - }, - 'properties': { - 'name': 'properties', - 'in': 'query', - 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'items': { - 'type': 'string' - } - } - }, - 'skipGeometry': { - 'name': 'skipGeometry', - 'in': 'query', - 'description': 'This option can be used to skip response geometries for each feature.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'boolean', - 'default': False - } - }, - 'crs': { - 'name': 'crs', - 'in': 'query', - 'description': 'Indicates the coordinate reference system for the results.', # noqa - 'style': 'form', - 'required': False, - 'explode': False, - 'schema': { - 'format': 'uri', - 'type': 'string' - } - }, - 'bbox': { - 'name': 'bbox', - 'in': 'query', - 'description': 'Only features that have a geometry that intersects the bounding box are selected.' # noqa - 'The bounding box is provided as four or six numbers, depending on whether the ' # noqa - 'coordinate reference system includes a vertical axis (height or depth).', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'minItems': 4, - 'maxItems': 6, - 'items': { - 'type': 'number' - } - } - }, - 'bbox-crs': { - 'name': 'bbox-crs', - 'in': 'query', - 'description': 'Indicates the coordinate reference system for the given bbox coordinates.', # noqa - 'style': 'form', - 'required': False, - 'explode': False, - 'schema': { - 'format': 'uri', - 'type': 'string' - } - }, - # FIXME: This is not compatible with the bbox-crs definition in - # OGCAPI Features Part 2! - # We need to change the mapscript provider and - # get_collection_map() method in the API! - # So this is for de map-provider only. - 'bbox-crs-epsg': { - 'name': 'bbox-crs', - 'in': 'query', - 'description': 'Indicates the EPSG for the given bbox coordinates.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'integer', - 'default': 4326 - } - }, - 'offset': { - 'name': 'offset', - 'in': 'query', - 'description': 'The optional offset parameter indicates the index within the result set from which the server shall begin presenting results in the response document. The first element has an index of 0 (default).', # noqa - 'required': False, - 'schema': { - 'type': 'integer', - 'minimum': 0, - 'default': 0 - }, - 'style': 'form', - 'explode': False - }, - 'vendorSpecificParameters': { - 'name': 'vendorSpecificParameters', - 'in': 'query', - 'description': 'Additional "free-form" parameters that are not explicitly defined', # noqa - 'schema': { - 'type': 'object', - 'additionalProperties': True - }, - 'style': 'form' - }, - 'resourceId': { - 'name': 'resourceId', - 'in': 'path', - 'description': 'Configuration resource identifier', - 'required': True, - 'schema': { - 'type': 'string' - } - } - }, + 'parameters': get_oas_30_parameters(cfg=cfg, locale_=locale_), 'schemas': { # TODO: change this schema once OGC will definitively publish it 'queryable': { @@ -975,79 +832,6 @@ def get_oas_30(cfg): } } } - LOGGER.debug('setting up maps endpoints') - map_extension = filter_providers_by_type( - collections[k]['providers'], 'map') - - if map_extension: - mp = load_plugin('provider', map_extension) - - map_f = deepcopy(oas['components']['parameters']['f']) - map_f['schema']['enum'] = [map_extension['format']['name']] - map_f['schema']['default'] = map_extension['format']['name'] - - pth = f'/collections/{k}/map' - paths[pth] = { - 'get': { - 'summary': 'Get map', - 'description': f"{v['description']} map", - 'tags': [k], - 'operationId': 'getMap', - 'parameters': [ - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - { - 'name': 'width', - 'in': 'query', - 'description': 'Response image width', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'height', - 'in': 'query', - 'description': 'Response image height', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'transparent', - 'in': 'query', - 'description': 'Background transparency of map (default=true).', # noqa - 'required': False, - 'schema': { - 'type': 'boolean', - 'default': True, - }, - 'style': 'form', - 'explode': False - }, - {'$ref': '#/components/parameters/bbox-crs-epsg'}, - map_f - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/json': {} - } - }, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - if mp.time_field is not None: - paths[pth]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa LOGGER.debug('setting up STAC') stac_collections = filter_dict_by_key_value(cfg['resources'], @@ -1083,6 +867,153 @@ def get_oas_30(cfg): return oas +def get_oas_30_parameters(cfg: dict, locale_: str): + server_locales = l10n.get_locales(cfg) + return { + 'f': { + 'name': 'f', + 'in': 'query', + 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document. The default format is GeoJSON.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': ['json', 'html', 'jsonld'], + 'default': 'json' + }, + 'style': 'form', + 'explode': False + }, + 'lang': { + 'name': 'lang', + 'in': 'query', + 'description': 'The optional lang parameter instructs the server return a response in a certain language, if supported. If the language is not among the available values, the Accept-Language header language will be used if it is supported. If the header is missing, the default server language is used. Note that providers may only support a single language (or often no language at all), that can be different from the server language. Language strings can be written in a complex (e.g. "fr-CA,fr;q=0.9,en-US;q=0.8,en;q=0.7"), simple (e.g. "de") or locale-like (e.g. "de-CH" or "fr_BE") fashion.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [l10n.locale2str(sl) for sl in server_locales], + 'default': l10n.locale2str(locale_) + } + }, + 'properties': { + 'name': 'properties', + 'in': 'query', + 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + }, + 'skipGeometry': { + 'name': 'skipGeometry', + 'in': 'query', + 'description': 'This option can be used to skip response geometries for each feature.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'boolean', + 'default': False + } + }, + 'crs': { + 'name': 'crs', + 'in': 'query', + 'description': 'Indicates the coordinate reference system for the results.', # noqa + 'style': 'form', + 'required': False, + 'explode': False, + 'schema': { + 'format': 'uri', + 'type': 'string' + } + }, + 'bbox': { + 'name': 'bbox', + 'in': 'query', + 'description': 'Only features that have a geometry that intersects the bounding box are selected.' # noqa + 'The bounding box is provided as four or six numbers, depending on whether the ' # noqa + 'coordinate reference system includes a vertical axis (height or depth).', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'minItems': 4, + 'maxItems': 6, + 'items': { + 'type': 'number' + } + } + }, + 'bbox-crs': { + 'name': 'bbox-crs', + 'in': 'query', + 'description': 'Indicates the coordinate reference system for the given bbox coordinates.', # noqa + 'style': 'form', + 'required': False, + 'explode': False, + 'schema': { + 'format': 'uri', + 'type': 'string' + } + }, + # FIXME: This is not compatible with the bbox-crs definition in + # OGCAPI Features Part 2! + # We need to change the mapscript provider and + # get_collection_map() method in the API! + # So this is for de map-provider only. + 'bbox-crs-epsg': { + 'name': 'bbox-crs', + 'in': 'query', + 'description': 'Indicates the EPSG for the given bbox coordinates.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'integer', + 'default': 4326 + } + }, + 'offset': { + 'name': 'offset', + 'in': 'query', + 'description': 'The optional offset parameter indicates the index within the result set from which the server shall begin presenting results in the response document. The first element has an index of 0 (default).', # noqa + 'required': False, + 'schema': { + 'type': 'integer', + 'minimum': 0, + 'default': 0 + }, + 'style': 'form', + 'explode': False + }, + 'vendorSpecificParameters': { + 'name': 'vendorSpecificParameters', + 'in': 'query', + 'description': 'Additional "free-form" parameters that are not explicitly defined', # noqa + 'schema': { + 'type': 'object', + 'additionalProperties': True + }, + 'style': 'form' + }, + 'resourceId': { + 'name': 'resourceId', + 'in': 'path', + 'description': 'Configuration resource identifier', + 'required': True, + 'schema': { + 'type': 'string' + } + } + } + + def get_config_schema(): schema_file = os.path.join(THISDIR, 'schemas', 'config', 'pygeoapi-config-0.x.yml') From 6df83211f66abc59fe31c131d1650b10ee24f06a Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 15:01:44 +0100 Subject: [PATCH 18/71] Move stac views to own file --- pygeoapi/api/__init__.py | 171 ----------------------------- pygeoapi/api/stac.py | 232 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+), 171 deletions(-) create mode 100644 pygeoapi/api/stac.py diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index d452932b7..cbc40b57c 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -3093,177 +3093,6 @@ def get_collection_map_legend( return headers, HTTPStatus.BAD_REQUEST, to_json( data, self.pretty_print) - @gzip - @pre_process - @jsonldify - def get_stac_root( - self, request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide STAC root page - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - id_ = 'pygeoapi-stac' - stac_version = '1.0.0-rc.2' - stac_url = f'{self.base_url}/stac' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': l10n.translate( - self.config['metadata']['identification']['description'], - request.locale), - 'links': [] - } - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - for key, value in stac_collections.items(): - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}?f={F_JSON}', - 'type': FORMAT_TYPES[F_JSON] - }) - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}', - 'type': FORMAT_TYPES[F_HTML] - }) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'stac/collection.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_stac_path(self, request: Union[APIRequest, Any], - path) -> Tuple[dict, int, str]: - """ - Provide STAC resource path - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - dataset = None - LOGGER.debug(f'Path: {path}') - dir_tokens = path.split('/') - if dir_tokens: - dataset = dir_tokens[0] - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - if dataset not in stac_collections: - msg = 'Collection not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - stac_collections[dataset]['providers'], 'stac')) - except ProviderConnectionError as err: - LOGGER.error(err) - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - id_ = f'{dataset}-stac' - stac_version = '1.0.0-rc.2' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'description': l10n.translate( - stac_collections[dataset]['description'], request.locale), - 'links': [] - } - try: - stac_data = p.get_data_path( - f'{self.base_url}/stac', - path, - path.replace(dataset, '', 1) - ) - except ProviderNotFoundError as err: - LOGGER.error(err) - msg = 'resource not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - except Exception as err: - LOGGER.error(err) - msg = 'data query error' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - if isinstance(stac_data, dict): - content.update(stac_data) - content['links'].extend(stac_collections[dataset]['links']) - - if request.format == F_HTML: # render - content['path'] = path - if 'assets' in content: # item view - if content['type'] == 'Collection': - content = render_j2_template( - self.tpl_config, - 'stac/collection_base.html', - content, - request.locale - ) - elif content['type'] == 'Feature': - content = render_j2_template( - self.tpl_config, - 'stac/item.html', - content, - request.locale - ) - else: - msg = f'Unknown STAC type {content.type}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, - request.format, - 'NoApplicableCode', - msg) - else: - content = render_j2_template(self.tpl_config, - 'stac/catalog.html', - content, request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - else: # send back file - headers.pop('Content-Type', None) - return headers, HTTPStatus.OK, stac_data - def get_exception(self, status, headers, format_, code, description) -> Tuple[dict, int, str]: """ diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py new file mode 100644 index 000000000..623af899b --- /dev/null +++ b/pygeoapi/api/stac.py @@ -0,0 +1,232 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi.openapi import get_oas_30_parameters +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_providers_by_type, + filter_dict_by_key_value, +) + +from . import APIRequest, API, validate_datetime + + +LOGGER = logging.getLogger(__name__) + + +@gzip +@pre_process +@jsonldify +def get_stac_root( + self, request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: + """ + Provide STAC root page + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(**self.api_headers) + + id_ = 'pygeoapi-stac' + stac_version = '1.0.0-rc.2' + stac_url = f'{self.base_url}/stac' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'title': l10n.translate( + self.config['metadata']['identification']['title'], + request.locale), + 'description': l10n.translate( + self.config['metadata']['identification']['description'], + request.locale), + 'links': [] + } + + stac_collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'stac-collection') + + for key, value in stac_collections.items(): + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}?f={F_JSON}', + 'type': FORMAT_TYPES[F_JSON] + }) + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}', + 'type': FORMAT_TYPES[F_HTML] + }) + + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, + 'stac/collection.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + +@gzip +@pre_process +@jsonldify +def get_stac_path(self, request: Union[APIRequest, Any], + path) -> Tuple[dict, int, str]: + """ + Provide STAC resource path + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(**self.api_headers) + + dataset = None + LOGGER.debug(f'Path: {path}') + dir_tokens = path.split('/') + if dir_tokens: + dataset = dir_tokens[0] + + stac_collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'stac-collection') + + if dataset not in stac_collections: + msg = 'Collection not found' + return self.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + stac_collections[dataset]['providers'], 'stac')) + except ProviderConnectionError as err: + LOGGER.error(err) + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + id_ = f'{dataset}-stac' + stac_version = '1.0.0-rc.2' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'description': l10n.translate( + stac_collections[dataset]['description'], request.locale), + 'links': [] + } + try: + stac_data = p.get_data_path( + f'{self.base_url}/stac', + path, + path.replace(dataset, '', 1) + ) + except ProviderNotFoundError as err: + LOGGER.error(err) + msg = 'resource not found' + return self.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + except Exception as err: + LOGGER.error(err) + msg = 'data query error' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + if isinstance(stac_data, dict): + content.update(stac_data) + content['links'].extend(stac_collections[dataset]['links']) + + if request.format == F_HTML: # render + content['path'] = path + if 'assets' in content: # item view + if content['type'] == 'Collection': + content = render_j2_template( + self.tpl_config, + 'stac/collection_base.html', + content, + request.locale + ) + elif content['type'] == 'Feature': + content = render_j2_template( + self.tpl_config, + 'stac/item.html', + content, + request.locale + ) + else: + msg = f'Unknown STAC type {content.type}' + LOGGER.error(msg) + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, + request.format, + 'NoApplicableCode', + msg) + else: + content = render_j2_template(self.tpl_config, + 'stac/catalog.html', + content, request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + + else: # send back file + headers.pop('Content-Type', None) + return headers, HTTPStatus.OK, stac_data + + + From abc7116cfd255228d6ae5d614da2e367e20fb59b Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 15:10:01 +0100 Subject: [PATCH 19/71] Refactor stac views to new file --- pygeoapi/api/stac.py | 32 ++++++++++++-------------------- pygeoapi/django_/views.py | 11 +++-------- pygeoapi/flask_app.py | 6 ++++-- pygeoapi/starlette_app.py | 5 +++-- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index 623af899b..a281b63bb 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -43,25 +43,27 @@ from http import HTTPStatus from typing import Tuple +from pygeoapi import __version__, l10n from pygeoapi.openapi import get_oas_30_parameters from pygeoapi.plugin import load_plugin -from pygeoapi.provider.base import ProviderGenericError + +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, +) from pygeoapi.util import ( get_provider_by_type, to_json, filter_providers_by_type, - filter_dict_by_key_value, + filter_dict_by_key_value, render_j2_template, ) -from . import APIRequest, API, validate_datetime +from . import APIRequest, API, validate_datetime, FORMAT_TYPES, F_JSON, F_HTML LOGGER = logging.getLogger(__name__) -@gzip -@pre_process -@jsonldify +# TODO: no tests for this? def get_stac_root( - self, request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: + self, request: APIRequest) -> Tuple[dict, int, str]: """ Provide STAC root page @@ -69,9 +71,6 @@ def get_stac_root( :returns: tuple of headers, status code, content """ - - if not request.is_valid(): - return self.get_format_exception(request) headers = request.get_response_headers(**self.api_headers) id_ = 'pygeoapi-stac' @@ -114,10 +113,9 @@ def get_stac_root( return headers, HTTPStatus.OK, to_json(content, self.pretty_print) -@gzip -@pre_process -@jsonldify -def get_stac_path(self, request: Union[APIRequest, Any], + +# TODO: no tests for this? +def get_stac_path(self, request: APIRequest, path) -> Tuple[dict, int, str]: """ Provide STAC resource path @@ -126,9 +124,6 @@ def get_stac_path(self, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - - if not request.is_valid(): - return self.get_format_exception(request) headers = request.get_response_headers(**self.api_headers) dataset = None @@ -227,6 +222,3 @@ def get_stac_path(self, request: Union[APIRequest, Any], else: # send back file headers.pop('Content-Type', None) return headers, HTTPStatus.OK, stac_data - - - diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 7c272b92f..1f151a80c 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -44,6 +44,7 @@ import pygeoapi.api.environmental_data_retrieval as edr_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -462,10 +463,7 @@ def stac_catalog_root(request: HttpRequest) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_root') - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_root, request) def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: @@ -478,10 +476,7 @@ def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_path', path) - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_path, request, path) def admin_config(request: HttpRequest) -> HttpResponse: diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index ca17eb659..8a15f82e9 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -41,6 +41,7 @@ import pygeoapi.api.environmental_data_retrieval as edr_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -482,7 +483,8 @@ def stac_catalog_root(): :returns: HTTP response """ - return get_response(api_.get_stac_root(request)) + + return execute_from_flask(stac_api.get_stac_root, request) @BLUEPRINT.route('/stac/') @@ -494,7 +496,7 @@ def stac_catalog_path(path): :returns: HTTP response """ - return get_response(api_.get_stac_path(request, path)) + return execute_from_flask(stac_api.get_stac_path, request, path) @ADMIN_BLUEPRINT.route('/admin/config', methods=['GET', 'PUT', 'PATCH']) diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 757ceb343..1ba7f8003 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -54,6 +54,7 @@ import pygeoapi.api.environmental_data_retrieval as edr_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -546,7 +547,7 @@ async def stac_catalog_root(request: Request): :returns: Starlette HTTP response """ - return await get_response(api_.get_stac_root, request) + return await execute_from_starlette(stac_api.get_stac_root, request) async def stac_catalog_path(request: Request): @@ -558,7 +559,7 @@ async def stac_catalog_path(request: Request): :returns: Starlette HTTP response """ path = request.path_params["path"] - return await get_response(api_.get_stac_path, request, path) + return await execute_from_starlette(stac_api.get_stac_path, request, path) async def admin_config(request: Request): From 3f314e40cecdf0d654b629ad56b7e70eb9d6cc19 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 15:17:45 +0100 Subject: [PATCH 20/71] Move stac openapi to stac api file --- pygeoapi/api/__init__.py | 7 ++-- pygeoapi/api/stac.py | 88 ++++++++++++++++++++++++++-------------- pygeoapi/openapi.py | 23 ----------- 3 files changed, 60 insertions(+), 58 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index cbc40b57c..5ed8a7ac2 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -64,8 +64,7 @@ from pygeoapi.process.manager.base import get_manager from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ( - ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, - ProviderTypeError) + ProviderGenericError, ProviderConnectionError, ProviderTypeError) from pygeoapi.models.provider.base import (TilesMetadataFormat, TileMatrixSetEnum) @@ -185,9 +184,9 @@ def all_apis(): # NOTE: this is a function and not a constant to avoid import loops - from . import environmental_data_retrieval, maps, processes + from . import environmental_data_retrieval, maps, processes, stac - return [environmental_data_retrieval, maps, processes] + return [environmental_data_retrieval, maps, processes, stac] def pre_process(func): diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index a281b63bb..ddf4d251d 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -38,32 +38,29 @@ # ================================================================= -from copy import deepcopy import logging from http import HTTPStatus from typing import Tuple -from pygeoapi import __version__, l10n -from pygeoapi.openapi import get_oas_30_parameters +from pygeoapi import l10n from pygeoapi.plugin import load_plugin from pygeoapi.provider.base import ( - ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, + ProviderConnectionError, ProviderNotFoundError, ) from pygeoapi.util import ( - get_provider_by_type, to_json, filter_providers_by_type, - filter_dict_by_key_value, render_j2_template, + get_provider_by_type, to_json, filter_dict_by_key_value, + render_j2_template, ) -from . import APIRequest, API, validate_datetime, FORMAT_TYPES, F_JSON, F_HTML +from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML LOGGER = logging.getLogger(__name__) # TODO: no tests for this? -def get_stac_root( - self, request: APIRequest) -> Tuple[dict, int, str]: +def get_stac_root(api: API, request: APIRequest) -> Tuple[dict, int, str]: """ Provide STAC root page @@ -71,26 +68,26 @@ def get_stac_root( :returns: tuple of headers, status code, content """ - headers = request.get_response_headers(**self.api_headers) + headers = request.get_response_headers(**api.api_headers) id_ = 'pygeoapi-stac' stac_version = '1.0.0-rc.2' - stac_url = f'{self.base_url}/stac' + stac_url = f'{api.base_url}/stac' content = { 'id': id_, 'type': 'Catalog', 'stac_version': stac_version, 'title': l10n.translate( - self.config['metadata']['identification']['title'], + api.config['metadata']['identification']['title'], request.locale), 'description': l10n.translate( - self.config['metadata']['identification']['description'], + api.config['metadata']['identification']['description'], request.locale), 'links': [] } - stac_collections = filter_dict_by_key_value(self.config['resources'], + stac_collections = filter_dict_by_key_value(api.config['resources'], 'type', 'stac-collection') for key, value in stac_collections.items(): @@ -106,16 +103,16 @@ def get_stac_root( }) if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'stac/collection.html', content, request.locale) return headers, HTTPStatus.OK, content - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) # TODO: no tests for this? -def get_stac_path(self, request: APIRequest, +def get_stac_path(api: API, request: APIRequest, path) -> Tuple[dict, int, str]: """ Provide STAC resource path @@ -124,7 +121,7 @@ def get_stac_path(self, request: APIRequest, :returns: tuple of headers, status code, content """ - headers = request.get_response_headers(**self.api_headers) + headers = request.get_response_headers(**api.api_headers) dataset = None LOGGER.debug(f'Path: {path}') @@ -132,13 +129,13 @@ def get_stac_path(self, request: APIRequest, if dir_tokens: dataset = dir_tokens[0] - stac_collections = filter_dict_by_key_value(self.config['resources'], + stac_collections = filter_dict_by_key_value(api.config['resources'], 'type', 'stac-collection') if dataset not in stac_collections: msg = 'Collection not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) LOGGER.debug('Loading provider') try: @@ -147,7 +144,7 @@ def get_stac_path(self, request: APIRequest, except ProviderConnectionError as err: LOGGER.error(err) msg = 'connection error (check logs)' - return self.get_exception( + return api.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) @@ -164,19 +161,19 @@ def get_stac_path(self, request: APIRequest, } try: stac_data = p.get_data_path( - f'{self.base_url}/stac', + f'{api.base_url}/stac', path, path.replace(dataset, '', 1) ) except ProviderNotFoundError as err: LOGGER.error(err) msg = 'resource not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) except Exception as err: LOGGER.error(err) msg = 'data query error' - return self.get_exception( + return api.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) @@ -189,14 +186,14 @@ def get_stac_path(self, request: APIRequest, if 'assets' in content: # item view if content['type'] == 'Collection': content = render_j2_template( - self.tpl_config, + api.tpl_config, 'stac/collection_base.html', content, request.locale ) elif content['type'] == 'Feature': content = render_j2_template( - self.tpl_config, + api.tpl_config, 'stac/item.html', content, request.locale @@ -204,21 +201,50 @@ def get_stac_path(self, request: APIRequest, else: msg = f'Unknown STAC type {content.type}' LOGGER.error(msg) - return self.get_exception( + return api.get_exception( HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, 'NoApplicableCode', msg) else: - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'stac/catalog.html', content, request.locale) return headers, HTTPStatus.OK, content - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) else: # send back file headers.pop('Content-Type', None) return headers, HTTPStatus.OK, stac_data + + +def get_oas_30(cfg: dict, locale: str) -> dict: + LOGGER.debug('setting up STAC') + stac_collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'stac-collection') + paths = {} + if stac_collections: + paths['/stac'] = { + 'get': { + 'summary': 'SpatioTemporal Asset Catalog', + 'description': 'SpatioTemporal Asset Catalog', + 'tags': ['stac'], + 'operationId': 'getStacCatalog', + 'parameters': [], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + return { + 'tags': [{ + 'name': 'stac', + 'description': 'SpatioTemporal Asset Catalog' + }], + 'paths': paths, + + } diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index b4e653c97..e3dfc32dd 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -271,11 +271,6 @@ def get_oas_30(cfg): 'url': cfg['metadata']['identification']['url']} } ) - oas['tags'].append({ - 'name': 'stac', - 'description': 'SpatioTemporal Asset Catalog' - } - ) oas['components'] = { 'responses': { @@ -833,24 +828,6 @@ def get_oas_30(cfg): } } - LOGGER.debug('setting up STAC') - stac_collections = filter_dict_by_key_value(cfg['resources'], - 'type', 'stac-collection') - if stac_collections: - paths['/stac'] = { - 'get': { - 'summary': 'SpatioTemporal Asset Catalog', - 'description': 'SpatioTemporal Asset Catalog', - 'tags': ['stac'], - 'operationId': 'getStacCatalog', - 'parameters': [], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } - oas['paths'] = paths for api in all_apis(): From 8ab7ec9d975c6a397cd44fa4dddd67fc54f6536c Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 16:04:06 +0100 Subject: [PATCH 21/71] Move tiles api to own file --- pygeoapi/api/__init__.py | 262 ------------------------------- pygeoapi/api/tiles.py | 324 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 324 insertions(+), 262 deletions(-) create mode 100644 pygeoapi/api/tiles.py diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 5ed8a7ac2..9ecb32fce 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -2769,268 +2769,6 @@ def get_collection_coverage(self, request: Union[APIRequest, Any], else: return self.get_format_exception(request) - @gzip - @pre_process - @jsonldify - def get_collection_tiles(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection tiles - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except (KeyError, ProviderTypeError): - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - tiles = { - 'links': [], - 'tilesets': [] - } - - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSON}' - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_HTML}' - }) - - tile_services = p.get_tiles_service( - baseurl=self.base_url, - servicepath=f'{self.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa - ) - - for service in tile_services['links']: - tiles['links'].append(service) - - tiling_schemes = p.get_tiling_schemes() - - for matrix in tiling_schemes: - tile_matrix = { - 'title': dataset, - 'tileMatrixSetURI': matrix.tileMatrixSetURI, - 'crs': matrix.crs, - 'dataType': 'vector', - 'links': [] - } - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', - 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa - 'href': f'{self.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa - }) - - tiles['tilesets'].append(tile_matrix) - - if request.format == F_HTML: # render - tiles['id'] = dataset - tiles['title'] = l10n.translate( - self.config['resources'][dataset]['title'], SYSTEM_LOCALE) - tiles['tilesets'] = [ - scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] - tiles['bounds'] = \ - self.config['resources'][dataset]['extents']['spatial']['bbox'] - tiles['minzoom'] = p.options['zoom']['min'] - tiles['maxzoom'] = p.options['zoom']['max'] - tiles['collections_path'] = self.get_collections_url() - tiles['tile_type'] = p.tile_type - - content = render_j2_template(self.tpl_config, - 'collections/tiles/index.html', tiles, - request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tiles, self.pretty_print) - - @pre_process - def get_collection_tiles_data( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None, - z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - :param z_idx: z index - :param y_idx: y index - :param x_idx: x index - - :returns: tuple of headers, status code, content - """ - - format_ = request.format - if not format_: - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - LOGGER.debug('Processing tiles') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading tile provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - - format_ = p.format_type - headers['Content-Type'] = format_ - - LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa - content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, - z=z_idx, y=y_idx, x=x_idx, format_=format_) - if content is None: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) - else: - return headers, HTTPStatus.OK, content - - # @TODO: figure out if the spec requires to return json errors - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles_metadata( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid([TilesMetadataFormat.TILEJSON]): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(t, request.raw_locale) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - tiles_metadata = p.get_metadata( - dataset=dataset, server_url=self.base_url, - layer=p.get_layer(), tileset=matrix_id, - metadata_format=request._format, title=l10n.translate( - self.config['resources'][dataset]['title'], - request.locale), - description=l10n.translate( - self.config['resources'][dataset]['description'], - request.locale), - language=prv_locale) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/tiles/metadata.html', - tiles_metadata, request.locale) - - return headers, HTTPStatus.OK, content - else: - return headers, HTTPStatus.OK, tiles_metadata - @gzip def get_collection_map_legend( self, request: Union[APIRequest, Any], diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py new file mode 100644 index 000000000..81bb48498 --- /dev/null +++ b/pygeoapi/api/tiles.py @@ -0,0 +1,324 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin + +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderNotFoundError, +) +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + render_j2_template, +) + +from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML + + +LOGGER = logging.getLogger(__name__) + + +@gzip +@pre_process +@jsonldify +def get_collection_tiles(self, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection tiles + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + if any([dataset is None, + dataset not in self.config['resources'].keys()]): + + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + self.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except (KeyError, ProviderTypeError): + msg = 'Invalid collection tiles' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + tiles = { + 'links': [], + 'tilesets': [] + } + + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSON}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_HTML}' + }) + + tile_services = p.get_tiles_service( + baseurl=self.base_url, + servicepath=f'{self.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa + ) + + for service in tile_services['links']: + tiles['links'].append(service) + + tiling_schemes = p.get_tiling_schemes() + + for matrix in tiling_schemes: + tile_matrix = { + 'title': dataset, + 'tileMatrixSetURI': matrix.tileMatrixSetURI, + 'crs': matrix.crs, + 'dataType': 'vector', + 'links': [] + } + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', + 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa + 'href': f'{self.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', + 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', + 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa + }) + + tiles['tilesets'].append(tile_matrix) + + if request.format == F_HTML: # render + tiles['id'] = dataset + tiles['title'] = l10n.translate( + self.config['resources'][dataset]['title'], SYSTEM_LOCALE) + tiles['tilesets'] = [ + scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] + tiles['bounds'] = \ + self.config['resources'][dataset]['extents']['spatial']['bbox'] + tiles['minzoom'] = p.options['zoom']['min'] + tiles['maxzoom'] = p.options['zoom']['max'] + tiles['collections_path'] = self.get_collections_url() + tiles['tile_type'] = p.tile_type + + content = render_j2_template(self.tpl_config, + 'collections/tiles/index.html', tiles, + request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tiles, self.pretty_print) + +@pre_process +def get_collection_tiles_data( + self, request: Union[APIRequest, Any], + dataset=None, matrix_id=None, + z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + :param z_idx: z index + :param y_idx: y index + :param x_idx: x index + + :returns: tuple of headers, status code, content + """ + + format_ = request.format + if not format_: + return self.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + LOGGER.debug('Processing tiles') + + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading tile provider') + try: + t = get_provider_by_type( + self.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + + format_ = p.format_type + headers['Content-Type'] = format_ + + LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa + content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, + z=z_idx, y=y_idx, x=x_idx, format_=format_) + if content is None: + msg = 'identifier not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) + else: + return headers, HTTPStatus.OK, content + + # @TODO: figure out if the spec requires to return json errors + except KeyError: + msg = 'Invalid collection tiles' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + +@gzip +@pre_process +@jsonldify +def get_collection_tiles_metadata( + self, request: Union[APIRequest, Any], + dataset=None, matrix_id=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid([TilesMetadataFormat.TILEJSON]): + return self.get_format_exception(request) + headers = request.get_response_headers(**self.api_headers) + + if any([dataset is None, + dataset not in self.config['resources'].keys()]): + + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + self.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except KeyError: + msg = 'Invalid collection tiles' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(t, request.raw_locale) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + tiles_metadata = p.get_metadata( + dataset=dataset, server_url=self.base_url, + layer=p.get_layer(), tileset=matrix_id, + metadata_format=request._format, title=l10n.translate( + self.config['resources'][dataset]['title'], + request.locale), + description=l10n.translate( + self.config['resources'][dataset]['description'], + request.locale), + language=prv_locale) + + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, + 'collections/tiles/metadata.html', + tiles_metadata, request.locale) + + return headers, HTTPStatus.OK, content + else: + return headers, HTTPStatus.OK, tiles_metadata + + From c2b332f3b2f51f81c00547feeaa58885e4587393 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 16:13:10 +0100 Subject: [PATCH 22/71] Adapt tiles api to new style --- pygeoapi/api/__init__.py | 1 + pygeoapi/api/tiles.py | 106 +++++++++++++++++++-------------------- tests/test_api.py | 15 +++--- 3 files changed, 60 insertions(+), 62 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 9ecb32fce..3f7aa2735 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -922,6 +922,7 @@ def conformance(self, return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) + # TODO: also move to tiles @gzip @pre_process def tilematrixsets(self, diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 81bb48498..7e9d8dde2 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -44,25 +44,25 @@ from pygeoapi import l10n from pygeoapi.plugin import load_plugin - +from pygeoapi.models.provider.base import (TilesMetadataFormat, + TileMatrixSetEnum) from pygeoapi.provider.base import ( - ProviderConnectionError, ProviderNotFoundError, + ProviderGenericError, ProviderTypeError, ) from pygeoapi.util import ( get_provider_by_type, to_json, filter_dict_by_key_value, render_j2_template, ) -from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML +from . import ( + APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML, SYSTEM_LOCALE, F_JSONLD +) LOGGER = logging.getLogger(__name__) -@gzip -@pre_process -@jsonldify -def get_collection_tiles(self, request: Union[APIRequest, Any], +def get_collection_tiles(api: API, request: APIRequest, dataset=None) -> Tuple[dict, int, str]: """ Provide collection tiles @@ -73,31 +73,29 @@ def get_collection_tiles(self, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) + **api.api_headers) if any([dataset is None, - dataset not in self.config['resources'].keys()]): + dataset not in api.config['resources'].keys()]): msg = 'Collection not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) LOGGER.debug('Creating collection tiles') LOGGER.debug('Loading provider') try: t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') + api.config['resources'][dataset]['providers'], 'tile') p = load_plugin('provider', t) except (KeyError, ProviderTypeError): msg = 'Invalid collection tiles' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) @@ -110,24 +108,24 @@ def get_collection_tiles(self, request: Union[APIRequest, Any], 'type': FORMAT_TYPES[F_JSON], 'rel': request.get_linkrel(F_JSON), 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSON}' + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSON}' }) tiles['links'].append({ 'type': FORMAT_TYPES[F_JSONLD], 'rel': request.get_linkrel(F_JSONLD), 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa }) tiles['links'].append({ 'type': FORMAT_TYPES[F_HTML], 'rel': request.get_linkrel(F_HTML), 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_HTML}' + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_HTML}' }) tile_services = p.get_tiles_service( - baseurl=self.base_url, - servicepath=f'{self.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa + baseurl=api.base_url, + servicepath=f'{api.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa ) for service in tile_services['links']: @@ -147,19 +145,19 @@ def get_collection_tiles(self, request: Union[APIRequest, Any], 'type': FORMAT_TYPES[F_JSON], 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa - 'href': f'{self.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + 'href': f'{api.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa }) tile_matrix['links'].append({ 'type': FORMAT_TYPES[F_JSON], 'rel': request.get_linkrel(F_JSON), 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa }) tile_matrix['links'].append({ 'type': FORMAT_TYPES[F_HTML], 'rel': request.get_linkrel(F_HTML), 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa }) tiles['tilesets'].append(tile_matrix) @@ -167,27 +165,28 @@ def get_collection_tiles(self, request: Union[APIRequest, Any], if request.format == F_HTML: # render tiles['id'] = dataset tiles['title'] = l10n.translate( - self.config['resources'][dataset]['title'], SYSTEM_LOCALE) + api.config['resources'][dataset]['title'], SYSTEM_LOCALE) tiles['tilesets'] = [ scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] tiles['bounds'] = \ - self.config['resources'][dataset]['extents']['spatial']['bbox'] + api.config['resources'][dataset]['extents']['spatial']['bbox'] tiles['minzoom'] = p.options['zoom']['min'] tiles['maxzoom'] = p.options['zoom']['max'] - tiles['collections_path'] = self.get_collections_url() + tiles['collections_path'] = api.get_collections_url() tiles['tile_type'] = p.tile_type - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'collections/tiles/index.html', tiles, request.locale) return headers, HTTPStatus.OK, content - return headers, HTTPStatus.OK, to_json(tiles, self.pretty_print) + return headers, HTTPStatus.OK, to_json(tiles, api.pretty_print) + -@pre_process +# TODO: no test for this function? def get_collection_tiles_data( - self, request: Union[APIRequest, Any], + api: API, request: APIRequest, dataset=None, matrix_id=None, z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: """ @@ -205,23 +204,23 @@ def get_collection_tiles_data( format_ = request.format if not format_: - return self.get_format_exception(request) + return api.get_format_exception(request) headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) + **api.api_headers) LOGGER.debug('Processing tiles') - collections = filter_dict_by_key_value(self.config['resources'], + collections = filter_dict_by_key_value(api.config['resources'], 'type', 'collection') if dataset not in collections.keys(): msg = 'Collection not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) LOGGER.debug('Loading tile provider') try: t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') + api.config['resources'][dataset]['providers'], 'tile') p = load_plugin('provider', t) format_ = p.format_type @@ -232,7 +231,7 @@ def get_collection_tiles_data( z=z_idx, y=y_idx, x=x_idx, format_=format_) if content is None: msg = 'identifier not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) else: return headers, HTTPStatus.OK, content @@ -240,20 +239,19 @@ def get_collection_tiles_data( # @TODO: figure out if the spec requires to return json errors except KeyError: msg = 'Invalid collection tiles' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, format_, 'InvalidParameterValue', msg) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) -@gzip -@pre_process -@jsonldify + +# TODO: no test for this function? def get_collection_tiles_metadata( - self, request: Union[APIRequest, Any], + api: API, request: APIRequest, dataset=None, matrix_id=None) -> Tuple[dict, int, str]: """ Get collection items tiles @@ -266,30 +264,30 @@ def get_collection_tiles_metadata( """ if not request.is_valid([TilesMetadataFormat.TILEJSON]): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) + return api.get_format_exception(request) + headers = request.get_response_headers(**api.api_headers) if any([dataset is None, - dataset not in self.config['resources'].keys()]): + dataset not in api.config['resources'].keys()]): msg = 'Collection not found' - return self.get_exception( + return api.get_exception( HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) LOGGER.debug('Creating collection tiles') LOGGER.debug('Loading provider') try: t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') + api.config['resources'][dataset]['providers'], 'tile') p = load_plugin('provider', t) except KeyError: msg = 'Invalid collection tiles' - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', msg) except ProviderGenericError as err: LOGGER.error(err) - return self.get_exception( + return api.get_exception( err.http_status_code, headers, request.format, err.ogc_exception_code, err.message) @@ -302,23 +300,21 @@ def get_collection_tiles_metadata( l10n.set_response_language(headers, prv_locale, request.locale) tiles_metadata = p.get_metadata( - dataset=dataset, server_url=self.base_url, + dataset=dataset, server_url=api.base_url, layer=p.get_layer(), tileset=matrix_id, metadata_format=request._format, title=l10n.translate( - self.config['resources'][dataset]['title'], + api.config['resources'][dataset]['title'], request.locale), description=l10n.translate( - self.config['resources'][dataset]['description'], + api.config['resources'][dataset]['description'], request.locale), language=prv_locale) if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'collections/tiles/metadata.html', tiles_metadata, request.locale) return headers, HTTPStatus.OK, content else: return headers, HTTPStatus.OK, tiles_metadata - - diff --git a/tests/test_api.py b/tests/test_api.py index a83a5d99e..5b2b9881f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -51,6 +51,7 @@ from pygeoapi.api.processes import ( describe_processes, execute_process, delete_job, get_job_result, ) +from pygeoapi.api.tiles import get_collection_tiles from pygeoapi.util import (yaml_load, get_crs_from_uri, get_api_rules, get_base_url) @@ -1583,18 +1584,18 @@ def test_get_collection_map(config, api_): def test_get_collection_tiles(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_tiles(req, 'obs') + req = mock_api_request() + rsp_headers, code, response = get_collection_tiles(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - rsp_headers, code, response = api_.get_collection_tiles( - req, 'naturalearth/lakes') + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') assert code == HTTPStatus.OK # Language settings should be ignored (return system default) - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_tiles( - req, 'naturalearth/lakes') + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') assert rsp_headers['Content-Language'] == 'en-US' content = json.loads(response) assert len(content['links']) > 0 From b6e3f2dc58378e13102548cf685dc559542a3fd4 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 16:13:49 +0100 Subject: [PATCH 23/71] Also move tilematrixset to tiles api --- pygeoapi/api/tiles.py | 119 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 7e9d8dde2..7c6a3bbfd 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -318,3 +318,122 @@ def get_collection_tiles_metadata( return headers, HTTPStatus.OK, content else: return headers, HTTPStatus.OK, tiles_metadata + +# TODO: also move to tiles +@gzip +@pre_process +def tilematrixsets(self, + request: Union[APIRequest, Any]) -> Tuple[dict, int, + str]: + """ + Provide tileMatrixSets definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers(**self.api_headers) + + # Retrieve available TileMatrixSets + enums = [e.value for e in TileMatrixSetEnum] + + tms = {"tileMatrixSets": []} + + for e in enums: + tms['tileMatrixSets'].append({ + "title": e.title, + "id": e.tileMatrixSet, + "uri": e.tileMatrixSetURI, + "links": [ + { + "rel": "self", + "type": "text/html", + "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa + }, + { + "rel": "self", + "type": "application/json", + "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa + } + ] + }) + + tms['links'] = [{ + "rel": "alternate", + "type": "text/html", + "title": "This document as HTML", + "href": f"{self.base_url}/tileMatrixSets?f=html" + }, { + "rel": "self", + "type": "application/json", + "title": "This document", + "href": f"{self.base_url}/tileMatrixSets?f=json" + }] + + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, + 'tilematrixsets/index.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) + +@gzip +@pre_process +def tilematrixset(self, + request: Union[APIRequest, Any], + tileMatrixSetId) -> Tuple[dict, + int, str]: + """ + Provide tile matrix definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers(**self.api_headers) + + # Retrieve relevant TileMatrixSet + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + try: + for e in enums: + if tileMatrixSetId == e.tileMatrixSet: + enum = e + if not enum: + raise ValueError('could not find this tilematrixset') + except ValueError as err: + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + tms = { + "title": enum.tileMatrixSet, + "crs": enum.crs, + "id": enum.tileMatrixSet, + "uri": enum.tileMatrixSetURI, + "orderedAxes": enum.orderedAxes, + "wellKnownScaleSet": enum.wellKnownScaleSet, + "tileMatrices": enum.tileMatrices + } + + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, + 'tilematrixsets/tilematrixset.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) + + From b5d1769339db210cc2415e8a77ae1fcfe7c33a0a Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 13 Mar 2024 16:18:30 +0100 Subject: [PATCH 24/71] Adapt tilesetmatrix views to new style NOTE: I had to remove one tilematrixsets test because it tested that an invalid format would produce an error. This now happens by default for all views, but the actual code is outside of the endpoint function. --- pygeoapi/api/__init__.py | 119 -------------------------------------- pygeoapi/api/tiles.py | 45 ++++++-------- pygeoapi/django_/views.py | 25 +++----- pygeoapi/flask_app.py | 19 +++--- pygeoapi/starlette_app.py | 22 ++++--- tests/test_api.py | 26 ++++----- 6 files changed, 62 insertions(+), 194 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 3f7aa2735..263b2a0a6 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -65,8 +65,6 @@ from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ( ProviderGenericError, ProviderConnectionError, ProviderTypeError) -from pygeoapi.models.provider.base import (TilesMetadataFormat, - TileMatrixSetEnum) from pygeoapi.models.cql import CQLModel from pygeoapi.util import (dategetter, @@ -922,123 +920,6 @@ def conformance(self, return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) - # TODO: also move to tiles - @gzip - @pre_process - def tilematrixsets(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, - str]: - """ - Provide tileMatrixSets definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve available TileMatrixSets - enums = [e.value for e in TileMatrixSetEnum] - - tms = {"tileMatrixSets": []} - - for e in enums: - tms['tileMatrixSets'].append({ - "title": e.title, - "id": e.tileMatrixSet, - "uri": e.tileMatrixSetURI, - "links": [ - { - "rel": "self", - "type": "text/html", - "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa - }, - { - "rel": "self", - "type": "application/json", - "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa - } - ] - }) - - tms['links'] = [{ - "rel": "alternate", - "type": "text/html", - "title": "This document as HTML", - "href": f"{self.base_url}/tileMatrixSets?f=html" - }, { - "rel": "self", - "type": "application/json", - "title": "This document", - "href": f"{self.base_url}/tileMatrixSets?f=json" - }] - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/index.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - def tilematrixset(self, - request: Union[APIRequest, Any], - tileMatrixSetId) -> Tuple[dict, - int, str]: - """ - Provide tile matrix definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve relevant TileMatrixSet - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - try: - for e in enums: - if tileMatrixSetId == e.tileMatrixSet: - enum = e - if not enum: - raise ValueError('could not find this tilematrixset') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - tms = { - "title": enum.tileMatrixSet, - "crs": enum.crs, - "id": enum.tileMatrixSet, - "uri": enum.tileMatrixSetURI, - "orderedAxes": enum.orderedAxes, - "wellKnownScaleSet": enum.wellKnownScaleSet, - "tileMatrices": enum.tileMatrices - } - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/tilematrixset.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - @gzip @pre_process @jsonldify diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 7c6a3bbfd..3f1aa85a3 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -319,12 +319,9 @@ def get_collection_tiles_metadata( else: return headers, HTTPStatus.OK, tiles_metadata -# TODO: also move to tiles -@gzip -@pre_process -def tilematrixsets(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, - str]: + +def tilematrixsets(api: API, + request: APIRequest) -> Tuple[dict, int, str]: """ Provide tileMatrixSets definition @@ -333,10 +330,7 @@ def tilematrixsets(self, :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) + headers = request.get_response_headers(**api.api_headers) # Retrieve available TileMatrixSets enums = [e.value for e in TileMatrixSetEnum] @@ -353,13 +347,13 @@ def tilematrixsets(self, "rel": "self", "type": "text/html", "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa }, { "rel": "self", "type": "application/json", "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa } ] }) @@ -368,26 +362,25 @@ def tilematrixsets(self, "rel": "alternate", "type": "text/html", "title": "This document as HTML", - "href": f"{self.base_url}/tileMatrixSets?f=html" + "href": f"{api.base_url}/tileMatrixSets?f=html" }, { "rel": "self", "type": "application/json", "title": "This document", - "href": f"{self.base_url}/tileMatrixSets?f=json" + "href": f"{api.base_url}/tileMatrixSets?f=json" }] if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'tilematrixsets/index.html', tms, request.locale) return headers, HTTPStatus.OK, content - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + -@gzip -@pre_process -def tilematrixset(self, - request: Union[APIRequest, Any], +def tilematrixset(api: API, + request: APIRequest, tileMatrixSetId) -> Tuple[dict, int, str]: """ @@ -398,10 +391,7 @@ def tilematrixset(self, :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) + headers = request.get_response_headers(**api.api_headers) # Retrieve relevant TileMatrixSet enums = [e.value for e in TileMatrixSetEnum] @@ -414,7 +404,7 @@ def tilematrixset(self, if not enum: raise ValueError('could not find this tilematrixset') except ValueError as err: - return self.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'InvalidParameterValue', str(err)) @@ -429,11 +419,12 @@ def tilematrixset(self, } if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, + content = render_j2_template(api.tpl_config, 'tilematrixsets/tilematrixset.html', tms, request.locale) return headers, HTTPStatus.OK, content - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) +# TODO: openapi diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 1f151a80c..63b503bdd 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -45,6 +45,7 @@ import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -299,10 +300,8 @@ def collection_tiles(request: HttpRequest, collection_id: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_collection_tiles', collection_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(tiles_api.get_collection_tiles, request, + collection_id) def collection_tiles_metadata(request: HttpRequest, collection_id: str, @@ -317,15 +316,10 @@ def collection_tiles_metadata(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( - request, - 'get_collection_tiles_metadata', - collection_id, - tileMatrixSetId, + return execute_from_django( + tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, ) - response = _to_django_response(*response_) - - return response def collection_item_tiles(request: HttpRequest, collection_id: str, @@ -344,18 +338,15 @@ def collection_item_tiles(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( + return execute_from_django( + tiles_api.get_collection_tiles_data, request, - 'get_collection_tiles_metadata', collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, ) - response = _to_django_response(*response_) - - return response def processes(request: HttpRequest, diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 8a15f82e9..7c7f1e29e 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -42,6 +42,7 @@ import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -184,7 +185,8 @@ def get_tilematrix_set(tileMatrixSetId=None): :param tileMatrixSetId: identifier of tile matrix set :returns: HTTP response """ - return get_response(api_.tilematrixset(request, tileMatrixSetId)) + return execute_from_flask(tiles_api.tilematrixset, request, + tileMatrixSetId) @BLUEPRINT.route('/TileMatrixSets') @@ -194,7 +196,7 @@ def get_tilematrix_sets(): :returns: HTTP response """ - return get_response(api_.tilematrixsets(request)) + return execute_from_flask(tiles_api.tilematrixsets, request) @BLUEPRINT.route('/collections') @@ -305,8 +307,8 @@ def get_collection_tiles(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles( - request, collection_id)) + return execute_from_flask(tiles_api.get_collection_tiles, request, + collection_id) @BLUEPRINT.route('/collections//tiles/') @@ -320,8 +322,8 @@ def get_collection_tiles_metadata(collection_id=None, tileMatrixSetId=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles_metadata( - request, collection_id, tileMatrixSetId)) + return execute_from_flask(tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId) @BLUEPRINT.route('/collections//tiles/\ @@ -339,8 +341,9 @@ def get_collection_tiles_data(collection_id=None, tileMatrixSetId=None, :returns: HTTP response """ - return get_response(api_.get_collection_tiles_data( - request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol)) + return execute_from_flask( + tiles_api.get_collection_tiles_data, + request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol) @BLUEPRINT.route('/collections//map') diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 1ba7f8003..f5bd218de 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -55,6 +55,7 @@ import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -200,8 +201,9 @@ async def get_tilematrix_set(request: Request, tileMatrixSetId=None): if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.tilematrixset, request, tileMatrixSetId) + return await execute_from_starlette( + tiles_api.tilematrixset, request, tileMatrixSetId, + ) async def get_tilematrix_sets(request: Request): @@ -210,7 +212,7 @@ async def get_tilematrix_sets(request: Request): :returns: HTTP response """ - return await get_response(api_.tilematrixsets, request) + return await execute_from_starlette(tiles_api.tilematrixsets, request) async def collection_schema(request: Request, collection_id=None): @@ -256,8 +258,9 @@ async def get_collection_tiles(request: Request, collection_id=None): """ if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_tiles, request, collection_id) + + return await execute_from_starlette( + tiles_api.get_collection_tiles, request, collection_id) async def get_collection_tiles_metadata(request: Request, collection_id=None, @@ -274,8 +277,9 @@ async def get_collection_tiles_metadata(request: Request, collection_id=None, collection_id = request.path_params['collection_id'] if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.get_collection_tiles_metadata, request, + + return await execute_from_starlette( + tiles_api.get_collection_tiles_metadata, request, collection_id, tileMatrixSetId ) @@ -305,8 +309,8 @@ async def get_collection_items_tiles(request: Request, collection_id=None, tileRow = request.path_params['tileRow'] if 'tileCol' in request.path_params: tileCol = request.path_params['tileCol'] - return await get_response( - api_.get_collection_tiles_data, request, collection_id, + return await execute_from_starlette( + tiles_api.get_collection_tiles_data, request, collection_id, tileMatrixSetId, tile_matrix, tileRow, tileCol ) diff --git a/tests/test_api.py b/tests/test_api.py index 5b2b9881f..2e0e17040 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -51,7 +51,9 @@ from pygeoapi.api.processes import ( describe_processes, execute_process, delete_job, get_job_result, ) -from pygeoapi.api.tiles import get_collection_tiles +from pygeoapi.api.tiles import ( + get_collection_tiles, tilematrixset, tilematrixsets, +) from pygeoapi.util import (yaml_load, get_crs_from_uri, get_api_rules, get_base_url) @@ -631,8 +633,8 @@ def test_conformance(config, api_): def test_tilematrixsets(config, api_): - req = mock_request() - rsp_headers, code, response = api_.tilematrixsets(req) + req = mock_api_request() + rsp_headers, code, response = tilematrixsets(api_, req) root = json.loads(response) assert isinstance(root, dict) @@ -643,26 +645,22 @@ def test_tilematrixsets(config, api_): assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ in root['tileMatrixSets'][1]['uri'] - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.tilematrixsets(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.tilematrixsets(req) + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixsets(api_, req) assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] # No language requested: should be set to default from YAML assert rsp_headers['Content-Language'] == 'en-US' def test_tilematrixset(config, api_): - req = mock_request() + req = mock_api_request() enums = [e.value for e in TileMatrixSetEnum] enum = None for e in enums: enum = e.tileMatrixSet - rsp_headers, code, response = api_.tilematrixset(req, enum) + rsp_headers, code, response = tilematrixset(api_, req, enum) root = json.loads(response) assert isinstance(root, dict) @@ -671,11 +669,11 @@ def test_tilematrixset(config, api_): assert 'tileMatrices' in root assert len(root['tileMatrices']) == 30 - rsp_headers, code, response = api_.tilematrixset(req, 'foo') + rsp_headers, code, response = tilematrixset(api_, req, 'foo') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.tilematrixset(req, enum) + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixset(api_, req, enum) assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] # No language requested: should be set to default from YAML assert rsp_headers['Content-Language'] == 'en-US' From bbff4f9c041c73ff4085a99c53f717bddfa2ca59 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 17 Mar 2024 07:57:11 -0400 Subject: [PATCH 25/71] update features, records, coverages --- pygeoapi/api/__init__.py | 1541 +--------------- pygeoapi/api/coverages.py | 244 +++ pygeoapi/api/environmental_data_retrieval.py | 33 +- pygeoapi/api/itemtypes.py | 1688 ++++++++++++++++++ pygeoapi/api/maps.py | 75 +- pygeoapi/api/processes.py | 26 +- pygeoapi/api/stac.py | 8 +- pygeoapi/api/tiles.py | 107 +- pygeoapi/django_/views.py | 34 +- pygeoapi/flask_app.py | 85 +- pygeoapi/openapi.py | 472 +---- pygeoapi/starlette_app.py | 40 +- tests/test_api.py | 384 ++-- 13 files changed, 2525 insertions(+), 2212 deletions(-) create mode 100644 pygeoapi/api/coverages.py create mode 100644 pygeoapi/api/itemtypes.py diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 263b2a0a6..2fd768e1f 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -34,7 +34,9 @@ # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= -""" Root level code of pygeoapi, parsing content provided by web framework. + +""" +Root level code of pygeoapi, parsing content provided by web framework. Returns content from plugins and sets responses. """ @@ -48,34 +50,23 @@ import logging import re from typing import Any, Tuple, Union, Optional -import urllib.parse from dateutil.parser import parse as dateparse -from pygeofilter.parsers.ecql import parse as parse_ecql_text -from pygeofilter.parsers.cql_json import parse as parse_cql_json -from pyproj.exceptions import CRSError import pytz from pygeoapi import __version__, l10n -from pygeoapi.formatter.base import FormatterSerializationError -from pygeoapi.linked_data import (geojson2jsonld, jsonldify, - jsonldify_collection) +from pygeoapi.linked_data import jsonldify, jsonldify_collection from pygeoapi.log import setup_logger +from pygeoapi.plugin import load_plugin from pygeoapi.process.manager.base import get_manager -from pygeoapi.plugin import load_plugin, PLUGINS -from pygeoapi.provider.base import ( - ProviderGenericError, ProviderConnectionError, ProviderTypeError) - -from pygeoapi.models.cql import CQLModel -from pygeoapi.util import (dategetter, - UrlPrefetcher, - filter_dict_by_key_value, get_provider_by_type, - get_provider_default, get_typed_value, - render_j2_template, str2bool, - TEMPLATES, to_json, get_api_rules, get_base_url, - get_crs_from_uri, get_supported_crs_list, - modify_pygeofilter, CrsTransformSpec, - transform_bbox) +from pygeoapi.provider.base import ProviderConnectionError, ProviderTypeError + +from pygeoapi.util import ( + CrsTransformSpec, TEMPLATES, UrlPrefetcher, dategetter, + filter_dict_by_key_value, get_api_rules, get_base_url, + get_provider_by_type, get_provider_default, get_typed_value, + get_crs_from_uri, get_supported_crs_list, render_j2_template, to_json +) LOGGER = logging.getLogger(__name__) @@ -109,65 +100,14 @@ #: Locale used for system responses (e.g. exceptions) SYSTEM_LOCALE = l10n.Locale('en', 'US') -CONFORMANCE = { - 'common': [ - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' - ], - 'feature': [ - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', - 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa - 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa - 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', - 'http://www.opengis.net/spec/ogcapi-features-5/1.0/req/core-roles-features' # noqa - ], - 'coverage': [ - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' # noqa - ], - 'map': [ - 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' - ], - 'tile': [ - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' - ], - 'record': [ - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' - ], - 'process': [ - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30' - ], - 'edr': [ - 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' - ] -} +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' +] OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' @@ -180,11 +120,27 @@ DEFAULT_STORAGE_CRS = DEFAULT_CRS -def all_apis(): - # NOTE: this is a function and not a constant to avoid import loops - from . import environmental_data_retrieval, maps, processes, stac +def all_apis() -> dict: + """ + Return all supported API modules + + NOTE: this is a function and not a constant to avoid import loops + + :returns: `dict` of API provider type, API module + """ + + from . import (coverages, environmental_data_retrieval, itemtypes, maps, + processes, tiles, stac) - return [environmental_data_retrieval, maps, processes, stac] + return { + 'coverage': coverages, + 'edr': environmental_data_retrieval, + 'itemtypes': itemtypes, + 'map': maps, + 'process': processes, + 'tile': tiles, + 'stac': stac + } def pre_process(func): @@ -895,21 +851,31 @@ def conformance(self, :returns: tuple of headers, status code, content """ + apis_dict = all_apis() + if not request.is_valid(): return self.get_format_exception(request) - conformance_list = CONFORMANCE['common'] + conformance_list = CONFORMANCE_CLASSES for key, value in self.config['resources'].items(): if value['type'] == 'process': - conformance_list.extend(CONFORMANCE[value['type']]) + conformance_list.extend( + apis_dict['process'].CONFORMANCE_CLASSES) else: for provider in value['providers']: - if provider['type'] in CONFORMANCE: - conformance_list.extend(CONFORMANCE[provider['type']]) + if provider['type'] in apis_dict: + conformance_list.extend( + apis_dict[provider['type']].CONFORMANCE_CLASSES) + if provider['type'] == 'feature': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_FEATURES) # noqa + if provider['type'] == 'record': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_RECORDS) conformance = { - 'conformsTo': list(set(conformance_list)) + 'conformsTo': sorted(list(set(conformance_list))) } headers = request.get_response_headers(**self.api_headers) @@ -1317,1401 +1283,6 @@ def describe_collections(self, request: Union[APIRequest, Any], return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - @gzip - @pre_process - @jsonldify - def get_collection_schema( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a collection schema - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection schema') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - schema = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/schema' - } - - if p.type != 'coverage': - schema['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - schema['properties'][k] = v - - if k == p.id_field: - schema['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - schema['properties'][k]['x-ogc-role'] = 'primary-instant' - - if request.format == F_HTML: # render - schema['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - schema['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/schema.html', - schema, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(schema, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_collection_queryables(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection queryables - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection queryables') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - queryables = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/queryables' - } - - if p.fields: - queryables['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - show_field = False - if p.properties: - if k in p.properties: - show_field = True - else: - show_field = True - - if show_field: - queryables['properties'][k] = { - 'title': k, - 'type': v['type'] - } - if 'values' in v: - queryables['properties'][k]['enum'] = v['values'] - - if k == p.id_field: - queryables['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa - - if request.format == F_HTML: # render - queryables['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - queryables['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/queryables.html', - queryables, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(queryables, self.pretty_print) - - @gzip - @pre_process - def get_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', - 'offset', 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter', 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - q = request.params.get('q') or None - - LOGGER.debug('Loading provider') - - provider_def = None - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - LOGGER.debug('Processing bbox-crs parameter') - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - # Validate bbox-crs parameter - if len(bbox) == 0: - msg = 'bbox-crs specified without bbox parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - if len(bbox_crs) == 0: - msg = 'bbox-crs specified but is empty' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa - if bbox_crs not in supported_crs_list: - msg = f'bbox-crs {bbox_crs} not supported for this collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - elif len(bbox) > 0: - # bbox but no bbox-crs parm: assume bbox is in default CRS - bbox_crs = DEFAULT_CRS - - # Transform bbox to storageCRS - # when bbox-crs different from storageCRS. - if len(bbox) > 0: - try: - # Get a pyproj CRS instance for the Collection's Storage CRS - storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - - # Do the (optional) Transform to the Storage CRS - bbox = transform_bbox(bbox, bbox_crs, storage_crs) - except CRSError as e: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', str(e)) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k in list(p.fields.keys()): - LOGGER.debug(f'Adding property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('processing filter parameter') - cql_text = request.params.get('filter') - if cql_text is not None: - try: - filter_ = parse_ecql_text(cql_text) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs_uri, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field'), - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {cql_text}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - filter_ = None - - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - # Currently only cql-text is handled, but it is optional - if filter_lang not in [None, 'cql-text']: - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - # Get provider locale (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - if provider_type == 'feature': - LOGGER.debug(f'crs: {query_crs_uri}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {properties}') - LOGGER.debug(f'select properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'language: {prv_locale}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'cql_text: {cql_text}') - LOGGER.debug(f'filter_: {filter_}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs_uri}') - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, skip_geometry=skip_geometry, - select_properties=select_properties, - crs_transform_spec=crs_transform_spec, - q=q, language=prv_locale, filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - serialized_query_params = '' - for k, v in request.params.items(): - if k not in ('f', 'offset'): - serialized_query_params += '&' - serialized_query_params += urllib.parse.quote(k, safe='') - serialized_query_params += '=' - serialized_query_params += urllib.parse.quote(str(v), safe=',') - - # TODO: translate titles - uri = f'{self.get_collections_url()}/{dataset}/items' - content['links'] = [{ - 'type': 'application/geo+json', - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}{serialized_query_params}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}{serialized_query_params}' - }] - - if offset > 0: - prev = max(0, offset - limit) - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'prev', - 'title': 'items (prev)', - 'href': f'{uri}?offset={prev}{serialized_query_params}' - }) - - if 'numberMatched' in content: - if content['numberMatched'] > (limit + offset): - next_ = offset + limit - next_href = f'{uri}?offset={next_}{serialized_query_params}' - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'next', - 'title': 'items (next)', - 'href': next_href - }) - - content['links'].append( - { - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate( - collections[dataset]['title'], request.locale), - 'rel': 'collection', - 'href': uri - }) - - content['timeStamp'] = datetime.utcnow().strftime( - '%Y-%m-%dT%H:%M:%S.%fZ') - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - # For constructing proper URIs to items - - content['items_path'] = uri - content['dataset_path'] = '/'.join(uri.split('/')[:-1]) - content['collections_path'] = self.get_collections_url() - - content['offset'] = offset - - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - # If title exists, use it as id in html templates - content['id_field'] = content['title_field'] - content = render_j2_template(self.tpl_config, - 'collections/items/index.html', - content, request.locale) - return headers, HTTPStatus.OK, content - elif request.format == 'csv': # render - formatter = load_plugin('formatter', - {'name': 'CSV', 'geom': True}) - - try: - content = formatter.write( - data=content, - options={ - 'provider_def': get_provider_by_type( - collections[dataset]['providers'], - 'feature') - } - ) - except FormatterSerializationError as err: - LOGGER.error(err) - msg = 'Error serializing output' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) - - headers['Content-Type'] = formatter.mimetype - - if p.filename is None: - filename = f'{dataset}.csv' - else: - filename = f'{p.filename}' - - cd = f'attachment; filename="{filename}"' - headers['Content-Disposition'] = cd - - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, id_field=(p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def post_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection or filter an item - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - request_headers = request.headers - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', - 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Invalid collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - val = request.params.get('q') - - q = None - if val is not None: - q = val - - LOGGER.debug('Loading provider') - - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - try: - p = load_plugin('provider', provider_def) - except ProviderGenericError as err: - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k not in p.fields.keys(): - msg = f'unknown query parameter: {k}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif k not in reserved_fieldnames and k in p.fields.keys(): - LOGGER.debug(f'Add property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - if filter_lang != 'cql-json': # @TODO add check from the configuration - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs}') - - LOGGER.debug('Processing headers') - - LOGGER.debug('Processing request content-type header') - if (request_headers.get( - 'Content-Type') or request_headers.get( - 'content-type')) != 'application/query-cql-json': - msg = ('Invalid body content-type') - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidHeaderValue', msg) - - LOGGER.debug('Processing body') - - if not request.data: - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - filter_ = None - try: - # Parse bytes data, if applicable - data = request.data.decode() - LOGGER.debug(data) - except UnicodeDecodeError as err: - LOGGER.error(err) - msg = 'Unicode error in data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - # FIXME: remove testing backend in use once CQL support is normalized - if p.name == 'PostgreSQL': - LOGGER.debug('processing PostgreSQL CQL_JSON data') - try: - filter_ = parse_cql_json(data) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field') - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - LOGGER.debug('processing Elasticsearch CQL_JSON data') - try: - filter_ = CQLModel.model_validate_json(data) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, - select_properties=select_properties, - skip_geometry=skip_geometry, - q=q, - filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def manage_collection_item( - self, request: Union[APIRequest, Any], - action, dataset, identifier=None) -> Tuple[dict, int, str]: - """ - Adds an item to a collection - - :param request: A request object - :param action: an action among 'create', 'update', 'delete', 'options' - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'options': - headers['Allow'] = 'HEAD, GET' - if p.editable: - if identifier is None: - headers['Allow'] += ', POST' - else: - headers['Allow'] += ', PUT, DELETE' - return headers, HTTPStatus.OK, '' - - if not p.editable: - msg = 'Collection is not editable' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action in ['create', 'update'] and not request.data: - msg = 'No data found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'create': - LOGGER.debug('Creating item') - try: - identifier = p.create(request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - headers['Location'] = f'{self.get_collections_url()}/{dataset}/items/{identifier}' # noqa - - return headers, HTTPStatus.CREATED, '' - - if action == 'update': - LOGGER.debug('Updating item') - try: - _ = p.update(identifier, request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.NO_CONTENT, '' - - if action == 'delete': - LOGGER.debug('Deleting item') - try: - _ = p.delete(identifier) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, '' - - @gzip - @pre_process - def get_collection_item(self, request: Union[APIRequest, Any], - dataset, identifier) -> Tuple[dict, int, str]: - """ - Get a single collection item - - :param request: A request object - :param dataset: dataset name - :param identifier: item identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Processing query parameters') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - try: - LOGGER.debug(f'Fetching id {identifier}') - content = p.get( - identifier, - language=prv_locale, - crs_transform_spec=crs_transform_spec, - ) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if content is None: - msg = 'identifier not found' - return self.get_exception(HTTPStatus.BAD_REQUEST, headers, - request.format, 'NotFound', msg) - - uri = content['properties'].get(p.uri_field) if p.uri_field else \ - f'{self.get_collections_url()}/{dataset}/items/{identifier}' - - if 'links' not in content: - content['links'] = [] - - content['links'].extend([{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }, { - 'rel': request.get_linkrel(F_JSON), - 'type': 'application/geo+json', - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}' - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}' - }, { - 'rel': 'collection', - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate(collections[dataset]['title'], - request.locale), - 'href': f'{self.get_collections_url()}/{dataset}' - }]) - - link_request_format = ( - request.format if request.format is not None else F_JSON - ) - if 'prev' in content: - content['links'].append({ - 'rel': 'prev', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa - }) - if 'next' in content: - content['links'].append({ - 'rel': 'next', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa - }) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - content['title'] = l10n.translate(collections[dataset]['title'], - request.locale) - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - content['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/items/item.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, uri, (p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @pre_process - @jsonldify - def get_collection_coverage(self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection coverage - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - query_args = {} - format_ = request.format or F_JSON - - # Force response content type and language (en-US only) headers - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage') - - p = load_plugin('provider', collection_def) - except KeyError: - msg = 'collection does not exist' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, - 'InvalidParameterValue', msg) - - query_args['bbox'] = bbox - - LOGGER.debug('Processing bbox-crs parameter') - - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - query_args['bbox_crs'] = bbox_crs - - LOGGER.debug('Processing datetime parameter') - - datetime_ = request.params.get('datetime') - - try: - datetime_ = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['datetime_'] = datetime_ - query_args['format_'] = format_ - - properties = request.params.get('properties') - if properties: - LOGGER.debug('Processing properties parameter') - query_args['properties'] = [rs for - rs in properties.split(',') if rs] - LOGGER.debug(f"Fields: {query_args['properties']}") - - for a in query_args['properties']: - if a not in p.fields: - msg = 'Invalid field specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if 'subset' in request.params: - LOGGER.debug('Processing subset parameter') - try: - subsets = validate_subset(request.params['subset'] or '') - except (AttributeError, ValueError) as err: - msg = f'Invalid subset: {err}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if not set(subsets.keys()).issubset(p.axes): - msg = 'Invalid axis name' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['subsets'] = subsets - LOGGER.debug(f"Subsets: {query_args['subsets']}") - - LOGGER.debug('Querying coverage') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - if format_ == mt: # native format - if p.filename is not None: - cd = f'attachment; filename="{p.filename}"' - headers['Content-Disposition'] = cd - - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ == F_JSON: - headers['Content-Type'] = 'application/prs.coverage+json' - return headers, HTTPStatus.OK, to_json(data, self.pretty_print) - else: - return self.get_format_exception(request) - - @gzip - def get_collection_map_legend( - self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map legend - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - format_ = 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Generating legend') - try: - data = p.get_legend(style, request.params.get('f', 'png')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - def get_exception(self, status, headers, format_, code, description) -> Tuple[dict, int, str]: """ diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py new file mode 100644 index 000000000..63e52812c --- /dev/null +++ b/pygeoapi/api/coverages.py @@ -0,0 +1,244 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError, ProviderTypeError +from pygeoapi.util import ( + filter_dict_by_key_value, get_provider_by_type, to_json +) + +from . import ( + APIRequest, API, F_JSON, SYSTEM_LOCALE, validate_bbox, validate_datetime, + validate_subset +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' +] + + +def get_collection_coverage( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection coverage + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + query_args = {} + format_ = request.format or F_JSON + + # Force response content type and language (en-US only) headers + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage') + + p = load_plugin('provider', collection_def) + except KeyError: + msg = 'collection does not exist' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, + 'InvalidParameterValue', msg) + + query_args['bbox'] = bbox + + LOGGER.debug('Processing bbox-crs parameter') + + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + query_args['bbox_crs'] = bbox_crs + + LOGGER.debug('Processing datetime parameter') + + datetime_ = request.params.get('datetime') + + try: + datetime_ = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['datetime_'] = datetime_ + query_args['format_'] = format_ + + properties = request.params.get('properties') + if properties: + LOGGER.debug('Processing properties parameter') + query_args['properties'] = [rs for + rs in properties.split(',') if rs] + LOGGER.debug(f"Fields: {query_args['properties']}") + + for a in query_args['properties']: + if a not in p.fields: + msg = 'Invalid field specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if 'subset' in request.params: + LOGGER.debug('Processing subset parameter') + try: + subsets = validate_subset(request.params['subset'] or '') + except (AttributeError, ValueError) as err: + msg = f'Invalid subset: {err}' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if not set(subsets.keys()).issubset(p.axes): + msg = 'Invalid axis name' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['subsets'] = subsets + LOGGER.debug(f"Subsets: {query_args['subsets']}") + + LOGGER.debug('Querying coverage') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + if format_ == mt: # native format + if p.filename is not None: + cd = f'attachment; filename="{p.filename}"' + headers['Content-Disposition'] = cd + + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ == F_JSON: + headers['Content-Type'] = 'application/prs.coverage+json' + return headers, HTTPStatus.OK, to_json(data, api.pretty_print) + else: + return api.get_format_exception(request) + + +def get_oas_30(cfg: dict, locale: str) -> dict: + from pygeoapi.openapi import OPENAPI_YAML + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in collections.items(): + try: + load_plugin('provider', get_provider_by_type( + collections[k]['providers'], 'coverage')) + except ProviderTypeError: + LOGGER.debug('collection is not coverage based') + continue + + coverage_path = f'/collections/{k}/coverage' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[coverage_path] = { + 'get': { + 'summary': f'Get {title} coverage', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Coverage', + 'parameters': [ + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': '#/components/parameters/bbox-crs'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + return {'tags': [], 'paths': paths} diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index d708cc0b2..8c01895d9 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -8,7 +8,7 @@ # Ricardo Garcia Silva # Bernhard Mallinger # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Ricardo Garcia Silva @@ -38,8 +38,8 @@ # ================================================================= -import logging from http import HTTPStatus +import logging from typing import Tuple from shapely.errors import WKTReadingError @@ -49,21 +49,21 @@ from pygeoapi.provider.base import ProviderGenericError from pygeoapi.util import ( filter_providers_by_type, get_provider_by_type, render_j2_template, - to_json, filter_dict_by_key_value, -) - -from . import ( - APIRequest, API, F_HTML, validate_datetime, validate_bbox + to_json, filter_dict_by_key_value ) +from . import APIRequest, API, F_HTML, validate_datetime, validate_bbox LOGGER = logging.getLogger(__name__) +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' +] -def get_collection_edr_query( - api: API, request: APIRequest, dataset, instance, query_type, - location_id=None -) -> Tuple[dict, int, str]: + +def get_collection_edr_query(api: API, request: APIRequest, + dataset, instance, query_type, + location_id=None) -> Tuple[dict, int, str]: """ Queries collection EDR @@ -209,19 +209,20 @@ def get_collection_edr_query( def get_oas_30(cfg: dict, locale: str) -> dict: from pygeoapi.openapi import OPENAPI_YAML + LOGGER.debug('setting up edr endpoints') + paths = {} collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') for k, v in collections.items(): - LOGGER.debug('setting up edr endpoints') edr_extension = filter_providers_by_type( collections[k]['providers'], 'edr') - collection_name_path = f'/collections/{k}' - if edr_extension: + collection_name_path = f'/collections/{k}' + ep = load_plugin('provider', edr_extension) edr_query_endpoints = [] @@ -246,7 +247,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: spatial_parameter = f"{eqe['qt']}Coords" paths[eqe['path']] = { 'get': { - 'summary': f"query {v['description']} by {eqe['qt']}", # noqa + 'summary': f"query {v['description']} by {eqe['qt']}", 'description': v['description'], 'tags': [k], 'operationId': eqe['op_id'], @@ -292,7 +293,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: } paths[f'{collection_name_path}/locations/{{locId}}'] = { 'get': { - 'summary': f"query {v['description']} by location", # noqa + 'summary': f"query {v['description']} by location", 'description': v['description'], 'tags': [k], 'operationId': f'queryLOCATIONSBYID{k.capitalize()}', diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py new file mode 100644 index 000000000..ac7bcea0e --- /dev/null +++ b/pygeoapi/api/itemtypes.py @@ -0,0 +1,1688 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime +from http import HTTPStatus +import logging +from typing import Any, Tuple, Union, Optional +import urllib.parse + +from pygeofilter.parsers.ecql import parse as parse_ecql_text +from pygeofilter.parsers.cql_json import parse as parse_cql_json +from pyproj.exceptions import CRSError + +from pygeoapi import l10n +from pygeoapi.formatter.base import FormatterSerializationError +from pygeoapi.linked_data import geojson2jsonld +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError, SchemaType) + +from pygeoapi.models.cql import CQLModel +from pygeoapi.util import (CrsTransformSpec, filter_providers_by_type, + filter_dict_by_key_value, get_crs_from_uri, + get_provider_by_type, get_supported_crs_list, + modify_pygeofilter, render_j2_template, str2bool, + to_json, transform_bbox) + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, + validate_bbox, validate_datetime +) + +LOGGER = logging.getLogger(__name__) + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + +CONFORMANCE_CLASSES_FEATURES = [ + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', + 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa + 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/req/core-roles-features' +] + +CONFORMANCE_CLASSES_RECORDS = [ + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' +] + + +def get_collection_schema( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Returns a collection schema + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return api.get_format_exception(request) + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection schema') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + schema = { + 'type': 'object', + 'title': l10n.translate( + api.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{api.get_collections_url()}/{dataset}/schema' + } + + if p.type != 'coverage': + schema['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + schema['properties'][k] = v + + if k == p.id_field: + schema['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + schema['properties'][k]['x-ogc-role'] = 'primary-instant' + + if request.format == F_HTML: # render + schema['title'] = l10n.translate( + api.config['resources'][dataset]['title'], request.locale) + + schema['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/schema.html', + schema, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(schema, api.pretty_print) + + +def get_collection_queryables(api: API, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection queryables + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return api.get_format_exception(request) + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection queryables') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + queryables = { + 'type': 'object', + 'title': l10n.translate( + api.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{api.get_collections_url()}/{dataset}/queryables' + } + + if p.fields: + queryables['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + show_field = False + if p.properties: + if k in p.properties: + show_field = True + else: + show_field = True + + if show_field: + queryables['properties'][k] = { + 'title': k, + 'type': v['type'] + } + if 'values' in v: + queryables['properties'][k]['enum'] = v['values'] + + if k == p.id_field: + queryables['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa + + if request.format == F_HTML: # render + queryables['title'] = l10n.translate( + api.config['resources'][dataset]['title'], request.locale) + + queryables['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/queryables.html', + queryables, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(queryables, api.pretty_print) + + +def get_collection_items( + api: API, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Queries collection + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', + 'offset', 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter', 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(api.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + q = request.params.get('q') or None + + LOGGER.debug('Loading provider') + + provider_def = None + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + LOGGER.debug('Processing bbox-crs parameter') + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + # Validate bbox-crs parameter + if len(bbox) == 0: + msg = 'bbox-crs specified without bbox parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + if len(bbox_crs) == 0: + msg = 'bbox-crs specified but is empty' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa + if bbox_crs not in supported_crs_list: + msg = f'bbox-crs {bbox_crs} not supported for this collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + elif len(bbox) > 0: + # bbox but no bbox-crs parm: assume bbox is in default CRS + bbox_crs = DEFAULT_CRS + + # Transform bbox to storageCRS + # when bbox-crs different from storageCRS. + if len(bbox) > 0: + try: + # Get a pyproj CRS instance for the Collection's Storage CRS + storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + + # Do the (optional) Transform to the Storage CRS + bbox = transform_bbox(bbox, bbox_crs, storage_crs) + except CRSError as e: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', str(e)) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k in list(p.fields.keys()): + LOGGER.debug(f'Adding property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) + LOGGER.debug('processing filter parameter') + cql_text = request.params.get('filter') + if cql_text is not None: + try: + filter_ = parse_ecql_text(cql_text) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs_uri, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field'), + ) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {cql_text}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + filter_ = None + + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + # Currently only cql-text is handled, but it is optional + if filter_lang not in [None, 'cql-text']: + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + # Get provider locale (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + if provider_type == 'feature': + LOGGER.debug(f'crs: {query_crs_uri}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {properties}') + LOGGER.debug(f'select properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'language: {prv_locale}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'cql_text: {cql_text}') + LOGGER.debug(f'filter_: {filter_}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs_uri}') + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, skip_geometry=skip_geometry, + select_properties=select_properties, + crs_transform_spec=crs_transform_spec, + q=q, language=prv_locale, filterq=filter_) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + # TODO: translate titles + uri = f'{api.get_collections_url()}/{dataset}/items' + content['links'] = [{ + 'type': 'application/geo+json', + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as GeoJSON', + 'href': f'{uri}?f={F_JSON}{serialized_query_params}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}{serialized_query_params}' + }] + + if offset > 0: + prev = max(0, offset - limit) + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'prev', + 'title': 'items (prev)', + 'href': f'{uri}?offset={prev}{serialized_query_params}' + }) + + if 'numberMatched' in content: + if content['numberMatched'] > (limit + offset): + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'next', + 'title': 'items (next)', + 'href': next_href + }) + + content['links'].append( + { + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate( + collections[dataset]['title'], request.locale), + 'rel': 'collection', + 'href': uri + }) + + content['timeStamp'] = datetime.utcnow().strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + # For constructing proper URIs to items + + content['items_path'] = uri + content['dataset_path'] = '/'.join(uri.split('/')[:-1]) + content['collections_path'] = api.get_collections_url() + + content['offset'] = offset + + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + # If title exists, use it as id in html templates + content['id_field'] = content['title_field'] + content = render_j2_template(api.tpl_config, + 'collections/items/index.html', + content, request.locale) + return headers, HTTPStatus.OK, content + elif request.format == 'csv': # render + formatter = load_plugin('formatter', + {'name': 'CSV', 'geom': True}) + + try: + content = formatter.write( + data=content, + options={ + 'provider_def': get_provider_by_type( + collections[dataset]['providers'], + 'feature') + } + ) + except FormatterSerializationError as err: + LOGGER.error(err) + msg = 'Error serializing output' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + headers['Content-Type'] = formatter.mimetype + + if p.filename is None: + filename = f'{dataset}.csv' + else: + filename = f'{p.filename}' + + cd = f'attachment; filename="{filename}"' + headers['Content-Disposition'] = cd + + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, id_field=(p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def post_collection_items( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Queries collection or filter an item + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + request_headers = request.headers + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', + 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Invalid collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(api.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + val = request.params.get('q') + + q = None + if val is not None: + q = val + + LOGGER.debug('Loading provider') + + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + try: + p = load_plugin('provider', provider_def) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k not in p.fields.keys(): + msg = f'unknown query parameter: {k}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif k not in reserved_fieldnames and k in p.fields.keys(): + LOGGER.debug(f'Add property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs = request.params.get('filter-crs', DEFAULT_CRS) + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + if filter_lang != 'cql-json': # @TODO add check from the configuration + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs}') + + LOGGER.debug('Processing headers') + + LOGGER.debug('Processing request content-type header') + if (request_headers.get( + 'Content-Type') or request_headers.get( + 'content-type')) != 'application/query-cql-json': + msg = ('Invalid body content-type') + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidHeaderValue', msg) + + LOGGER.debug('Processing body') + + if not request.data: + msg = 'missing request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + filter_ = None + try: + # Parse bytes data, if applicable + data = request.data.decode() + LOGGER.debug(data) + except UnicodeDecodeError as err: + LOGGER.error(err) + msg = 'Unicode error in data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + # FIXME: remove testing backend in use once CQL support is normalized + if p.name == 'PostgreSQL': + LOGGER.debug('processing PostgreSQL CQL_JSON data') + try: + filter_ = parse_cql_json(data) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field') + ) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {data}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + LOGGER.debug('processing Elasticsearch CQL_JSON data') + try: + filter_ = CQLModel.model_validate_json(data) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {data}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, + select_properties=select_properties, + skip_geometry=skip_geometry, + q=q, + filterq=filter_) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def manage_collection_item( + api: API, request: APIRequest, + action, dataset, identifier=None) -> Tuple[dict, int, str]: + """ + Adds an item to a collection + + :param request: A request object + :param action: an action among 'create', 'update', 'delete', 'options' + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'options': + headers['Allow'] = 'HEAD, GET' + if p.editable: + if identifier is None: + headers['Allow'] += ', POST' + else: + headers['Allow'] += ', PUT, DELETE' + return headers, HTTPStatus.OK, '' + + if not p.editable: + msg = 'Collection is not editable' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action in ['create', 'update'] and not request.data: + msg = 'No data found' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'create': + LOGGER.debug('Creating item') + try: + identifier = p.create(request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + headers['Location'] = f'{api.get_collections_url()}/{dataset}/items/{identifier}' # noqa + + return headers, HTTPStatus.CREATED, '' + + if action == 'update': + LOGGER.debug('Updating item') + try: + _ = p.update(identifier, request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.NO_CONTENT, '' + + if action == 'delete': + LOGGER.debug('Deleting item') + try: + _ = p.delete(identifier) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, '' + + +def get_collection_item(api: API, request: APIRequest, + dataset, identifier) -> Tuple[dict, int, str]: + """ + Get a single collection item + + :param request: A request object + :param dataset: dataset name + :param identifier: item identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Processing query parameters') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + try: + LOGGER.debug(f'Fetching id {identifier}') + content = p.get( + identifier, + language=prv_locale, + crs_transform_spec=crs_transform_spec, + ) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if content is None: + msg = 'identifier not found' + return api.get_exception(HTTPStatus.BAD_REQUEST, headers, + request.format, 'NotFound', msg) + + uri = content['properties'].get(p.uri_field) if p.uri_field else \ + f'{api.get_collections_url()}/{dataset}/items/{identifier}' + + if 'links' not in content: + content['links'] = [] + + content['links'].extend([{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{api.base_url}?f={F_JSON}" + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{api.base_url}?f={F_HTML}" + }, { + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/geo+json', + 'title': 'This document as GeoJSON', + 'href': f'{uri}?f={F_JSON}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}' + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}' + }, { + 'rel': 'collection', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate(collections[dataset]['title'], + request.locale), + 'href': f'{api.get_collections_url()}/{dataset}' + }]) + + link_request_format = ( + request.format if request.format is not None else F_JSON + ) + if 'prev' in content: + content['links'].append({ + 'rel': 'prev', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa + }) + if 'next' in content: + content['links'].append({ + 'rel': 'next', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa + }) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + content['title'] = l10n.translate(collections[dataset]['title'], + request.locale) + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + content['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/items/item.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, uri, (p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +@staticmethod +def create_crs_transform_spec( + config: dict, query_crs_uri: Optional[str] = None) -> Union[None, CrsTransformSpec]: # noqa + """ + Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + +@staticmethod +def set_content_crs_header( + headers: dict, config: dict, query_crs_uri: Optional[str] = None): + """ + Set the *Content-Crs* header in responses from providers of Feature type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :returns: None + """ + + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +def get_oas_30(cfg: dict, locale: str) -> dict: + from pygeoapi.openapi import OPENAPI_YAML + + properties = { + 'name': 'properties', + 'in': 'query', + 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + } + + LOGGER.debug('setting up collection endpoints') + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in collections.items(): + try: + ptype = None + + if filter_providers_by_type( + collections[k]['providers'], 'feature'): + ptype = 'feature' + + if filter_providers_by_type( + collections[k]['providers'], 'record'): + ptype = 'record' + + p = load_plugin('provider', get_provider_by_type( + collections[k]['providers'], ptype)) + + collection_name_path = f'/collections/{k}' + items_path = f'/collections/{k}/items' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + coll_properties = deepcopy(properties) + + coll_properties['schema']['items']['enum'] = list(p.fields.keys()) + + paths[items_path] = { + 'get': { + 'summary': f'Get {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Features', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/bbox-crs'}, + {'properties': coll_properties}, + {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa + {'$ref': '#/components/parameters/skipGeometry'}, + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa + {'$ref': '#/components/parameters/offset'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Features', + 'responses': { + '200': {'description': 'options response'} + } + } + } + + if p.editable: + LOGGER.debug('Provider is editable; adding post') + + paths[items_path]['post'] = { + 'summary': f'Add {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'add{k.capitalize()}Features', + 'requestBody': { + 'description': 'Adds item to collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '201': {'description': 'Successful creation'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.create) + paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + if ptype == 'record': + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) + if p.fields: + schema_path = f'{collection_name_path}/schema' + + paths[schema_path] = { + 'get': { + 'summary': f'Get {title} schema', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Queryables', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + queryables_path = f'{collection_name_path}/queryables' + + paths[queryables_path] = { + 'get': { + 'summary': f'Get {title} queryables', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Queryables', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + if p.time_field is not None: + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + for field, type_ in p.fields.items(): + + if p.properties and field not in p.properties: + LOGGER.debug('Provider specified not to advertise property') # noqa + continue + + if field == 'q' and ptype == 'record': + LOGGER.debug('q parameter already declared, skipping') + continue + + if type_ == 'date': + schema = { + 'type': 'string', + 'format': 'date' + } + elif type_ == 'float': + schema = { + 'type': 'number', + 'format': 'float' + } + elif type_ == 'long': + schema = { + 'type': 'integer', + 'format': 'int64' + } + else: + schema = type_ + + path_ = f'{collection_name_path}/items' + paths[path_]['get']['parameters'].append({ + 'name': field, + 'in': 'query', + 'required': False, + 'schema': schema, + 'style': 'form', + 'explode': False + }) + + paths[f'{collection_name_path}/items/{{featureId}}'] = { + 'get': { + 'summary': f'Get {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'responses': { + '200': {'description': 'options response'} + } + } + } + + try: + schema_ref = p.get_schema() + paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa + 'content': { + schema_ref[0]: { + 'schema': schema_ref[1] + } + } + } + except Exception as err: + LOGGER.debug(err) + + if p.editable: + LOGGER.debug('Provider is editable; adding put/delete') + put_path = f'{collection_name_path}/items/{{featureId}}' # noqa + paths[put_path]['put'] = { # noqa + 'summary': f'Update {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'update{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'requestBody': { + 'description': 'Updates item in collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.replace) + paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { # noqa + 'summary': f'Delete {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'delete{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + ], + 'responses': { + '200': {'description': 'Successful delete'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + except ProviderTypeError: + LOGGER.debug('collection is not feature/item based') + + return {'tags': [], 'paths': {}} diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index 7f5db7dae..beb8b695c 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -8,7 +8,7 @@ # Ricardo Garcia Silva # Bernhard Mallinger # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Ricardo Garcia Silva @@ -39,8 +39,8 @@ from copy import deepcopy -import logging from http import HTTPStatus +import logging from typing import Tuple from pygeoapi.openapi import get_oas_30_parameters @@ -48,14 +48,17 @@ from pygeoapi.provider.base import ProviderGenericError from pygeoapi.util import ( get_provider_by_type, to_json, filter_providers_by_type, - filter_dict_by_key_value, + filter_dict_by_key_value ) from . import APIRequest, API, validate_datetime - LOGGER = logging.getLogger(__name__) +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' +] + def get_collection_map(api: API, request: APIRequest, dataset, style=None) -> Tuple[dict, int, str]: @@ -180,9 +183,71 @@ def get_collection_map(api: API, request: APIRequest, data, api.pretty_print) +def get_collection_map_legend(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map legend + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + format_ = 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Generating legend') + try: + data = p.get_legend(style, request.params.get('f', 'png')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + def get_oas_30(cfg: dict, locale: str) -> dict: from pygeoapi.openapi import OPENAPI_YAML + LOGGER.debug('setting up maps endpoints') + paths = {} collections = filter_dict_by_key_value(cfg['resources'], @@ -190,8 +255,6 @@ def get_oas_30(cfg: dict, locale: str) -> dict: parameters = get_oas_30_parameters(cfg, locale) for k, v in collections.items(): - - LOGGER.debug('setting up maps endpoints') map_extension = filter_providers_by_type( collections[k]['providers'], 'map') diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 0b07e47bd..2e4dd35ea 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -8,7 +8,7 @@ # Ricardo Garcia Silva # Bernhard Mallinger # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Ricardo Garcia Silva @@ -40,9 +40,9 @@ from copy import deepcopy from datetime import datetime, timezone -import logging from http import HTTPStatus import json +import logging from typing import Tuple from pygeoapi import l10n @@ -50,9 +50,7 @@ json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, to_json, DATETIME_FORMAT) from pygeoapi.process.base import ( - JobNotFoundError, - JobResultNotFoundError, - ProcessorExecuteError, + JobNotFoundError, JobResultNotFoundError, ProcessorExecuteError ) from pygeoapi.process.manager.base import get_manager @@ -60,9 +58,15 @@ APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, ) - LOGGER = logging.getLogger(__name__) +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30' +] + def describe_processes(api: API, request: APIRequest, process=None) -> Tuple[dict, int, str]: @@ -534,6 +538,8 @@ def delete_job( def get_oas_30(cfg: dict, locale_: str): from pygeoapi.openapi import OPENAPI_YAML + LOGGER.debug('setting up processes endpoints') + oas = {'tags': []} paths = {} @@ -569,7 +575,7 @@ def get_oas_30(cfg: dict, locale_: str): process_name_path = f'/processes/{name}' tag = { 'name': name, - 'description': md_desc, # noqa + 'description': md_desc, 'externalDocs': {} } for link in p.metadata.get('links', []): @@ -666,7 +672,7 @@ def get_oas_30(cfg: dict, locale_: str): 'responses': { '200': {'$ref': '#/components/responses/200'}, '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa + 'default': {'$ref': '#/components/responses/default'} } }, 'delete': { @@ -680,7 +686,7 @@ def get_oas_30(cfg: dict, locale_: str): 'responses': { '204': {'$ref': '#/components/responses/204'}, '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa + 'default': {'$ref': '#/components/responses/default'} } }, } @@ -698,7 +704,7 @@ def get_oas_30(cfg: dict, locale_: str): 'responses': { '200': {'$ref': '#/components/responses/200'}, '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa + 'default': {'$ref': '#/components/responses/default'} } } } diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index ddf4d251d..7ccf14cda 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -8,7 +8,7 @@ # Ricardo Garcia Silva # Bernhard Mallinger # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Ricardo Garcia Silva @@ -38,19 +38,19 @@ # ================================================================= -import logging from http import HTTPStatus +import logging from typing import Tuple from pygeoapi import l10n from pygeoapi.plugin import load_plugin from pygeoapi.provider.base import ( - ProviderConnectionError, ProviderNotFoundError, + ProviderConnectionError, ProviderNotFoundError ) from pygeoapi.util import ( get_provider_by_type, to_json, filter_dict_by_key_value, - render_j2_template, + render_j2_template ) from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 3f1aa85a3..75cc80c9d 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -8,7 +8,7 @@ # Ricardo Garcia Silva # Bernhard Mallinger # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Ricardo Garcia Silva @@ -47,20 +47,29 @@ from pygeoapi.models.provider.base import (TilesMetadataFormat, TileMatrixSetEnum) from pygeoapi.provider.base import ( - ProviderGenericError, ProviderTypeError, + ProviderGenericError, ProviderTypeError ) + from pygeoapi.util import ( get_provider_by_type, to_json, filter_dict_by_key_value, - render_j2_template, + filter_providers_by_type, render_j2_template ) from . import ( APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML, SYSTEM_LOCALE, F_JSONLD ) - LOGGER = logging.getLogger(__name__) +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' +] + def get_collection_tiles(api: API, request: APIRequest, dataset=None) -> Tuple[dict, int, str]: @@ -114,7 +123,7 @@ def get_collection_tiles(api: API, request: APIRequest, 'type': FORMAT_TYPES[F_JSONLD], 'rel': request.get_linkrel(F_JSONLD), 'title': 'This document as RDF (JSON-LD)', - 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' }) tiles['links'].append({ 'type': FORMAT_TYPES[F_HTML], @@ -427,4 +436,90 @@ def tilematrixset(api: API, return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) -# TODO: openapi +def get_oas_30(cfg: dict, locale: str) -> dict: + from pygeoapi.openapi import OPENAPI_YAML + + paths = {} + + LOGGER.debug('setting up tiles endpoints') + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in collections.items(): + tile_extension = filter_providers_by_type( + collections[k]['providers'], 'tile') + + if tile_extension: + tp = load_plugin('provider', tile_extension) + + tiles_path = f'/collections/{k}/tiles' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[tiles_path] = { + 'get': { + 'summary': f'Fetch a {title} tiles description', + 'description': description, + 'tags': [k], + 'operationId': f'describe{k.capitalize()}Tiles', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Tiles'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + tiles_data_path = f'{tiles_path}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa + + paths[tiles_data_path] = { + 'get': { + 'summary': f'Get a {title} tile', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Tiles', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa + { + 'name': 'f', + 'in': 'query', + 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [tp.format_type], + 'default': tp.format_type + }, + 'style': 'form', + 'explode': False + } + ], + 'responses': { + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + mimetype = tile_extension['format']['mimetype'] + paths[tiles_data_path]['get']['responses']['200'] = { + 'description': 'successful operation', + 'content': { + mimetype: { + 'schema': { + 'type': 'string', + 'format': 'binary' + } + } + } + } + + return {'tags': [], 'paths': paths} diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 63b503bdd..e279b14a7 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -8,7 +8,7 @@ # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 Luca Delucchi # Copyright (c) 2022 Krishna Lodha -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -41,7 +41,9 @@ from django.http import HttpRequest, HttpResponse from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages_api as coverages_api import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes_api as itemtypes_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api @@ -143,8 +145,8 @@ def collection_schema(request: HttpRequest, :returns: Django HTTP Response """ - response_ = _feed_response( - request, 'get_collection_schema', collection_id + response_ = execute_from_django( + itemtypes_api.get_collection_schema, request, collection_id ) response = _to_django_response(*response_) @@ -162,8 +164,8 @@ def collection_queryables(request: HttpRequest, :returns: Django HTTP Response """ - response_ = _feed_response( - request, 'get_collection_queryables', collection_id + response_ = execute_from_django( + itemtypes_api.get_collection_queryables, request, collection_id ) response = _to_django_response(*response_) @@ -181,22 +183,24 @@ def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: """ if request.method == 'GET': - response_ = _feed_response( + response_ = execute_from_django( + itemtypes_api.get_collection_items, request, - 'get_collection_items', collection_id, ) elif request.method == 'POST': if request.content_type is not None: if request.content_type == 'application/geo+json': - response_ = _feed_response(request, 'manage_collection_item', - request, 'create', collection_id) + response_ = execute_from_django( + itemtypes_api.manage_collection_item, request, + 'create', collection_id) else: - response_ = _feed_response(request, 'post_collection_items', - request, collection_id) + response_ = execute_from_django( + itemtypes_api.post_collection_items, + request, collection_id) elif request.method == 'OPTIONS': - response_ = _feed_response(request, 'manage_collection_item', - request, 'options', collection_id) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'options', collection_id) response = _to_django_response(*response_) @@ -282,8 +286,8 @@ def collection_coverage(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response( - request, 'get_collection_coverage', collection_id + response_ = execute_from_django( + coverages_api.get_collection_coverage, request, collection_id ) response = _to_django_response(*response_) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 7c7f1e29e..1de5385c7 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -38,7 +38,9 @@ send_from_directory, Response, Request) from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api @@ -121,7 +123,7 @@ def get_response(result: tuple): :param result: The result of the API call. This should be a tuple of (headers, status, content). - :returns: A Response instance. + :returns: A Response instance """ headers, status, content = result @@ -132,14 +134,24 @@ def get_response(result: tuple): return response -def execute_from_flask(api_function, request: Request, *args - ) -> Response: +def execute_from_flask(api_function, request: Request, *args) -> Response: + """ + Executes API function from Flask + + :param api_function: API function + :param request: request object + :param *args: variable length additional arguments + + :returns: A Response instance + """ + api_request = APIRequest.from_flask(request, api_.locales) + content: str | bytes + if not api_request.is_valid(): headers, status, content = api_.get_format_exception(api_request) else: - headers, status, content = api_function(api_, api_request, *args) content = apply_gzip(headers, content) # handle jsonld too? @@ -164,6 +176,7 @@ def openapi(): :returns: HTTP response """ + return get_response(api_.openapi_(request)) @@ -174,6 +187,7 @@ def conformance(): :returns: HTTP response """ + return get_response(api_.conformance(request)) @@ -183,8 +197,10 @@ def get_tilematrix_set(tileMatrixSetId=None): OGC API TileMatrixSet endpoint :param tileMatrixSetId: identifier of tile matrix set + :returns: HTTP response """ + return execute_from_flask(tiles_api.tilematrixset, request, tileMatrixSetId) @@ -196,6 +212,7 @@ def get_tilematrix_sets(): :returns: HTTP response """ + return execute_from_flask(tiles_api.tilematrixsets, request) @@ -209,6 +226,7 @@ def collections(collection_id=None): :returns: HTTP response """ + return get_response(api_.describe_collections(request, collection_id)) @@ -221,7 +239,9 @@ def collection_schema(collection_id): :returns: HTTP response """ - return get_response(api_.get_collection_schema(request, collection_id)) + + return execute_from_flask(itemtypes_api.get_collection_schema, request, + collection_id) @BLUEPRINT.route('/collections//queryables') @@ -233,7 +253,9 @@ def collection_queryables(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_queryables(request, collection_id)) + + return execute_from_flask(itemtypes_api.get_collection_queryables, request, + collection_id) @BLUEPRINT.route('/collections//items', @@ -254,36 +276,35 @@ def collection_items(collection_id, item_id=None): if item_id is None: if request.method == 'GET': # list items - return get_response( - api_.get_collection_items(request, collection_id)) + return execute_from_flask(itemtypes_api.get_collection_items, + request, collection_id) elif request.method == 'POST': # filter or manage items if request.content_type is not None: if request.content_type == 'application/geo+json': - return get_response( - api_.manage_collection_item(request, 'create', - collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, + request, 'create', collection_id) else: - return get_response( - api_.post_collection_items(request, collection_id)) + return execute_from_flask( + itemtypes_api.post_collection_items, request, + collection_id) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, request, 'options', + collection_id) elif request.method == 'DELETE': - return get_response( - api_.manage_collection_item(request, 'delete', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'delete', collection_id, item_id) elif request.method == 'PUT': - return get_response( - api_.manage_collection_item(request, 'update', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'update', collection_id, item_id) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'options', collection_id, item_id) else: - return get_response( - api_.get_collection_item(request, collection_id, item_id)) + return execute_from_flask(itemtypes_api.get_collection_item, request, + collection_id, item_id) @BLUEPRINT.route('/collections//coverage') @@ -295,7 +316,9 @@ def collection_coverage(collection_id): :returns: HTTP response """ - return get_response(api_.get_collection_coverage(request, collection_id)) + + return execute_from_flask(coverages_api.get_collection_coverage, request, + collection_id) @BLUEPRINT.route('/collections//tiles') @@ -307,6 +330,7 @@ def get_collection_tiles(collection_id=None): :returns: HTTP response """ + return execute_from_flask(tiles_api.get_collection_tiles, request, collection_id) @@ -322,6 +346,7 @@ def get_collection_tiles_metadata(collection_id=None, tileMatrixSetId=None): :returns: HTTP response """ + return execute_from_flask(tiles_api.get_collection_tiles_metadata, request, collection_id, tileMatrixSetId) @@ -341,6 +366,7 @@ def get_collection_tiles_data(collection_id=None, tileMatrixSetId=None, :returns: HTTP response """ + return execute_from_flask( tiles_api.get_collection_tiles_data, request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol) @@ -373,6 +399,7 @@ def get_processes(process_id=None): :returns: HTTP response """ + return execute_from_flask(processes_api.describe_processes, request, process_id) @@ -422,6 +449,7 @@ def get_job_result(job_id=None): :returns: HTTP response """ + return execute_from_flask(processes_api.get_job_result, request, job_id) @@ -436,6 +464,7 @@ def get_job_result_resource(job_id, resource): :returns: HTTP response """ + # TODO: this does not seem to exist? return get_response(api_.get_job_result_resource( request, job_id, resource)) @@ -468,6 +497,7 @@ def get_collection_edr_query(collection_id, instance_id=None, :returns: HTTP response """ + if location_id: query_type = 'locations' else: @@ -499,6 +529,7 @@ def stac_catalog_path(path): :returns: HTTP response """ + return execute_from_flask(stac_api.get_stac_path, request, path) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index e3dfc32dd..cbb35d9a0 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -46,10 +46,7 @@ from pygeoapi import l10n from pygeoapi.api import all_apis from pygeoapi.models.openapi import OAPIFormat -from pygeoapi.plugin import load_plugin -from pygeoapi.provider.base import ProviderTypeError, SchemaType -from pygeoapi.util import (filter_dict_by_key_value, get_provider_by_type, - filter_providers_by_type, to_json, yaml_load, +from pygeoapi.util import (filter_dict_by_key_value, to_json, yaml_load, get_api_rules, get_base_url) LOGGER = logging.getLogger(__name__) @@ -350,7 +347,6 @@ def get_oas_30(cfg): items_f = deepcopy(oas['components']['parameters']['f']) items_f['schema']['enum'].append('csv') - items_l = deepcopy(oas['components']['parameters']['lang']) LOGGER.debug('setting up datasets') collections = filter_dict_by_key_value(cfg['resources'], @@ -398,440 +394,60 @@ def get_oas_30(cfg): } } - LOGGER.debug('setting up collection endpoints') - try: - ptype = None - - if filter_providers_by_type( - collections[k]['providers'], 'feature'): - ptype = 'feature' - - if filter_providers_by_type( - collections[k]['providers'], 'record'): - ptype = 'record' - - p = load_plugin('provider', get_provider_by_type( - collections[k]['providers'], ptype)) - - items_path = f'{collection_name_path}/items' - - coll_properties = deepcopy(oas['components']['parameters']['properties']) # noqa - - coll_properties['schema']['items']['enum'] = list(p.fields.keys()) - - paths[items_path] = { - 'get': { - 'summary': f'Get {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Features', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/bbox-crs'}, # noqa - coll_properties, - {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa - {'$ref': '#/components/parameters/skipGeometry'}, - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa - {'$ref': '#/components/parameters/offset'}, - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Features', - 'responses': { - '200': {'description': 'options response'} - } - } - } - - if p.editable: - LOGGER.debug('Provider is editable; adding post') - - paths[items_path]['post'] = { - 'summary': f'Add {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'add{name.capitalize()}Features', - 'requestBody': { - 'description': 'Adds item to collection', - 'content': { - 'application/geo+json': { - 'schema': {} + oas['components']['responses'].update({ + 'Tiles': { + 'description': 'Retrieves the tiles description for this collection', # noqa + 'content': { + 'application/json': { + 'schema': { + '$ref': '#/components/schemas/tiles' } - }, - 'required': True - }, - 'responses': { - '201': {'description': 'Successful creation'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.create) - paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - if ptype == 'record': - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) - if p.fields: - schema_path = f'{collection_name_path}/schema' - - paths[schema_path] = { - 'get': { - 'summary': f'Get {title} schema', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - - queryables_path = f'{collection_name_path}/queryables' - - paths[queryables_path] = { - 'get': { - 'summary': f'Get {title} queryables', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa } } } - - if p.time_field is not None: - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - for field, type_ in p.fields.items(): - - if p.properties and field not in p.properties: - LOGGER.debug('Provider specified not to advertise property') # noqa - continue - - if field == 'q' and ptype == 'record': - LOGGER.debug('q parameter already declared, skipping') - continue - - if type_ == 'date': - schema = { - 'type': 'string', - 'format': 'date' - } - elif type_ == 'float': - schema = { - 'type': 'number', - 'format': 'float' - } - elif type_ == 'long': - schema = { - 'type': 'integer', - 'format': 'int64' - } - else: - schema = type_ - - path_ = f'{collection_name_path}/items' - paths[path_]['get']['parameters'].append({ - 'name': field, - 'in': 'query', - 'required': False, - 'schema': schema, - 'style': 'form', - 'explode': False - }) - - paths[f'{collection_name_path}/items/{{featureId}}'] = { - 'get': { - 'summary': f'Get {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/f'}, - {'$ref': '#/components/parameters/lang'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa - ], - 'responses': { - '200': {'description': 'options response'} - } - } } + ) - try: - schema_ref = p.get_schema() - paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa - 'content': { - schema_ref[0]: { - 'schema': schema_ref[1] + oas['components']['schemas'].update({ + 'tilematrixsetlink': { + 'type': 'object', + 'required': ['tileMatrixSet'], + 'properties': { + 'tileMatrixSet': { + 'type': 'string' + }, + 'tileMatrixSetURI': { + 'type': 'string' } } - } - except Exception as err: - LOGGER.debug(err) - - if p.editable: - LOGGER.debug('Provider is editable; adding put/delete') - put_path = f'{collection_name_path}/items/{{featureId}}' # noqa - paths[put_path]['put'] = { # noqa - 'summary': f'Update {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'update{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + }, + 'tiles': { + 'type': 'object', + 'required': [ + 'tileMatrixSetLinks', + 'links' ], - 'requestBody': { - 'description': 'Updates item in collection', - 'content': { - 'application/geo+json': { - 'schema': {} + 'properties': { + 'tileMatrixSetLinks': { + 'type': 'array', + 'items': { + '$ref': '#/components/schemas/tilematrixsetlink' # noqa } }, - 'required': True - }, - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.replace) - paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { # noqa - 'summary': f'Delete {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'delete{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - ], - 'responses': { - '200': {'description': 'Successful delete'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - except ProviderTypeError: - LOGGER.debug('collection is not feature based') - - LOGGER.debug('setting up coverage endpoints') - try: - load_plugin('provider', get_provider_by_type( - collections[k]['providers'], 'coverage')) - - coverage_path = f'{collection_name_path}/coverage' - - paths[coverage_path] = { - 'get': { - 'summary': f'Get {title} coverage', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Coverage', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': '#/components/parameters/bbox-crs'}, # noqa - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - except ProviderTypeError: - LOGGER.debug('collection is not coverage based') - - LOGGER.debug('setting up tiles endpoints') - tile_extension = filter_providers_by_type( - collections[k]['providers'], 'tile') - - if tile_extension: - tp = load_plugin('provider', tile_extension) - oas['components']['responses'].update({ - 'Tiles': { - 'description': 'Retrieves the tiles description for this collection', # noqa - 'content': { - 'application/json': { - 'schema': { - '$ref': '#/components/schemas/tiles' - } - } - } - } - } - ) - - oas['components']['schemas'].update({ - 'tilematrixsetlink': { - 'type': 'object', - 'required': ['tileMatrixSet'], - 'properties': { - 'tileMatrixSet': { - 'type': 'string' - }, - 'tileMatrixSetURI': { - 'type': 'string' - } - } - }, - 'tiles': { - 'type': 'object', - 'required': [ - 'tileMatrixSetLinks', - 'links' - ], - 'properties': { - 'tileMatrixSetLinks': { - 'type': 'array', - 'items': { - '$ref': '#/components/schemas/tilematrixsetlink' # noqa - } - }, - 'links': { - 'type': 'array', - 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa - } - } - } - } - ) - - tiles_path = f'{collection_name_path}/tiles' - - paths[tiles_path] = { - 'get': { - 'summary': f'Fetch a {title} tiles description', - 'description': desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Tiles', - 'parameters': [ - items_f, - # items_l TODO: is this useful? - ], - 'responses': { - '200': {'$ref': '#/components/responses/Tiles'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - tiles_data_path = f'{collection_name_path}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa - - paths[tiles_data_path] = { - 'get': { - 'summary': f'Get a {title} tile', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Tiles', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa - { - 'name': 'f', - 'in': 'query', - 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': [tp.format_type], - 'default': tp.format_type - }, - 'style': 'form', - 'explode': False - } - ], - 'responses': { - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - mimetype = tile_extension['format']['mimetype'] - paths[tiles_data_path]['get']['responses']['200'] = { - 'description': 'successful operation', - 'content': { - mimetype: { - 'schema': { - 'type': 'string', - 'format': 'binary' + 'links': { + 'type': 'array', + 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa } } } } + ) oas['paths'] = paths - for api in all_apis(): - sub_oas = api.get_oas_30(cfg, locale_) + for api_name, api_module in all_apis().items(): + LOGGER.debug(f'Adding OpenAPI definitions for {api_name}') + sub_oas = api_module.get_oas_30(cfg, locale_) oas['paths'].update(sub_oas['paths']) oas['tags'].extend(sub_oas['tags']) @@ -871,20 +487,6 @@ def get_oas_30_parameters(cfg: dict, locale_: str): 'default': l10n.locale2str(locale_) } }, - 'properties': { - 'name': 'properties', - 'in': 'query', - 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'items': { - 'type': 'string' - } - } - }, 'skipGeometry': { 'name': 'skipGeometry', 'in': 'query', diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index f5bd218de..8e7b25c6e 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -51,7 +51,9 @@ import uvicorn from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api @@ -332,45 +334,45 @@ async def collection_items(request: Request, collection_id=None, item_id=None): item_id = request.path_params['item_id'] if item_id is None: if request.method == 'GET': # list items - return await get_response( - api_.get_collection_items, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_items, request, collection_id) elif request.method == 'POST': # filter or manage items content_type = request.headers.get('content-type') if content_type is not None: if content_type == 'application/geo+json': - return await get_response( - api_.manage_collection_item, request, + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'create', collection_id) else: - return await get_response( - api_.post_collection_items, + return await execute_from_starlette( + itemtypes_api.post_collection_items, request, collection_id ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'options', collection_id ) elif request.method == 'DELETE': - return await get_response( - api_.manage_collection_item, request, 'delete', + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'delete', collection_id, item_id ) elif request.method == 'PUT': - return await get_response( - api_.manage_collection_item, request, 'update', + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'update', collection_id, item_id ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, 'options', + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'options', collection_id, item_id ) else: - return await get_response( - api_.get_collection_item, request, collection_id, item_id) + return await execute_from_starlette( + itemtypes_api.get_collection_item, request, collection_id, item_id) async def collection_coverage(request: Request, collection_id=None): @@ -385,8 +387,8 @@ async def collection_coverage(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_coverage, request, collection_id) + return await execute_from_starlette( + coverages_api.get_collection_coverage, request, collection_id) async def collection_map(request: Request, collection_id, style_id=None): @@ -405,7 +407,7 @@ async def collection_map(request: Request, collection_id, style_id=None): style_id = request.path_params['style_id'] return await execute_from_starlette( - maps_api.get_collection_map, request, collection_id, style_id, + maps_api.get_collection_map, request, collection_id, style_id ) diff --git a/tests/test_api.py b/tests/test_api.py index 2e0e17040..9399cb91a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,7 +4,7 @@ # John A Stevenson # Colin Blackburn # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 John A Stevenson and Colin Blackburn # # Permission is hereby granted, free of charge, to any person @@ -46,7 +46,11 @@ API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ ) +from pygeoapi.api.coverages import get_collection_coverage from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query +from pygeoapi.api.itemtypes import ( + get_collection_schema, get_collection_queryables, get_collection_item, + get_collection_items, manage_collection_item) from pygeoapi.api.maps import get_collection_map from pygeoapi.api.processes import ( describe_processes, execute_process, delete_job, get_job_result, @@ -526,13 +530,13 @@ def test_gzip(config, api_): def test_gzip_csv(config, api_): - req_csv = mock_request({'f': 'csv'}) - rsp_csv_headers, _, rsp_csv = api_.get_collection_items(req_csv, 'obs') + req_csv = mock_api_request({'f': 'csv'}) + rsp_csv_headers, _, rsp_csv = get_collection_items(api_, req_csv, 'obs') assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' rsp_csv = rsp_csv.decode('utf-8') - req_csv = mock_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = api_.get_collection_items(req_csv, 'obs') # noqa + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') assert rsp_csv == rsp_csv_ @@ -541,8 +545,8 @@ def test_gzip_csv(config, api_): config['server']['encoding'] = 'utf-16' api_ = API(config, openapi) - req_csv = mock_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = api_.get_collection_items(req_csv, 'obs') # noqa + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') assert rsp_csv == rsp_csv_ @@ -796,26 +800,25 @@ def test_describe_collections_hidden_resources( def test_get_collection_schema(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_schema(req, - 'notfound') + req = mock_api_request() + rsp_headers, code, response = get_collection_schema(api_, req, 'notfound') assert code == HTTPStatus.NOT_FOUND - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_schema(api_, req, 'obs') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_schema(api_, req, 'obs') assert rsp_headers['Content-Type'] == 'application/schema+json' schema = json.loads(response) assert 'properties' in schema assert len(schema['properties']) == 5 - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_schema( - req, 'gdps-temperature') + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_schema( + api_, req, 'gdps-temperature') assert rsp_headers['Content-Type'] == 'application/schema+json' schema = json.loads(response) @@ -825,17 +828,17 @@ def test_get_collection_schema(config, api_): def test_get_collection_queryables(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_queryables(req, - 'notfound') + req = mock_api_request() + rsp_headers, code, response = get_collection_queryables( + api_, req, 'notfound') assert code == HTTPStatus.NOT_FOUND - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') assert rsp_headers['Content-Type'] == 'application/schema+json' queryables = json.loads(response) @@ -845,7 +848,7 @@ def test_get_collection_queryables(config, api_): # test with provider filtered properties api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') queryables = json.loads(response) assert 'properties' in queryables @@ -893,104 +896,104 @@ def test_describe_collections_json_ld(config, api_): def test_get_collection_items(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_items(req, 'foo') + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'foo') features = json.loads(response) assert code == HTTPStatus.NOT_FOUND - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'f': 'foo'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'bbox': '1,2,3'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '1,2,3'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'bbox': '1,2,3,4c'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '1,2,3,4c'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'bbox-crs': 'bad_value'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST # bbox-crs must be in configured values for Collection - req = mock_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST # bbox-crs must be in configured values for Collection (CSV will ignore) - req = mock_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK # bbox-crs can be a default even if not configured - req = mock_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK # bbox-crs can be a default even if not configured - req = mock_request({'bbox': '4,52,5,53'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'bbox': '4,52,5,53'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'f': 'html', 'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'f': 'html', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] assert rsp_headers['Content-Language'] == 'fr-CA' - req = mock_request() - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) # No language requested: should be set to default from YAML assert rsp_headers['Content-Language'] == 'en-US' assert len(features['features']) == 5 - req = mock_request({'resulttype': 'hits'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'resulttype': 'hits'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert len(features['features']) == 0 # Invalid limit - req = mock_request({'limit': 0}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'limit': 0}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'stn_id': '35'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'stn_id': '35'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert len(features['features']) == 2 assert features['numberMatched'] == 2 - req = mock_request({'stn_id': '35', 'value': '93.9'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'stn_id': '35', 'value': '93.9'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert len(features['features']) == 1 assert features['numberMatched'] == 1 - req = mock_request({'limit': 2}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'limit': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert len(features['features']) == 2 @@ -1008,14 +1011,14 @@ def test_get_collection_items(config, api_): assert links[3]['rel'] == 'collection' # Invalid offset - req = mock_request({'offset': -1}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'offset': -1}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'offset': 2}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'offset': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert len(features['features']) == 3 @@ -1034,12 +1037,12 @@ def test_get_collection_items(config, api_): assert '/collections/obs' in links[4]['href'] assert links[4]['rel'] == 'collection' - req = mock_request({ + req = mock_api_request({ 'offset': 1, 'limit': 1, 'bbox': '-180,90,180,90' }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert len(features['features']) == 1 @@ -1061,103 +1064,103 @@ def test_get_collection_items(config, api_): assert '/collections/obs' in links[4]['href'] assert links[4]['rel'] == 'collection' - req = mock_request({ + req = mock_api_request({ 'sortby': 'bad-property', 'stn_id': '35' }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'sortby': 'stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'sortby': 'stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.OK - req = mock_request({'sortby': '+stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'sortby': '+stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.OK - req = mock_request({'sortby': '-stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'sortby': '-stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') features = json.loads(response) assert code == HTTPStatus.OK - req = mock_request({'f': 'csv'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'f': 'csv'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' - req = mock_request({'datetime': '2003'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '2003'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'datetime': '1999'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '1999'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'datetime': '2010-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '2010-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'datetime': '2001-11-11/2003-12-18'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '2001-11-11/2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'datetime': '../2003-12-18'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '../2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'datetime': '2001-11-11/..'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '2001-11-11/..'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'datetime': '1999/2005-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '1999/2005-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'datetime': '1999/2000-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '1999/2000-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST api_.config['resources']['obs']['extents'].pop('temporal') - req = mock_request({'datetime': '2002/2014-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'datetime': '2002/2014-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.OK - req = mock_request({'scalerank': 1}) - rsp_headers, code, response = api_.get_collection_items( - req, 'naturalearth/lakes') + req = mock_api_request({'scalerank': 1}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') features = json.loads(response) assert len(features['features']) == 10 assert features['numberMatched'] == 11 assert features['numberReturned'] == 10 - req = mock_request({'datetime': '2005-04-22'}) - rsp_headers, code, response = api_.get_collection_items( - req, 'naturalearth/lakes') + req = mock_api_request({'datetime': '2005-04-22'}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'skipGeometry': 'true'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'skipGeometry': 'true'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert json.loads(response)['features'][0]['geometry'] is None - req = mock_request({'properties': 'foo,bar'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + req = mock_api_request({'properties': 'foo,bar'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST @@ -1165,14 +1168,15 @@ def test_get_collection_items(config, api_): def test_get_collection_items_crs(config, api_): # Invalid CRS query parameter - req = mock_request({'crs': '4326'}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') + req = mock_api_request({'crs': '4326'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') assert code == HTTPStatus.BAD_REQUEST # Unsupported CRS - req = mock_request({'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') + req = mock_api_request( + {'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') assert code == HTTPStatus.BAD_REQUEST @@ -1183,17 +1187,17 @@ def test_get_collection_items_crs(config, api_): supported_crs_list = [default_crs, storage_crs, crs_4258] for crs in supported_crs_list: - req = mock_request({'crs': crs}) - rsp_headers, code, response = api_.get_collection_items( - req, 'norway_pop', - ) + req = mock_api_request({'crs': crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs}>' # With CRS query parameter, using storageCRS - req = mock_request({'crs': storage_crs}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') + req = mock_api_request({'crs': storage_crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -1201,8 +1205,9 @@ def test_get_collection_items_crs(config, api_): features_25833 = json.loads(response) # With CRS query parameter resulting in coordinates transformation - req = mock_request({'crs': crs_4258}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') + req = mock_api_request({'crs': crs_4258}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' @@ -1225,8 +1230,9 @@ def test_get_collection_items_crs(config, api_): break # Without CRS query parameter: assume Transform to default WGS84 lon,lat - req = mock_request({}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') + req = mock_api_request({}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{default_crs}>' @@ -1253,18 +1259,18 @@ def test_get_collection_items_crs(config, api_): def test_manage_collection_item_read_only_options_req(config, api_): """Test OPTIONS request on a read-only items endpoint""" - req = mock_request() - _, code, _ = api_.manage_collection_item(req, 'options', 'foo') + req = mock_api_request() + _, code, _ = manage_collection_item(api_, req, 'options', 'foo') assert code == HTTPStatus.NOT_FOUND - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item(req, 'options', 'obs') + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') assert code == HTTPStatus.OK assert rsp_headers['Allow'] == 'HEAD, GET' - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item( - req, 'options', 'obs', 'ressource_id') + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') assert code == HTTPStatus.OK assert rsp_headers['Allow'] == 'HEAD, GET' @@ -1275,14 +1281,14 @@ def test_manage_collection_item_editable_options_req(config): config['resources']['obs']['providers'][0]['editable'] = True api_ = API(config, openapi) - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item(req, 'options', 'obs') + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') assert code == HTTPStatus.OK assert rsp_headers['Allow'] == 'HEAD, GET, POST' - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item( - req, 'options', 'obs', 'ressource_id') + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') assert code == HTTPStatus.OK assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' @@ -1319,11 +1325,11 @@ def test_describe_collections_enclosures(config_enclosure, enclosure_api): def test_get_collection_items_json_ld(config, api_): - req = mock_request({ + req = mock_api_request({ 'f': 'jsonld', 'limit': 2 }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') + rsp_headers, code, response = get_collection_items(api_, req, 'obs') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] # No language requested: return default from YAML @@ -1341,35 +1347,35 @@ def test_get_collection_items_json_ld(config, api_): def test_get_collection_item(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') + req = mock_api_request({'f': 'foo'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_item( - req, 'gdps-temperature', '371') + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_item( + api_, req, 'gdps-temperature', '371') assert code == HTTPStatus.BAD_REQUEST - req = mock_request() - rsp_headers, code, response = api_.get_collection_item(req, 'foo', '371') + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'foo', '371') assert code == HTTPStatus.NOT_FOUND - rsp_headers, code, response = api_.get_collection_item( - req, 'obs', 'notfound') + rsp_headers, code, response = get_collection_item( + api_, req, 'obs', 'notfound') assert code == HTTPStatus.NOT_FOUND - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] assert rsp_headers['Content-Language'] == 'en-US' - req = mock_request() - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') feature = json.loads(response) assert feature['properties']['stn_id'] == 35 @@ -1378,8 +1384,8 @@ def test_get_collection_item(config, api_): def test_get_collection_item_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, _, response = api_.get_collection_item(req, 'objects', '3') + req = mock_api_request({'f': 'jsonld'}) + rsp_headers, _, response = get_collection_item(api_, req, 'objects', '3') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] assert rsp_headers['Content-Language'] == 'en-US' feature = json.loads(response) @@ -1404,7 +1410,7 @@ def test_get_collection_item_json_ld(config, api_): 'https://schema.org/longitude'][0][ '@value'] == -85 - _, _, response = api_.get_collection_item(req, 'objects', '2') + _, _, response = get_collection_item(api_, req, 'objects', '2') feature = json.loads(response) assert feature['geometry']['type'] == 'MultiPoint' expanded = jsonld.expand(feature)[0] @@ -1415,7 +1421,7 @@ def test_get_collection_item_json_ld(config, api_): 'https://schema.org/polygon'][0][ '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" - _, _, response = api_.get_collection_item(req, 'objects', '1') + _, _, response = get_collection_item(api_, req, 'objects', '1') feature = json.loads(response) expanded = jsonld.expand(feature)[0] assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ @@ -1425,7 +1431,7 @@ def test_get_collection_item_json_ld(config, api_): 'https://schema.org/line'][0][ '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' - _, _, response = api_.get_collection_item(req, 'objects', '4') + _, _, response = get_collection_item(api_, req, 'objects', '4') feature = json.loads(response) expanded = jsonld.expand(feature)[0] assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ @@ -1437,7 +1443,7 @@ def test_get_collection_item_json_ld(config, api_): '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ '30.0,30.0 40.0,20.0 30.0,10.0' - _, _, response = api_.get_collection_item(req, 'objects', '5') + _, _, response = get_collection_item(api_, req, 'objects', '5') feature = json.loads(response) expanded = jsonld.expand(feature)[0] assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ @@ -1447,7 +1453,7 @@ def test_get_collection_item_json_ld(config, api_): 'https://schema.org/polygon'][0][ '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' - _, _, response = api_.get_collection_item(req, 'objects', '7') + _, _, response = get_collection_item(api_, req, 'objects', '7') feature = json.loads(response) expanded = jsonld.expand(feature)[0] assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ @@ -1459,47 +1465,47 @@ def test_get_collection_item_json_ld(config, api_): '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ '45.0,40.0 40.0,10.0 15.0,5.0' - req = mock_request({'f': 'jsonld', 'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') + req = mock_api_request({'f': 'jsonld', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] assert rsp_headers['Content-Language'] == 'fr-CA' def test_get_collection_coverage(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_coverage( - req, 'obs') + req = mock_api_request() + rsp_headers, code, response = get_collection_coverage( + api_, req, 'obs') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'properties': '12'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request({'properties': '12'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'subset': 'bad_axis(10:20)'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request({'subset': 'bad_axis(10:20)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'f': 'blah'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request({'f': 'blah'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.BAD_REQUEST - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.BAD_REQUEST assert rsp_headers['Content-Type'] == 'text/html' - req = mock_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') # NOTE: This test used to assert the code to be 200 OK, # but it requested HTML, which is not available, @@ -1507,9 +1513,9 @@ def test_get_collection_coverage(config, api_): assert code == HTTPStatus.BAD_REQUEST assert rsp_headers['Content-Type'] == 'text/html' - req = mock_request({'subset': 'Lat(5:10),Long(5:10)'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request({'subset': 'Lat(5:10),Long(5:10)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.OK content = json.loads(response) @@ -1520,9 +1526,9 @@ def test_get_collection_coverage(config, api_): assert 'TMP' in content['ranges'] assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] - req = mock_request({'bbox': '-79,45,-75,49'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + req = mock_api_request({'bbox': '-79,45,-75,49'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.OK content = json.loads(response) @@ -1532,38 +1538,38 @@ def test_get_collection_coverage(config, api_): assert content['domain']['axes']['y']['start'] == 49.0 assert content['domain']['axes']['y']['stop'] == 45.0 - req = mock_request({ + req = mock_api_request({ 'subset': 'Lat(5:10),Long(5:10)', 'f': 'GRIB' }) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') assert code == HTTPStatus.OK assert isinstance(response, bytes) - req = mock_request(HTTP_ACCEPT='application/x-netcdf') - rsp_headers, code, response = api_.get_collection_coverage( - req, 'cmip5') + req = mock_api_request(HTTP_ACCEPT='application/x-netcdf') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'cmip5') assert code == HTTPStatus.OK assert rsp_headers['Content-Type'] == 'application/x-netcdf' - # req = mock_request({ + # req = mock_api_request({ # 'subset': 'time("2006-07-01T06:00:00":"2007-07-01T06:00:00")' # }) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') + # rsp_headers, code, response = get_collection_coverage(api_, req, 'cmip5') # # assert code == HTTPStatus.OK # assert isinstance(json.loads(response), dict) - # req = mock_request({'subset': 'lat(1:2'}) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') + # req = mock_api_request({'subset': 'lat(1:2'}) + # rsp_headers, code, response = get_collection_coverage(api_, req, 'cmip5') # # assert code == HTTPStatus.BAD_REQUEST # - # req = mock_request({'subset': 'lat(1:2)'}) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') + # req = mock_api_request({'subset': 'lat(1:2)'}) + # rsp_headers, code, response = get_collection_coverage(api_ req, 'cmip5') # # assert code == HTTPStatus.NO_CONTENT From 79c3d15ade075c4273c17d46902232c749fb0226 Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Mon, 11 Mar 2024 13:01:33 +0200 Subject: [PATCH 26/71] update release version --- docs/source/conf.py | 3 ++- pygeoapi/__init__.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ff6633c4b..37e579be7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,6 +3,7 @@ # Authors: Tom Kralidis # # Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2024 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -111,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.16.dev0' +version = '0.16.0' # The full version, including alpha/beta/rc tags. release = version diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index 0f76addea..b54af7326 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -5,6 +5,7 @@ # # Copyright (c) 2021 Tom Kralidis # Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -29,7 +30,7 @@ # # ================================================================= -__version__ = '0.16.dev0' +__version__ = '0.16.0' import click try: From 9fca2812af51428695ed23f0f3e581d50f504c1d Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Mon, 11 Mar 2024 13:46:23 +0200 Subject: [PATCH 27/71] switch back to dev --- docs/source/conf.py | 2 +- pygeoapi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 37e579be7..b16e75efa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.16.0' +version = '0.17.dev0' # The full version, including alpha/beta/rc tags. release = version diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index b54af7326..96a91ce09 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -30,7 +30,7 @@ # # ================================================================= -__version__ = '0.16.0' +__version__ = '0.17.dev0' import click try: From 7842a00e1e948d6f60c3520c2bef7d499ac93bbd Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 17 Mar 2024 08:36:03 -0400 Subject: [PATCH 28/71] backport of #1313 --- .../data-publishing/ogcapi-processes.rst | 5 ++ pygeoapi/api/processes.py | 25 ++++++++- pygeoapi/process/manager/base.py | 56 +++++++++++++++++-- pygeoapi/process/manager/mongodb_.py | 1 + pygeoapi/process/manager/tinydb_.py | 1 + pygeoapi/util.py | 10 ++++ tests/test_api.py | 31 +++++++++- 7 files changed, 119 insertions(+), 10 deletions(-) diff --git a/docs/source/data-publishing/ogcapi-processes.rst b/docs/source/data-publishing/ogcapi-processes.rst index 919806249..55d1c022f 100644 --- a/docs/source/data-publishing/ogcapi-processes.rst +++ b/docs/source/data-publishing/ogcapi-processes.rst @@ -108,6 +108,11 @@ Processing examples -H "Prefer: respond-async" -d "{\"inputs\":{\"name\": \"hi there2\"}}" + # execute a job for the ``hello-world`` process with a success subscriber + curl -X POST http://localhost:5000/processes/hello-world/execution \ + -H "Content-Type: application/json" \ + -d "{\"inputs\":{\"name\": \"hi there2\"}, \ + \"subscriber\": {\"successUri\": \"https://www.example.com/success\"}}" .. _`OGC API - Processes`: https://ogcapi.ogc.org/processes .. _`sample`: https://github.com/geopython/pygeoapi/blob/master/pygeoapi/process/hello_world.py diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 2e4dd35ea..26ebbac17 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -52,7 +52,7 @@ from pygeoapi.process.base import ( JobNotFoundError, JobResultNotFoundError, ProcessorExecuteError ) -from pygeoapi.process.manager.base import get_manager +from pygeoapi.process.manager.base import get_manager, Subscriber from . import ( APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, @@ -64,7 +64,8 @@ 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30' + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback' ] @@ -374,6 +375,23 @@ def execute_process(api: API, request: APIRequest, data_dict = data.get('inputs', {}) LOGGER.debug(data_dict) + subscriber = None + subscriber_dict = data.get('subscriber') + if subscriber_dict: + try: + success_uri = subscriber_dict['successUri'] + except KeyError: + return api_.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', 'Missing successUri') + else: + subscriber = Subscriber( + # NOTE: successUri is mandatory according to the standard + success_uri=success_uri, + in_progress_uri=subscriber_dict.get('inProgressUri'), + failed_uri=subscriber_dict.get('failedUri'), + ) + try: execution_mode = RequestedProcessExecutionMode( request.headers.get('Prefer', request.headers.get('prefer')) @@ -383,7 +401,8 @@ def execute_process(api: API, request: APIRequest, try: LOGGER.debug('Executing process') result = api.manager.execute_process( - process_id, data_dict, execution_mode=execution_mode) + process_id, data_dict, execution_mode=execution_mode, + subscriber=subscriber) job_id, mime_type, outputs, status, additional_headers = result headers.update(additional_headers or {}) headers['Location'] = f'{api.base_url}/jobs/{job_id}' diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index b8065c177..0b112c471 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -38,6 +38,8 @@ from typing import Any, Dict, Tuple, Optional, OrderedDict import uuid +import requests + from pygeoapi.plugin import load_plugin from pygeoapi.process.base import ( BaseProcessor, @@ -50,6 +52,7 @@ JobStatus, ProcessExecutionMode, RequestedProcessExecutionMode, + Subscriber ) LOGGER = logging.getLogger(__name__) @@ -70,6 +73,7 @@ def __init__(self, manager_def: dict): self.name = manager_def['name'] self.is_async = False + self.supports_subscribing = False self.connection = manager_def.get('connection') self.output_dir = manager_def.get('output_dir') @@ -85,7 +89,7 @@ def __init__(self, manager_def: dict): for id_, process_conf in manager_def.get('processes', {}).items(): self.processes[id_] = dict(process_conf) - def get_processor(self, process_id: str) -> Optional[BaseProcessor]: + def get_processor(self, process_id: str) -> BaseProcessor: """Instantiate a processor. :param process_id: Identifier of the process @@ -178,7 +182,9 @@ def delete_job(self, job_id: str) -> bool: raise JobNotFoundError() def _execute_handler_async(self, p: BaseProcessor, job_id: str, - data_dict: dict) -> Tuple[str, None, JobStatus]: + data_dict: dict, + subscriber: Optional[Subscriber] = None, + ) -> Tuple[str, None, JobStatus]: """ This private execution handler executes a process in a background thread using `multiprocessing.dummy` @@ -194,13 +200,15 @@ def _execute_handler_async(self, p: BaseProcessor, job_id: str, """ _process = dummy.Process( target=self._execute_handler_sync, - args=(p, job_id, data_dict) + args=(p, job_id, data_dict, subscriber) ) _process.start() return 'application/json', None, JobStatus.accepted def _execute_handler_sync(self, p: BaseProcessor, job_id: str, - data_dict: dict) -> Tuple[str, Any, JobStatus]: + data_dict: dict, + subscriber: Optional[Subscriber] = None, + ) -> Tuple[str, Any, JobStatus]: """ Synchronous execution handler @@ -233,6 +241,7 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, } self.add_job(job_metadata) + self._send_in_progress_notification(subscriber) try: if self.output_dir is not None: @@ -276,6 +285,7 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, } self.update_job(job_id, job_update_metadata) + self._send_success_notification(subscriber, outputs=outputs) except Exception as err: # TODO assess correct exception type and description to help users @@ -307,6 +317,7 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, jfmt = 'application/json' self.update_job(job_id, job_metadata) + self._send_failed_notification(subscriber) return jfmt, outputs, current_status @@ -314,7 +325,8 @@ def execute_process( self, process_id: str, data_dict: dict, - execution_mode: Optional[RequestedProcessExecutionMode] = None + execution_mode: Optional[RequestedProcessExecutionMode] = None, + subscriber: Optional[Subscriber] = None ) -> Tuple[str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -323,6 +335,7 @@ def execute_process( :param data_dict: `dict` of data parameters :param execution_mode: `str` optionally specifying sync or async processing. + :param subscriber: `Subscriber` optionally specifying callback urls :raises UnknownProcessError: if the input process_id does not correspond to a known process @@ -367,9 +380,40 @@ def execute_process( response_headers = None # TODO: handler's response could also be allowed to include more HTTP # headers - mime_type, outputs, status = handler(processor, job_id, data_dict) + mime_type, outputs, status = handler( + processor, + job_id, + data_dict, + # only pass subscriber if supported, otherwise this breaks existing + # managers + **({'subscriber': subscriber} if self.supports_subscribing else {}) + ) + return job_id, mime_type, outputs, status, response_headers + def _send_in_progress_notification(self, subscriber: Optional[Subscriber]): + if subscriber and subscriber.in_progress_uri: + response = requests.post(subscriber.in_progress_uri, json={}) + LOGGER.debug( + f'In progress notification response: {response.status_code}' + ) + + def _send_success_notification( + self, subscriber: Optional[Subscriber], outputs: Any + ): + if subscriber: + response = requests.post(subscriber.success_uri, json=outputs) + LOGGER.debug( + f'Success notification response: {response.status_code}' + ) + + def _send_failed_notification(self, subscriber: Optional[Subscriber]): + if subscriber and subscriber.failed_uri: + response = requests.post(subscriber.failed_uri, json={}) + LOGGER.debug( + f'Failed notification response: {response.status_code}' + ) + def __repr__(self): return f' {self.name}' diff --git a/pygeoapi/process/manager/mongodb_.py b/pygeoapi/process/manager/mongodb_.py index 7e26aa770..66886f973 100644 --- a/pygeoapi/process/manager/mongodb_.py +++ b/pygeoapi/process/manager/mongodb_.py @@ -45,6 +45,7 @@ class MongoDBManager(BaseManager): def __init__(self, manager_def): super().__init__(manager_def) self.is_async = True + self.supports_subscribing = True def _connect(self): try: diff --git a/pygeoapi/process/manager/tinydb_.py b/pygeoapi/process/manager/tinydb_.py index 131ebc023..3966e9dd1 100644 --- a/pygeoapi/process/manager/tinydb_.py +++ b/pygeoapi/process/manager/tinydb_.py @@ -61,6 +61,7 @@ def __init__(self, manager_def: dict): super().__init__(manager_def) self.is_async = True + self.supports_subscribing = True @contextmanager def _db(self): diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 2e836dfe4..620b42a87 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -591,6 +591,16 @@ class JobStatus(Enum): dismissed = 'dismissed' +@dataclass(frozen=True) +class Subscriber: + """Store subscriber urls as defined in: + https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/subscriber.yaml # noqa + """ + success_uri: str + in_progress_uri: Optional[str] + failed_uri: Optional[str] + + def read_data(path: Union[Path, str]) -> Union[bytes, str]: """ helper function to read data (file or network) diff --git a/tests/test_api.py b/tests/test_api.py index 9399cb91a..09ba0a5ee 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -36,6 +36,7 @@ import time import gzip from http import HTTPStatus +from unittest import mock from pyld import jsonld import pytest @@ -621,7 +622,7 @@ def test_conformance(config, api_): assert isinstance(root, dict) assert 'conformsTo' in root - assert len(root['conformsTo']) == 36 + assert len(root['conformsTo']) == 37 assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ in root['conformsTo'] @@ -1749,6 +1750,16 @@ def test_execute_process(config, api_): 'name': None } } + req_body_7 = { + 'inputs': { + 'name': 'Test' + }, + 'subscriber': { + 'successUri': 'https://example.com/success', + 'inProgressUri': 'https://example.com/inProgress', + 'failedUri': 'https://example.com/failed', + } + } cleanup_jobs = set() @@ -1876,6 +1887,24 @@ def test_execute_process(config, api_): assert isinstance(response, dict) assert code == HTTPStatus.CREATED + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_7) + with mock.patch( + 'pygeoapi.process.manager.base.requests.post' + ) as post_mocker: + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert code == HTTPStatus.OK + post_mocker.assert_any_call( + req_body_7['subscriber']['inProgressUri'], json={} + ) + post_mocker.assert_any_call( + req_body_7['subscriber']['successUri'], + json={'id': 'echo', 'value': 'Hello Test!'} + ) + assert post_mocker.call_count == 2 + cleanup_jobs.add(tuple(['hello-world', rsp_headers['Location'].split('/')[-1]])) From 0ca2dc9e47e50ac392febab8f82e36cda6eb2d00 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 17 Mar 2024 08:41:50 -0400 Subject: [PATCH 29/71] backport of #1313 fix --- pygeoapi/api/processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index 26ebbac17..d8eee308c 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -381,7 +381,7 @@ def execute_process(api: API, request: APIRequest, try: success_uri = subscriber_dict['successUri'] except KeyError: - return api_.get_exception( + return api.get_exception( HTTPStatus.BAD_REQUEST, headers, request.format, 'MissingParameterValue', 'Missing successUri') else: From e3b588154f3cb2f062d52e049904f06022a51bdb Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 17 Mar 2024 09:05:40 -0400 Subject: [PATCH 30/71] backport of #1585 --- docs/source/administration.rst | 4 ++ docs/source/openapi.rst | 2 +- pygeoapi/api/itemtypes.py | 2 +- pygeoapi/openapi.py | 76 ++++++++++++++++++++++++---------- 4 files changed, 61 insertions(+), 23 deletions(-) diff --git a/docs/source/administration.rst b/docs/source/administration.rst index f07413b4a..fcebd291b 100644 --- a/docs/source/administration.rst +++ b/docs/source/administration.rst @@ -49,6 +49,10 @@ To generate the OpenAPI document as JSON, run: property names and their data types. Whenever you make changes to your pygeoapi configuration, always refresh the accompanying OpenAPI document. +.. note:: + By default, pygeoapi OpenAPI document generation will fail on any collection whose provider is improperly + configured or when encountering a connection issue. To skip failing collections, use the + --no-fail-on-invalid-collection flag. The OpenAPI document will then be generated without the failing collection(s) .. seealso:: :ref:`openapi` for more information on pygeoapi's OpenAPI support diff --git a/docs/source/openapi.rst b/docs/source/openapi.rst index 1d3593316..71383c61c 100644 --- a/docs/source/openapi.rst +++ b/docs/source/openapi.rst @@ -13,7 +13,7 @@ The official OpenAPI specification can be found `on GitHub dict: except ProviderTypeError: LOGGER.debug('collection is not feature/item based') - return {'tags': [], 'paths': {}} + return {'tags': [], 'paths': paths} diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index cbb35d9a0..b92fda87d 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -65,7 +65,14 @@ THISDIR = os.path.dirname(os.path.realpath(__file__)) -def get_ogc_schemas_location(server_config): +def get_ogc_schemas_location(server_config) -> dict: + """ + Determine OGC schemas location + + :param server_config: `dict` of server configuration + + :returns: `str` of OGC schemas location + """ osl = server_config.get('ogc_schemas_location') @@ -82,7 +89,7 @@ def get_ogc_schemas_location(server_config): # TODO: remove this function once OGC API - Processing is final -def gen_media_type_object(media_type, api_type, path): +def gen_media_type_object(media_type: str, api_type: str, path: str) -> dict: """ Generates an OpenAPI Media Type Object @@ -107,7 +114,8 @@ def gen_media_type_object(media_type, api_type, path): # TODO: remove this function once OGC API - Processing is final -def gen_response_object(description, media_type, api_type, path): +def gen_response_object(description: str, media_type: str, + api_type: str, path: str) -> dict: """ Generates an OpenAPI Response Object @@ -126,13 +134,15 @@ def gen_response_object(description, media_type, api_type, path): return response -def get_oas_30(cfg): +def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: """ Generates an OpenAPI 3.0 Document :param cfg: configuration object + :param fail_on_invalid_collection: `bool` of whether to fail on an invalid + collection - :returns: OpenAPI definition YAML dict + :returns: dict of OpenAPI definition """ paths = {} @@ -447,9 +457,16 @@ def get_oas_30(cfg): for api_name, api_module in all_apis().items(): LOGGER.debug(f'Adding OpenAPI definitions for {api_name}') - sub_oas = api_module.get_oas_30(cfg, locale_) - oas['paths'].update(sub_oas['paths']) - oas['tags'].extend(sub_oas['tags']) + + try: + sub_oas = api_module.get_oas_30(cfg, locale_) + oas['paths'].update(sub_oas['paths']) + oas['tags'].extend(sub_oas['tags']) + except Exception as err: + if fail_on_invalid_collection: + raise + else: + LOGGER.warning(f'Resource not added to OpenAPI: {err}') if cfg['server'].get('admin', False): schema_dict = get_config_schema() @@ -795,23 +812,27 @@ def get_admin(): return paths -def get_oas(cfg, version='3.0'): +def get_oas(cfg: dict, fail_on_invalid_collection: bool = True, + version='3.0') -> dict: """ Stub to generate OpenAPI Document - :param cfg: configuration object + :param cfg: `dict` configuration + :param fail_on_invalid_collection: `bool` of whether to fail on an + invalid collection :param version: version of OpenAPI (default 3.0) - :returns: OpenAPI definition YAML dict + :returns: `dict` of OpenAPI definition """ if version == '3.0': - return get_oas_30(cfg) + return get_oas_30( + cfg, fail_on_invalid_collection=fail_on_invalid_collection) else: raise RuntimeError('OpenAPI version not supported') -def validate_openapi_document(instance_dict): +def validate_openapi_document(instance_dict: dict) -> bool: """ Validate an OpenAPI document against the OpenAPI schema @@ -831,27 +852,36 @@ def validate_openapi_document(instance_dict): def generate_openapi_document(cfg_file: Union[Path, io.TextIOWrapper], - output_format: OAPIFormat): + output_format: OAPIFormat, + fail_on_invalid_collection: bool = True) -> str: """ Generate an OpenAPI document from the configuration file - :param cfg_file: configuration Path instance + :param cfg_file: configuration Path instance (`str` of filepath + or parsed `dict`) :param output_format: output format for OpenAPI document + :param fail_on_invalid_collection: `bool` of whether to fail on an + invalid collection - :returns: content of the OpenAPI document in the output - format requested + :returns: `str` of the OpenAPI document in the output format requested """ + + LOGGER.debug(f'Loading configuration {cfg_file}') + if isinstance(cfg_file, Path): with cfg_file.open(mode="r") as cf: s = yaml_load(cf) else: s = yaml_load(cfg_file) + pretty_print = s['server'].get('pretty_print', False) + oas = get_oas(s, fail_on_invalid_collection=fail_on_invalid_collection) + if output_format == 'yaml': - content = yaml.safe_dump(get_oas(s), default_flow_style=False) + content = yaml.safe_dump(oas, default_flow_style=False) else: - content = to_json(get_oas(s), pretty=pretty_print) + content = to_json(oas, pretty=pretty_print) return content @@ -882,17 +912,21 @@ def openapi(): @click.command() @click.pass_context @click.argument('config_file', type=click.File(encoding='utf-8')) +@click.option('--fail-on-invalid-collection/--no-fail-on-invalid-collection', + '-fic', default=True, help='Fail on invalid collection') @click.option('--format', '-f', 'format_', type=click.Choice(['json', 'yaml']), default='yaml', help='output format (json|yaml)') @click.option('--output-file', '-of', type=click.File('w', encoding='utf-8'), help='Name of output file') -def generate(ctx, config_file, output_file, format_='yaml'): +def generate(ctx, config_file, output_file, format_='yaml', + fail_on_invalid_collection=True): """Generate OpenAPI Document""" if config_file is None: raise click.ClickException('--config/-c required') - content = generate_openapi_document(config_file, format_) + content = generate_openapi_document( + config_file, format_, fail_on_invalid_collection) if output_file is None: click.echo(content) From 4fdb921952c06efeaad9adb9d107502ad5aa23c0 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 12 Mar 2024 09:16:34 -0400 Subject: [PATCH 31/71] Flask: sanitize OGC schema pathing (#1593) --- pygeoapi/flask_app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 1de5385c7..a15fac4ba 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -109,8 +109,11 @@ def schemas(path): dirname_ = os.path.dirname(full_filepath) basename_ = os.path.basename(full_filepath) - # TODO: better sanitization? - path_ = dirname_.replace('..', '').replace('//', '') + path_ = dirname_.replace('..', '').replace('//', '').replace('./', '') + + if '..' in path_: + return 'Invalid path', 400 + return send_from_directory(path_, basename_, mimetype=get_mimetype(basename_)) From b5c70a1e9383ede9d4cdc2e053f04b92454b727f Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Wed, 13 Mar 2024 11:01:14 +0200 Subject: [PATCH 32/71] update release version --- docs/source/conf.py | 2 +- pygeoapi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index b16e75efa..2be4d6bee 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.17.dev0' +version = '0.16.1' # The full version, including alpha/beta/rc tags. release = version diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index 96a91ce09..701b00031 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -30,7 +30,7 @@ # # ================================================================= -__version__ = '0.17.dev0' +__version__ = '0.16.1' import click try: From b91370501305b220a1ecaea51e21ed42fb8c8b89 Mon Sep 17 00:00:00 2001 From: Angelos Tzotsos Date: Wed, 13 Mar 2024 11:16:17 +0200 Subject: [PATCH 33/71] switch back to dev --- docs/source/conf.py | 2 +- pygeoapi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2be4d6bee..b16e75efa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.16.1' +version = '0.17.dev0' # The full version, including alpha/beta/rc tags. release = version diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index 701b00031..96a91ce09 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -30,7 +30,7 @@ # # ================================================================= -__version__ = '0.16.1' +__version__ = '0.17.dev0' import click try: From 23b4960feb2e85597729f1945cdeb5f0a0908a86 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 17 Mar 2024 09:07:24 -0400 Subject: [PATCH 34/71] backport of #1596 --- pygeoapi/api/itemtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 7b580d671..7f3e64fde 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -968,7 +968,7 @@ def post_collection_items( else: LOGGER.debug('processing Elasticsearch CQL_JSON data') try: - filter_ = CQLModel.model_validate_json(data) + filter_ = CQLModel.parse_raw(data) except Exception as err: LOGGER.error(err) msg = f'Bad CQL string : {data}' From 04458f4b34896bd2be1e04ee0bef2b2f0b261e21 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 18 Mar 2024 12:00:59 +0100 Subject: [PATCH 35/71] Port test_gzip_csv test Note that apply_gzip is now called by the web framework adapters, so to test it in general, we have to call it in the test manually --- tests/test_api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 09ba0a5ee..cd888346a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -45,7 +45,7 @@ from pygeoapi.api import ( API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, - validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ + validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__, apply_gzip, ) from pygeoapi.api.coverages import get_collection_coverage from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query @@ -531,13 +531,19 @@ def test_gzip(config, api_): def test_gzip_csv(config, api_): + # Add gzip to server + config['server']['gzip'] = True + api_ = API(config, openapi) + req_csv = mock_api_request({'f': 'csv'}) rsp_csv_headers, _, rsp_csv = get_collection_items(api_, req_csv, 'obs') + rsp_csv = apply_gzip(rsp_csv_headers, rsp_csv) assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' rsp_csv = rsp_csv.decode('utf-8') req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') assert rsp_csv == rsp_csv_ @@ -548,6 +554,7 @@ def test_gzip_csv(config, api_): req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') assert rsp_csv == rsp_csv_ From 110946b963be9dd2bb5274a4536a92764f3d1d2d Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 18 Mar 2024 12:42:34 +0100 Subject: [PATCH 36/71] Add empty conformance class list to stac api --- pygeoapi/api/stac.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index 7ccf14cda..e2b4fb623 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -59,6 +59,9 @@ LOGGER = logging.getLogger(__name__) +CONFORMANCE_CLASSES = [] + + # TODO: no tests for this? def get_stac_root(api: API, request: APIRequest) -> Tuple[dict, int, str]: """ From d6546ea08da8845cbd1544919078b935a7de2765 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 18 Mar 2024 12:44:25 +0100 Subject: [PATCH 37/71] Fix queryables call in starlette --- pygeoapi/starlette_app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 8e7b25c6e..a2e689e4d 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -245,8 +245,9 @@ async def collection_queryables(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_queryables, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_queryables, request, collection_id, + ) async def get_collection_tiles(request: Request, collection_id=None): From 247a83f517596e5543e151c2c1dc6bb52eb7f415 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 18 Mar 2024 08:22:44 -0400 Subject: [PATCH 38/71] fix ref --- pygeoapi/starlette_app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index a2e689e4d..2950951d1 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -229,8 +229,9 @@ async def collection_schema(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_schema, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_schema, request, collection_id + ) async def collection_queryables(request: Request, collection_id=None): From d9ca45a7b1cf3f8bc839a4eb7816e247a0c21824 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 18 Mar 2024 14:46:59 +0100 Subject: [PATCH 39/71] Unify request validity checking The default case is handled by the web framework adapters. If custom format handling is required, the check in the adapter must be skipped. --- pygeoapi/api/itemtypes.py | 7 ------- pygeoapi/django_/views.py | 25 +++++++++++++++---------- pygeoapi/flask_app.py | 33 ++++++++++++++++++++++----------- pygeoapi/starlette_app.py | 30 +++++++++++++++++------------- 4 files changed, 54 insertions(+), 41 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 7f3e64fde..08c439002 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -111,8 +111,6 @@ def get_collection_schema( :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return api.get_format_exception(request) headers = request.get_response_headers(**api.api_headers) if any([dataset is None, @@ -193,8 +191,6 @@ def get_collection_queryables(api: API, request: Union[APIRequest, Any], :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return api.get_format_exception(request) headers = request.get_response_headers(**api.api_headers) if any([dataset is None, @@ -1124,9 +1120,6 @@ def get_collection_item(api: API, request: APIRequest, :returns: tuple of headers, status code, content """ - if not request.is_valid(): - return api.get_format_exception(request) - # Set Content-Language to system locale until provider locale # has been determined headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index e279b14a7..05ad35807 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -187,20 +187,22 @@ def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: itemtypes_api.get_collection_items, request, collection_id, + skip_valid_check=True, ) elif request.method == 'POST': if request.content_type is not None: if request.content_type == 'application/geo+json': response_ = execute_from_django( itemtypes_api.manage_collection_item, request, - 'create', collection_id) + 'create', collection_id, skip_valid_check=True) else: response_ = execute_from_django( itemtypes_api.post_collection_items, - request, collection_id) + request, collection_id, skip_valid_check=True,) elif request.method == 'OPTIONS': response_ = execute_from_django(itemtypes_api.manage_collection_item, - request, 'options', collection_id) + request, 'options', collection_id, + skip_valid_check=True) response = _to_django_response(*response_) @@ -258,17 +260,17 @@ def collection_item(request: HttpRequest, elif request.method == 'PUT': response_ = _feed_response( request, 'manage_collection_item', request, 'update', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'DELETE': response_ = _feed_response( request, 'manage_collection_item', request, 'delete', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': response_ = _feed_response( request, 'manage_collection_item', request, 'options', - collection_id, item_id) + collection_id, item_id, skip_valid_check=True) response = _to_django_response(*response_) @@ -323,6 +325,7 @@ def collection_tiles_metadata(request: HttpRequest, collection_id: str, return execute_from_django( tiles_api.get_collection_tiles_metadata, request, collection_id, tileMatrixSetId, + skip_valid_check=True, ) @@ -350,6 +353,7 @@ def collection_item_tiles(request: HttpRequest, collection_id: str, tileMatrix, tileRow, tileCol, + skip_valid_check=True, ) @@ -445,7 +449,8 @@ def get_collection_edr_query( collection_id, instance_id, query_type, - location_id + location_id, + skip_valid_check=True, ) @@ -546,8 +551,8 @@ def _feed_response(request: HttpRequest, api_definition: str, return api(request, *args, **kwargs) -def execute_from_django(api_function, request: HttpRequest, *args - ) -> HttpResponse: +def execute_from_django(api_function, request: HttpRequest, *args, + skip_valid_check=False) -> HttpResponse: api_: API | "Admin" if settings.PYGEOAPI_CONFIG['server'].get('admin'): # noqa @@ -558,7 +563,7 @@ def execute_from_django(api_function, request: HttpRequest, *args api_request = APIRequest.from_django(request, api_.locales) content: str | bytes - if not api_request.is_valid(): + if not skip_valid_check and not api_request.is_valid(): headers, status, content = api_.get_format_exception(api_request) else: diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index a15fac4ba..7e77b75dd 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -137,13 +137,15 @@ def get_response(result: tuple): return response -def execute_from_flask(api_function, request: Request, *args) -> Response: +def execute_from_flask(api_function, request: Request, *args, + skip_valid_check=False) -> Response: """ Executes API function from Flask :param api_function: API function :param request: request object :param *args: variable length additional arguments + :param skip_validity_check: bool :returns: A Response instance """ @@ -152,7 +154,7 @@ def execute_from_flask(api_function, request: Request, *args) -> Response: content: str | bytes - if not api_request.is_valid(): + if not skip_valid_check and not api_request.is_valid(): headers, status, content = api_.get_format_exception(api_request) else: headers, status, content = api_function(api_, api_request, *args) @@ -280,31 +282,36 @@ def collection_items(collection_id, item_id=None): if item_id is None: if request.method == 'GET': # list items return execute_from_flask(itemtypes_api.get_collection_items, - request, collection_id) + request, collection_id, + skip_valid_check=True) elif request.method == 'POST': # filter or manage items if request.content_type is not None: if request.content_type == 'application/geo+json': return execute_from_flask( itemtypes_api.manage_collection_item, - request, 'create', collection_id) + request, 'create', collection_id, + skip_valid_check=True) else: return execute_from_flask( itemtypes_api.post_collection_items, request, - collection_id) + collection_id, skip_valid_check=True) elif request.method == 'OPTIONS': return execute_from_flask( itemtypes_api.manage_collection_item, request, 'options', - collection_id) + collection_id, skip_valid_check=True) elif request.method == 'DELETE': return execute_from_flask(itemtypes_api.manage_collection_item, - request, 'delete', collection_id, item_id) + request, 'delete', collection_id, item_id, + skip_valid_check=True) elif request.method == 'PUT': return execute_from_flask(itemtypes_api.manage_collection_item, - request, 'update', collection_id, item_id) + request, 'update', collection_id, item_id, + skip_valid_check=True) elif request.method == 'OPTIONS': return execute_from_flask(itemtypes_api.manage_collection_item, - request, 'options', collection_id, item_id) + request, 'options', collection_id, item_id, + skip_valid_check=True) else: return execute_from_flask(itemtypes_api.get_collection_item, request, collection_id, item_id) @@ -351,7 +358,8 @@ def get_collection_tiles_metadata(collection_id=None, tileMatrixSetId=None): """ return execute_from_flask(tiles_api.get_collection_tiles_metadata, - request, collection_id, tileMatrixSetId) + request, collection_id, tileMatrixSetId, + skip_valid_check=True) @BLUEPRINT.route('/collections//tiles/\ @@ -372,7 +380,9 @@ def get_collection_tiles_data(collection_id=None, tileMatrixSetId=None, return execute_from_flask( tiles_api.get_collection_tiles_data, - request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol) + request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True, + ) @BLUEPRINT.route('/collections//map') @@ -509,6 +519,7 @@ def get_collection_edr_query(collection_id, instance_id=None, return execute_from_flask( edr_api.get_collection_edr_query, request, collection_id, instance_id, query_type, location_id, + skip_valid_check=True, ) diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 2950951d1..71cefe09d 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -139,11 +139,11 @@ def _to_response(headers, status, content): return response -async def execute_from_starlette(api_function, request: Request, *args - ) -> Response: +async def execute_from_starlette(api_function, request: Request, *args, + skip_valid_check=False) -> Response: api_request = await APIRequest.from_starlette(request, api_.locales) content: str | bytes - if not api_request.is_valid(): + if not skip_valid_check and not api_request.is_valid(): headers, status, content = api_.get_format_exception(api_request) else: @@ -284,7 +284,7 @@ async def get_collection_tiles_metadata(request: Request, collection_id=None, return await execute_from_starlette( tiles_api.get_collection_tiles_metadata, request, - collection_id, tileMatrixSetId + collection_id, tileMatrixSetId, skip_valid_check=True, ) @@ -315,7 +315,8 @@ async def get_collection_items_tiles(request: Request, collection_id=None, tileCol = request.path_params['tileCol'] return await execute_from_starlette( tiles_api.get_collection_tiles_data, request, collection_id, - tileMatrixSetId, tile_matrix, tileRow, tileCol + tileMatrixSetId, tile_matrix, tileRow, tileCol, + skip_valid_check=True, ) @@ -337,40 +338,42 @@ async def collection_items(request: Request, collection_id=None, item_id=None): if item_id is None: if request.method == 'GET': # list items return await execute_from_starlette( - itemtypes_api.get_collection_items, request, collection_id) + itemtypes_api.get_collection_items, request, collection_id, + skip_valid_check=True) elif request.method == 'POST': # filter or manage items content_type = request.headers.get('content-type') if content_type is not None: if content_type == 'application/geo+json': return await execute_from_starlette( itemtypes_api.manage_collection_item, request, - 'create', collection_id) + 'create', collection_id, skip_valid_check=True) else: return await execute_from_starlette( itemtypes_api.post_collection_items, request, - collection_id + collection_id, + skip_valid_check=True, ) elif request.method == 'OPTIONS': return await execute_from_starlette( itemtypes_api.manage_collection_item, request, - 'options', collection_id + 'options', collection_id, skip_valid_check=True, ) elif request.method == 'DELETE': return await execute_from_starlette( itemtypes_api.manage_collection_item, request, 'delete', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'PUT': return await execute_from_starlette( itemtypes_api.manage_collection_item, request, 'update', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': return await execute_from_starlette( itemtypes_api.manage_collection_item, request, 'options', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) else: return await execute_from_starlette( @@ -528,7 +531,8 @@ async def get_collection_edr_query(request: Request, collection_id=None, instanc query_type = request["path"].split('/')[-1] # noqa return await execute_from_starlette( edr_api.get_collection_edr_query, request, collection_id, - instance_id, query_type + instance_id, query_type, + skip_valid_check=True, ) From 218bca15fd6e808bf036cc480c6dbc5463c13217 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 18 Mar 2024 15:19:00 +0100 Subject: [PATCH 40/71] Fix imports in django views --- pygeoapi/django_/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 05ad35807..84c50bae3 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -41,9 +41,9 @@ from django.http import HttpRequest, HttpResponse from pygeoapi.api import API, APIRequest, apply_gzip -import pygeoapi.api.coverages_api as coverages_api +import pygeoapi.api.coverages as coverages_api import pygeoapi.api.environmental_data_retrieval as edr_api -import pygeoapi.api.itemtypes_api as itemtypes_api +import pygeoapi.api.itemtypes as itemtypes_api import pygeoapi.api.maps as maps_api import pygeoapi.api.processes as processes_api import pygeoapi.api.stac as stac_api From c14fef9cdf1a8fc6cf423765cd3d823fa8af870d Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 18 Mar 2024 22:30:04 -0400 Subject: [PATCH 41/71] backport #1598 --- pygeoapi/process/manager/dummy.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pygeoapi/process/manager/dummy.py b/pygeoapi/process/manager/dummy.py index b27507b7d..eb35eee09 100644 --- a/pygeoapi/process/manager/dummy.py +++ b/pygeoapi/process/manager/dummy.py @@ -35,6 +35,7 @@ from pygeoapi.util import ( RequestedProcessExecutionMode, JobStatus, + Subscriber ) LOGGER = logging.getLogger(__name__) @@ -70,7 +71,8 @@ def execute_process( self, process_id: str, data_dict: dict, - execution_mode: Optional[RequestedProcessExecutionMode] = None + execution_mode: Optional[RequestedProcessExecutionMode] = None, + subscriber: Optional[Subscriber] = None ) -> Tuple[str, str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -94,10 +96,12 @@ def execute_process( LOGGER.debug('Dummy manager does not support asynchronous') LOGGER.debug('Forcing synchronous execution') + self._send_in_progress_notification(subscriber) processor = self.get_processor(process_id) try: jfmt, outputs = processor.execute(data_dict) current_status = JobStatus.successful + self._send_success_notification(subscriber, outputs) except Exception: outputs = { 'code': 'InvalidParameterValue', @@ -105,6 +109,7 @@ def execute_process( } current_status = JobStatus.failed LOGGER.exception('Process failed') + self._send_failed_notification(subscriber) job_id = str(uuid.uuid1()) return job_id, jfmt, outputs, current_status, response_headers From ab752498f078c2ec1cdd2b386a5136c559f83362 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 12:55:20 +0100 Subject: [PATCH 42/71] Remove test about format handling in endpoint This is now handled outside of the endpoint function --- tests/test_api.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index cd888346a..5fd49735d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1355,11 +1355,6 @@ def test_get_collection_items_json_ld(config, api_): def test_get_collection_item(config, api_): - req = mock_api_request({'f': 'foo'}) - rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') - - assert code == HTTPStatus.BAD_REQUEST - req = mock_api_request({'f': 'json'}) rsp_headers, code, response = get_collection_item( api_, req, 'gdps-temperature', '371') From 6ed3245c883014d559e723b163ed2f75b1bc3402 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 20 Mar 2024 07:29:57 -0400 Subject: [PATCH 43/71] add docstring to base process manager (#1603) --- pygeoapi/process/manager/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index 0b112c471..d4d8f23b4 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -194,6 +194,7 @@ def _execute_handler_async(self, p: BaseProcessor, job_id: str, :param p: `pygeoapi.process` object :param job_id: job identifier :param data_dict: `dict` of data parameters + :param subscriber: optional `Subscriber` specifying callback URLs :returns: tuple of None (i.e. initial response payload) and JobStatus.accepted (i.e. initial job status) From ff13394c714c34706b3f4742a894f930c4488b75 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 20 Mar 2024 08:07:06 -0400 Subject: [PATCH 44/71] backport of #1601 --- pygeoapi/django_/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/django_/urls.py b/pygeoapi/django_/urls.py index 25d4f59f8..81c9a62ff 100644 --- a/pygeoapi/django_/urls.py +++ b/pygeoapi/django_/urls.py @@ -147,8 +147,8 @@ def apply_slash_rule(url: str): name='collection-tiles-metadata', ), path( - 'collections//tiles/\ - ///', + 'collections//tiles/' + + '///', views.collection_item_tiles, name='collection-item-tiles', ), From 969c7e44a226d44d2bb888cae90af98e9e1b2b46 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 15:48:44 +0100 Subject: [PATCH 45/71] Port api ogr tests to new style --- tests/test_api_ogr_provider.py | 48 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/tests/test_api_ogr_provider.py b/tests/test_api_ogr_provider.py index 65d358783..cbf2e44ce 100644 --- a/tests/test_api_ogr_provider.py +++ b/tests/test_api_ogr_provider.py @@ -36,7 +36,9 @@ from pygeoapi.api import API from pygeoapi.util import yaml_load, geojson_to_geom -from .util import get_test_file_path, mock_request +from pygeoapi.api.itemtypes import get_collection_item, get_collection_items + +from .util import get_test_file_path, mock_api_request LOGGER = logging.getLogger(__name__) @@ -70,16 +72,16 @@ def test_get_collection_items_bbox_crs(config, api_): COLLECTIONS = ['dutch_addresses_4326', 'dutch_addresses_28992'] for coll in COLLECTIONS: # bbox-crs full extent - req = mock_request({'bbox': '5.670670, 52.042700, 5.829110, 52.123700', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '5.670670, 52.042700, 5.829110, 52.123700', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 # bbox-crs partial extent, 1 feature, request with multiple CRSs for crs in CRS_BBOX_DICT: - req = mock_request({'bbox': CRS_BBOX_DICT[crs], 'bbox-crs': crs}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': CRS_BBOX_DICT[crs], 'bbox-crs': crs}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 1 @@ -88,29 +90,29 @@ def test_get_collection_items_bbox_crs(config, api_): assert properties['huisnummer'] == '2' # bbox-crs outside extent - req = mock_request({'bbox': '5, 51.9, 5.1, 52.0', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '5, 51.9, 5.1, 52.0', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs outside extent - req = mock_request({'bbox': '130000, 440000, 140000, 450000', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/28992'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '130000, 440000, 140000, 450000', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/28992'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs outside extent - axis reversed CRS - req = mock_request({'bbox': '51.9, 5, 52.0, 5.1', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '51.9, 5, 52.0, 5.1', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs full extent - axis reversed CRS - req = mock_request({'bbox': '52.042700, 5.670670, 52.123700, 5.829110', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '52.042700, 5.670670, 52.123700, 5.829110', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -135,8 +137,8 @@ def test_get_collection_items_crs(config, api_): COLLECTIONS = ['dutch_addresses_4326', 'dutch_addresses_28992'] for coll in COLLECTIONS: # crs full extent to get target feature - req = mock_request({}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -145,13 +147,13 @@ def test_get_collection_items_crs(config, api_): # request with multiple CRSs for crs in CRS_DICT: # Do for query (/items) - req = mock_request({'crs': crs}) # noqa + req = mock_api_request({'crs': crs}) # noqa if crs == 'none': # Test for default bbox CRS - req = mock_request({}) # noqa + req = mock_api_request({}) # noqa crs = DEFAULT_CRS - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -173,8 +175,8 @@ def test_get_collection_items_crs(config, api_): assert test_geom.equals_exact(crs_geom, 1), f'coords not equal for CRS: {crs} {crs_geom} in COLL: {coll} {test_geom}' # noqa # Same for single Feature 'get' - req = mock_request({'crs': crs}) # noqa - rsp_headers, code, response = api_.get_collection_item(req, coll, feature_id) # noqa + req = mock_api_request({'crs': crs}) # noqa + rsp_headers, code, response = get_collection_item(api_, req, coll, feature_id) # noqa test_feature = json.loads(response) assert test_feature['id'] == feature_id @@ -195,13 +197,13 @@ def test_get_collection_items_crs(config, api_): # Test combining BBOX and BBOX-CRS for bbox_crs in CRS_BBOX_DICT: # Do for query (/items) - req = mock_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs], 'bbox-crs': bbox_crs}) # noqa + req = mock_api_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs], 'bbox-crs': bbox_crs}) # noqa if bbox_crs == 'none': # Test for default bbox CRS - req = mock_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs]}) # noqa + req = mock_api_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs]}) # noqa bbox_crs = DEFAULT_CRS - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 1 From 7768b957d7c97f0a8d4fabfa1171a3c115e4fb3d Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 15:44:23 +0100 Subject: [PATCH 46/71] Move processes tests to own file --- tests/api/__init__.py | 0 tests/api/test_processes.py | 428 ++++++++++++++++++++++++++++++++++++ tests/conftest.py | 52 +++++ tests/test_api.py | 405 +--------------------------------- 4 files changed, 483 insertions(+), 402 deletions(-) create mode 100644 tests/api/__init__.py create mode 100644 tests/api/test_processes.py create mode 100644 tests/conftest.py diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api/test_processes.py b/tests/api/test_processes.py new file mode 100644 index 000000000..b83305fae --- /dev/null +++ b/tests/api/test_processes.py @@ -0,0 +1,428 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus +import time +from unittest import mock + +from pygeoapi.api import FORMAT_TYPES, F_HTML, F_JSON +from pygeoapi.api.processes import ( + describe_processes, execute_process, delete_job, get_job_result, +) + +from tests.util import mock_api_request + + +def test_describe_processes(config, api_): + req = mock_api_request({'limit': 1}) + # Test for description of single processes + rsp_headers, code, response = describe_processes(api_, req) + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 1 + assert len(data['links']) == 3 + + req = mock_api_request() + + # Test for undefined process + rsp_headers, code, response = describe_processes(api_, req, 'foo') + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert data['code'] == 'NoSuchProcess' + + # Test for description of all processes + rsp_headers, code, response = describe_processes(api_, req) + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 2 + assert len(data['links']) == 3 + + # Test for particular, defined process + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + process = json.loads(response) + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert process['id'] == 'hello-world' + assert process['version'] == '0.2.0' + assert process['title'] == 'Hello World' + assert len(process['keywords']) == 3 + assert len(process['links']) == 6 + assert len(process['inputs']) == 2 + assert len(process['outputs']) == 1 + assert len(process['outputTransmission']) == 1 + assert len(process['jobControlOptions']) == 2 + assert 'sync-execute' in process['jobControlOptions'] + assert 'async-execute' in process['jobControlOptions'] + + # Check HTML response when requested in headers + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested in headers + req = mock_api_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'en-US' + + # Check HTML response when requested with query parameter + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested with query parameter + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested with French language parameter + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'fr-CA' + process = json.loads(response) + assert process['title'] == 'Bonjour le Monde' + + # Check JSON response when language requested in headers + req = mock_api_request(HTTP_ACCEPT_LANGUAGE='fr') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'fr-CA' + + # Test for undefined process + req = mock_api_request() + rsp_headers, code, response = describe_processes(api_, req, + 'goodbye-world') + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert data['code'] == 'NoSuchProcess' + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Test describe doesn't crash if example is missing + req = mock_api_request() + processor = api_.manager.get_processor("hello-world") + example = processor.metadata.pop("example") + rsp_headers, code, response = describe_processes(api_, req) + processor.metadata['example'] = example + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 2 + + +def test_execute_process(config, api_): + req_body_0 = { + 'inputs': { + 'name': 'Test' + } + } + req_body_1 = { + 'inputs': { + 'name': 'Test' + }, + 'response': 'document' + } + req_body_2 = { + 'inputs': { + 'name': 'Tést' + } + } + req_body_3 = { + 'inputs': { + 'name': 'Tést', + 'message': 'This is a test.' + } + } + req_body_4 = { + 'inputs': { + 'foo': 'Tést' + } + } + req_body_5 = { + 'inputs': {} + } + req_body_6 = { + 'inputs': { + 'name': None + } + } + req_body_7 = { + 'inputs': { + 'name': 'Test' + }, + 'subscriber': { + 'successUri': 'https://example.com/success', + 'inProgressUri': 'https://example.com/inProgress', + 'failedUri': 'https://example.com/failed', + } + } + + cleanup_jobs = set() + + # Test posting empty payload to existing process + req = mock_api_request(data='') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert rsp_headers['Content-Language'] == 'en-US' + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' not in rsp_headers + assert data['code'] == 'MissingParameterValue' + + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'foo') + + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert 'Location' not in rsp_headers + assert data['code'] == 'NoSuchProcess' + + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + + assert len(data.keys()) == 2 + assert data['id'] == 'echo' + assert data['value'] == 'Hello Test!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_1) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + + assert len(data.keys()) == 1 + assert data['outputs'][0]['id'] == 'echo' + assert data['outputs'][0]['value'] == 'Hello Test!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_2) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Tést!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_3) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Tést! This is a test.' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_4) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_5) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + assert data['description'] == 'Error updating job' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_6) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + assert data['description'] == 'Error updating job' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'goodbye-world') + + response = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert 'Location' not in rsp_headers + assert response['code'] == 'NoSuchProcess' + + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + response = json.loads(response) + assert code == HTTPStatus.OK + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_1, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + assert 'Location' in rsp_headers + response = json.loads(response) + assert isinstance(response, dict) + assert code == HTTPStatus.CREATED + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_7) + with mock.patch( + 'pygeoapi.process.manager.base.requests.post' + ) as post_mocker: + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert code == HTTPStatus.OK + post_mocker.assert_any_call( + req_body_7['subscriber']['inProgressUri'], json={} + ) + post_mocker.assert_any_call( + req_body_7['subscriber']['successUri'], + json={'id': 'echo', 'value': 'Hello Test!'} + ) + assert post_mocker.call_count == 2 + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + # Cleanup + time.sleep(2) # Allow time for any outstanding async jobs + for _, job_id in cleanup_jobs: + rsp_headers, code, response = delete_job(api_, mock_api_request(), + job_id) + assert code == HTTPStatus.OK + + +def _execute_a_job(api_): + req_body_sync = { + 'inputs': { + 'name': 'Sync Test' + } + } + + req = mock_api_request(data=req_body_sync) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Sync Test!' + + job_id = rsp_headers['Location'].split('/')[-1] + return job_id + + +def test_delete_job(api_): + rsp_headers, code, response = delete_job(api_, mock_api_request(), + 'does-not-exist') + + assert code == HTTPStatus.NOT_FOUND + req_body_async = { + 'inputs': { + 'name': 'Async Test Deletion' + } + } + job_id = _execute_a_job(api_) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + + assert code == HTTPStatus.OK + + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request(data=req_body_async, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + assert code == HTTPStatus.CREATED + assert 'Location' in rsp_headers + + time.sleep(2) # Allow time for async execution to complete + job_id = rsp_headers['Location'].split('/')[-1] + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.OK + + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.NOT_FOUND + + +def test_get_job_result(api_): + rsp_headers, code, response = get_job_result( + api_, mock_api_request(), 'not-exist', + ) + assert code == HTTPStatus.NOT_FOUND + + job_id = _execute_a_job(api_) + rsp_headers, code, response = get_job_result(api_, mock_api_request(), + job_id) + # default response is html + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'text/html' + assert 'Hello Sync Test!' in response + + rsp_headers, code, response = get_job_result( + api_, mock_api_request({'f': 'json'}), job_id, + ) + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'application/json' + assert json.loads(response)['value'] == "Hello Sync Test!" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..ddeb3b496 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Bernhard Mallinger +# +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import pytest + +from pygeoapi.api import API +from pygeoapi.util import yaml_load + +from tests.util import get_test_file_path + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def openapi(): + with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def api_(config, openapi): + return API(config, openapi) diff --git a/tests/test_api.py b/tests/test_api.py index 5fd49735d..a477b12d5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -33,10 +33,8 @@ import copy import json import logging -import time import gzip from http import HTTPStatus -from unittest import mock from pyld import jsonld import pytest @@ -53,9 +51,6 @@ get_collection_schema, get_collection_queryables, get_collection_item, get_collection_items, manage_collection_item) from pygeoapi.api.maps import get_collection_map -from pygeoapi.api.processes import ( - describe_processes, execute_process, delete_job, get_job_result, -) from pygeoapi.api.tiles import ( get_collection_tiles, tilematrixset, tilematrixsets, ) @@ -97,17 +92,6 @@ def config_hidden_resources(): return yaml_load(fh) -@pytest.fixture() -def openapi(): - with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def api_(config, openapi): - return API(config, openapi) - - @pytest.fixture() def enclosure_api(config_enclosure, openapi): """ Returns an API instance with a collection with enclosure links. """ @@ -435,7 +419,7 @@ def test_api_exception(config, api_): assert code == HTTPStatus.BAD_REQUEST -def test_gzip(config, api_): +def test_gzip(config, api_, openapi): # Requests for each response type and gzip encoding req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], HTTP_ACCEPT_ENCODING=F_GZIP) @@ -530,7 +514,7 @@ def test_gzip(config, api_): gzip.decompress(rsp_gzip_html).decode(enc_16) -def test_gzip_csv(config, api_): +def test_gzip_csv(config, api_, openapi): # Add gzip to server config['server']['gzip'] = True api_ = API(config, openapi) @@ -1283,7 +1267,7 @@ def test_manage_collection_item_read_only_options_req(config, api_): assert rsp_headers['Allow'] == 'HEAD, GET' -def test_manage_collection_item_editable_options_req(config): +def test_manage_collection_item_editable_options_req(config, openapi): """Test OPTIONS request on a editable items endpoint""" config = copy.deepcopy(config) config['resources']['obs']['providers'][0]['editable'] = True @@ -1609,389 +1593,6 @@ def test_get_collection_tiles(config, api_): assert len(content['tilesets']) > 0 -def test_describe_processes(config, api_): - req = mock_api_request({'limit': 1}) - # Test for description of single processes - rsp_headers, code, response = describe_processes(api_, req) - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 1 - assert len(data['links']) == 3 - - req = mock_api_request() - - # Test for undefined process - rsp_headers, code, response = describe_processes(api_, req, 'foo') - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert data['code'] == 'NoSuchProcess' - - # Test for description of all processes - rsp_headers, code, response = describe_processes(api_, req) - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 2 - assert len(data['links']) == 3 - - # Test for particular, defined process - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - process = json.loads(response) - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert process['id'] == 'hello-world' - assert process['version'] == '0.2.0' - assert process['title'] == 'Hello World' - assert len(process['keywords']) == 3 - assert len(process['links']) == 6 - assert len(process['inputs']) == 2 - assert len(process['outputs']) == 1 - assert len(process['outputTransmission']) == 1 - assert len(process['jobControlOptions']) == 2 - assert 'sync-execute' in process['jobControlOptions'] - assert 'async-execute' in process['jobControlOptions'] - - # Check HTML response when requested in headers - req = mock_api_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested in headers - req = mock_api_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'en-US' - - # Check HTML response when requested with query parameter - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested with query parameter - req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested with French language parameter - req = mock_api_request({'lang': 'fr'}) - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'fr-CA' - process = json.loads(response) - assert process['title'] == 'Bonjour le Monde' - - # Check JSON response when language requested in headers - req = mock_api_request(HTTP_ACCEPT_LANGUAGE='fr') - rsp_headers, code, response = describe_processes(api_, req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'fr-CA' - - # Test for undefined process - req = mock_api_request() - rsp_headers, code, response = describe_processes(api_, req, - 'goodbye-world') - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert data['code'] == 'NoSuchProcess' - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Test describe doesn't crash if example is missing - req = mock_api_request() - processor = api_.manager.get_processor("hello-world") - example = processor.metadata.pop("example") - rsp_headers, code, response = describe_processes(api_, req) - processor.metadata['example'] = example - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 2 - - -def test_execute_process(config, api_): - req_body_0 = { - 'inputs': { - 'name': 'Test' - } - } - req_body_1 = { - 'inputs': { - 'name': 'Test' - }, - 'response': 'document' - } - req_body_2 = { - 'inputs': { - 'name': 'Tést' - } - } - req_body_3 = { - 'inputs': { - 'name': 'Tést', - 'message': 'This is a test.' - } - } - req_body_4 = { - 'inputs': { - 'foo': 'Tést' - } - } - req_body_5 = { - 'inputs': {} - } - req_body_6 = { - 'inputs': { - 'name': None - } - } - req_body_7 = { - 'inputs': { - 'name': 'Test' - }, - 'subscriber': { - 'successUri': 'https://example.com/success', - 'inProgressUri': 'https://example.com/inProgress', - 'failedUri': 'https://example.com/failed', - } - } - - cleanup_jobs = set() - - # Test posting empty payload to existing process - req = mock_api_request(data='') - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - assert rsp_headers['Content-Language'] == 'en-US' - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' not in rsp_headers - assert data['code'] == 'MissingParameterValue' - - req = mock_api_request(data=req_body_0) - rsp_headers, code, response = execute_process(api_, req, 'foo') - - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert 'Location' not in rsp_headers - assert data['code'] == 'NoSuchProcess' - - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - - assert len(data.keys()) == 2 - assert data['id'] == 'echo' - assert data['value'] == 'Hello Test!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_1) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - - assert len(data.keys()) == 1 - assert data['outputs'][0]['id'] == 'echo' - assert data['outputs'][0]['value'] == 'Hello Test!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_2) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Tést!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_3) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Tést! This is a test.' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_4) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_5) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - assert data['description'] == 'Error updating job' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_6) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - assert data['description'] == 'Error updating job' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_0) - rsp_headers, code, response = execute_process(api_, req, 'goodbye-world') - - response = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert 'Location' not in rsp_headers - assert response['code'] == 'NoSuchProcess' - - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - response = json.loads(response) - assert code == HTTPStatus.OK - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_1, HTTP_Prefer='respond-async') - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - assert 'Location' in rsp_headers - response = json.loads(response) - assert isinstance(response, dict) - assert code == HTTPStatus.CREATED - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_api_request(data=req_body_7) - with mock.patch( - 'pygeoapi.process.manager.base.requests.post' - ) as post_mocker: - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - assert code == HTTPStatus.OK - post_mocker.assert_any_call( - req_body_7['subscriber']['inProgressUri'], json={} - ) - post_mocker.assert_any_call( - req_body_7['subscriber']['successUri'], - json={'id': 'echo', 'value': 'Hello Test!'} - ) - assert post_mocker.call_count == 2 - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - # Cleanup - time.sleep(2) # Allow time for any outstanding async jobs - for _, job_id in cleanup_jobs: - rsp_headers, code, response = delete_job(api_, mock_api_request(), - job_id) - assert code == HTTPStatus.OK - - -def _execute_a_job(api_): - req_body_sync = { - 'inputs': { - 'name': 'Sync Test' - } - } - - req = mock_api_request(data=req_body_sync) - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Sync Test!' - - job_id = rsp_headers['Location'].split('/')[-1] - return job_id - - -def test_delete_job(api_): - rsp_headers, code, response = delete_job(api_, mock_api_request(), - 'does-not-exist') - - assert code == HTTPStatus.NOT_FOUND - req_body_async = { - 'inputs': { - 'name': 'Async Test Deletion' - } - } - job_id = _execute_a_job(api_) - rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) - - assert code == HTTPStatus.OK - - rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request(data=req_body_async, HTTP_Prefer='respond-async') - rsp_headers, code, response = execute_process(api_, req, 'hello-world') - - assert code == HTTPStatus.CREATED - assert 'Location' in rsp_headers - - time.sleep(2) # Allow time for async execution to complete - job_id = rsp_headers['Location'].split('/')[-1] - rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) - assert code == HTTPStatus.OK - - rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) - assert code == HTTPStatus.NOT_FOUND - - -def test_get_job_result(api_): - rsp_headers, code, response = get_job_result( - api_, mock_api_request(), 'not-exist', - ) - assert code == HTTPStatus.NOT_FOUND - - job_id = _execute_a_job(api_) - rsp_headers, code, response = get_job_result(api_, mock_api_request(), - job_id) - # default response is html - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'text/html' - assert 'Hello Sync Test!' in response - - rsp_headers, code, response = get_job_result( - api_, mock_api_request({'f': 'json'}), job_id, - ) - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'application/json' - assert json.loads(response)['value'] == "Hello Sync Test!" - - def test_get_collection_edr_query(config, api_): # edr resource req = mock_api_request() From d91a2f1fa4ead18c0d2f04a65da0021851263cf7 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 15:55:47 +0100 Subject: [PATCH 47/71] Run api tests from new dir in CI --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de37862c6..baa38a2b8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,6 +110,7 @@ jobs: POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }} run: | pytest tests/test_api.py + pytest tests/api pytest tests/test_api_ogr_provider.py pytest tests/test_config.py pytest tests/test_csv__formatter.py From 5f83ae777f85b3ebb8dcd56769243e48fdf2838d Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 16:10:49 +0100 Subject: [PATCH 48/71] Move edr tests to own file --- .../api/test_environmental_data_retrieval.py | 226 ++++++++++++++++++ tests/test_api.py | 188 --------------- 2 files changed, 226 insertions(+), 188 deletions(-) create mode 100644 tests/api/test_environmental_data_retrieval.py diff --git a/tests/api/test_environmental_data_retrieval.py b/tests/api/test_environmental_data_retrieval.py new file mode 100644 index 000000000..0de57f023 --- /dev/null +++ b/tests/api/test_environmental_data_retrieval.py @@ -0,0 +1,226 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus + +from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query + +from tests.util import mock_api_request + + +def test_get_collection_edr_query(config, api_): + # edr resource + req = mock_api_request() + rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') + collection = json.loads(response) + parameter_names = list(collection['parameter_names'].keys()) + parameter_names.sort() + assert len(parameter_names) == 4 + assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] + + # no coords parameter + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # bad query type + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'corridor') + assert code == HTTPStatus.BAD_REQUEST + + # bad coords parameter + req = mock_api_request({'coords': 'gah'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # bad parameter_names parameter + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'parameter_names': 'bad' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # all parameters + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + + axes = list(data['domain']['axes'].keys()) + axes.sort() + assert len(axes) == 3 + assert axes == ['TIME', 'x', 'y'] + + assert data['domain']['axes']['x']['start'] == 11.0 + assert data['domain']['axes']['x']['stop'] == 11.0 + assert data['domain']['axes']['y']['start'] == 11.0 + assert data['domain']['axes']['y']['stop'] == 11.0 + + parameters = list(data['parameters'].keys()) + parameters.sort() + assert len(parameters) == 4 + assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] + + # single parameter + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'parameter_names': 'SST' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + + assert len(data['parameters'].keys()) == 1 + assert list(data['parameters'].keys())[0] == 'SST' + + # Zulu time zone + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + # bounded date range + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-01-17/2000-06-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-02-15T16:29:05.999999999' + assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' + assert time_dict['num'] == 5 + + # unbounded date range - start + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '../2000-06-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-01-16T06:00:00.000000000' + assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' + assert time_dict['num'] == 6 + + # unbounded date range - end + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-06-16/..' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-06-16T10:25:30.000000000' + assert time_dict['stop'] == '2000-12-16T01:20:05.999999996' + assert time_dict['num'] == 7 + + # some data + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + # no data + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.NO_CONTENT + + # position no coords + req = mock_api_request({ + 'datetime': '2000-01-17' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # cube bbox parameter 4 dimensional + req = mock_api_request({ + 'bbox': '0,0,10,10' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.OK + + # cube bad bbox parameter + req = mock_api_request({ + 'bbox': '0,0,10' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.BAD_REQUEST + + # cube no bbox parameter + req = mock_api_request({}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.BAD_REQUEST + + # cube decreasing latitude coords and S3 + req = mock_api_request({ + 'bbox': '-100,40,-99,45', + 'parameter_names': 'tmn', + 'datetime': '1994-01-01/1994-12-31', + }) + + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'usgs-prism', None, 'cube') + assert code == HTTPStatus.OK diff --git a/tests/test_api.py b/tests/test_api.py index a477b12d5..f319d3879 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -46,7 +46,6 @@ validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__, apply_gzip, ) from pygeoapi.api.coverages import get_collection_coverage -from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query from pygeoapi.api.itemtypes import ( get_collection_schema, get_collection_queryables, get_collection_item, get_collection_items, manage_collection_item) @@ -1592,193 +1591,6 @@ def test_get_collection_tiles(config, api_): assert len(content['links']) > 0 assert len(content['tilesets']) > 0 - -def test_get_collection_edr_query(config, api_): - # edr resource - req = mock_api_request() - rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') - collection = json.loads(response) - parameter_names = list(collection['parameter_names'].keys()) - parameter_names.sort() - assert len(parameter_names) == 4 - assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] - - # no coords parameter - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # bad query type - req = mock_api_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'corridor') - assert code == HTTPStatus.BAD_REQUEST - - # bad coords parameter - req = mock_api_request({'coords': 'gah'}) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # bad parameter_names parameter - req = mock_api_request({ - 'coords': 'POINT(11 11)', 'parameter_names': 'bad' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # all parameters - req = mock_api_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - - axes = list(data['domain']['axes'].keys()) - axes.sort() - assert len(axes) == 3 - assert axes == ['TIME', 'x', 'y'] - - assert data['domain']['axes']['x']['start'] == 11.0 - assert data['domain']['axes']['x']['stop'] == 11.0 - assert data['domain']['axes']['y']['start'] == 11.0 - assert data['domain']['axes']['y']['stop'] == 11.0 - - parameters = list(data['parameters'].keys()) - parameters.sort() - assert len(parameters) == 4 - assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] - - # single parameter - req = mock_api_request({ - 'coords': 'POINT(11 11)', 'parameter_names': 'SST' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - - assert len(data['parameters'].keys()) == 1 - assert list(data['parameters'].keys())[0] == 'SST' - - # Zulu time zone - req = mock_api_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - # bounded date range - req = mock_api_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-01-17/2000-06-16' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-02-15T16:29:05.999999999' - assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' - assert time_dict['num'] == 5 - - # unbounded date range - start - req = mock_api_request({ - 'coords': 'POINT(11 11)', - 'datetime': '../2000-06-16' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-01-16T06:00:00.000000000' - assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' - assert time_dict['num'] == 6 - - # unbounded date range - end - req = mock_api_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-06-16/..' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-06-16T10:25:30.000000000' - assert time_dict['stop'] == '2000-12-16T01:20:05.999999996' - assert time_dict['num'] == 7 - - # some data - req = mock_api_request({ - 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - # no data - req = mock_api_request({ - 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.NO_CONTENT - - # position no coords - req = mock_api_request({ - 'datetime': '2000-01-17' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # cube bbox parameter 4 dimensional - req = mock_api_request({ - 'bbox': '0,0,10,10' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.OK - - # cube bad bbox parameter - req = mock_api_request({ - 'bbox': '0,0,10' - }) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.BAD_REQUEST - - # cube no bbox parameter - req = mock_api_request({}) - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.BAD_REQUEST - - # cube decreasing latitude coords and S3 - req = mock_api_request({ - 'bbox': '-100,40,-99,45', - 'parameter_names': 'tmn', - 'datetime': '1994-01-01/1994-12-31', - }) - - rsp_headers, code, response = get_collection_edr_query( - api_, req, 'usgs-prism', None, 'cube') - assert code == HTTPStatus.OK - - def test_validate_bbox(): assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] From 6e5be3a214f22835a2efba1a82b401b60cfcad22 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 16:14:37 +0100 Subject: [PATCH 49/71] Move maps tests to own file --- tests/api/test_maps.py | 52 ++++++++++++++++++++++++++++++++++++++++++ tests/test_api.py | 15 +----------- 2 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 tests/api/test_maps.py diff --git a/tests/api/test_maps.py b/tests/api/test_maps.py new file mode 100644 index 000000000..3b284ac0a --- /dev/null +++ b/tests/api/test_maps.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus + +from pygeoapi.api.maps import get_collection_map + +from tests.util import mock_api_request + + +def test_get_collection_map(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_map(api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request() + rsp_headers, code, response = get_collection_map( + api_, req, 'mapserver_world_map') + assert code == HTTPStatus.OK + assert isinstance(response, bytes) + assert response[1:4] == b'PNG' diff --git a/tests/test_api.py b/tests/test_api.py index f319d3879..f5d6bf816 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -49,7 +49,6 @@ from pygeoapi.api.itemtypes import ( get_collection_schema, get_collection_queryables, get_collection_item, get_collection_items, manage_collection_item) -from pygeoapi.api.maps import get_collection_map from pygeoapi.api.tiles import ( get_collection_tiles, tilematrixset, tilematrixsets, ) @@ -1560,19 +1559,6 @@ def test_get_collection_coverage(config, api_): # assert code == HTTPStatus.NO_CONTENT -def test_get_collection_map(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_map(api_, req, 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request() - rsp_headers, code, response = get_collection_map( - api_, req, 'mapserver_world_map') - assert code == HTTPStatus.OK - assert isinstance(response, bytes) - assert response[1:4] == b'PNG' - - def test_get_collection_tiles(config, api_): req = mock_api_request() rsp_headers, code, response = get_collection_tiles(api_, req, 'obs') @@ -1591,6 +1577,7 @@ def test_get_collection_tiles(config, api_): assert len(content['links']) > 0 assert len(content['tilesets']) > 0 + def test_validate_bbox(): assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] From 880afcd3a4686040bf4b477edcdf0a4accdd36ec Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Wed, 20 Mar 2024 16:17:44 +0100 Subject: [PATCH 50/71] Move tiles tests to own file --- tests/api/test_tiles.py | 110 ++++++++++++++++++++++++++++++++++++++++ tests/test_api.py | 71 -------------------------- 2 files changed, 110 insertions(+), 71 deletions(-) create mode 100644 tests/api/test_tiles.py diff --git a/tests/api/test_tiles.py b/tests/api/test_tiles.py new file mode 100644 index 000000000..df11ca5f3 --- /dev/null +++ b/tests/api/test_tiles.py @@ -0,0 +1,110 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus + +from pygeoapi.api import FORMAT_TYPES, F_HTML +from pygeoapi.api.tiles import ( + get_collection_tiles, tilematrixset, tilematrixsets, +) +from pygeoapi.models.provider.base import TileMatrixSetEnum + +from tests.util import mock_api_request + + +def test_get_collection_tiles(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_tiles(api_, req, 'obs') + assert code == HTTPStatus.BAD_REQUEST + + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') + assert code == HTTPStatus.OK + + # Language settings should be ignored (return system default) + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') + assert rsp_headers['Content-Language'] == 'en-US' + content = json.loads(response) + assert len(content['links']) > 0 + assert len(content['tilesets']) > 0 + + +def test_tilematrixsets(config, api_): + req = mock_api_request() + rsp_headers, code, response = tilematrixsets(api_, req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'tileMatrixSets' in root + assert len(root['tileMatrixSets']) == 2 + assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad' \ + in root['tileMatrixSets'][0]['uri'] + assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ + in root['tileMatrixSets'][1]['uri'] + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixsets(api_, req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_tilematrixset(config, api_): + req = mock_api_request() + + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + for e in enums: + enum = e.tileMatrixSet + rsp_headers, code, response = tilematrixset(api_, req, enum) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'id' in root + assert root['id'] == enum + assert 'tileMatrices' in root + assert len(root['tileMatrices']) == 30 + + rsp_headers, code, response = tilematrixset(api_, req, 'foo') + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixset(api_, req, enum) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' diff --git a/tests/test_api.py b/tests/test_api.py index f5d6bf816..37c1e398a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -49,17 +49,12 @@ from pygeoapi.api.itemtypes import ( get_collection_schema, get_collection_queryables, get_collection_item, get_collection_items, manage_collection_item) -from pygeoapi.api.tiles import ( - get_collection_tiles, tilematrixset, tilematrixsets, -) from pygeoapi.util import (yaml_load, get_crs_from_uri, get_api_rules, get_base_url) from .util import (get_test_file_path, mock_request, mock_flask, mock_starlette, mock_api_request) -from pygeoapi.models.provider.base import TileMatrixSetEnum - LOGGER = logging.getLogger(__name__) @@ -626,53 +621,6 @@ def test_conformance(config, api_): assert rsp_headers['Content-Language'] == 'en-US' -def test_tilematrixsets(config, api_): - req = mock_api_request() - rsp_headers, code, response = tilematrixsets(api_, req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'tileMatrixSets' in root - assert len(root['tileMatrixSets']) == 2 - assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad' \ - in root['tileMatrixSets'][0]['uri'] - assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ - in root['tileMatrixSets'][1]['uri'] - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = tilematrixsets(api_, req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_tilematrixset(config, api_): - req = mock_api_request() - - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - for e in enums: - enum = e.tileMatrixSet - rsp_headers, code, response = tilematrixset(api_, req, enum) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'id' in root - assert root['id'] == enum - assert 'tileMatrices' in root - assert len(root['tileMatrices']) == 30 - - rsp_headers, code, response = tilematrixset(api_, req, 'foo') - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = tilematrixset(api_, req, enum) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - def test_describe_collections(config, api_): req = mock_request({"f": "foo"}) rsp_headers, code, response = api_.describe_collections(req) @@ -1559,25 +1507,6 @@ def test_get_collection_coverage(config, api_): # assert code == HTTPStatus.NO_CONTENT -def test_get_collection_tiles(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_tiles(api_, req, 'obs') - assert code == HTTPStatus.BAD_REQUEST - - rsp_headers, code, response = get_collection_tiles( - api_, req, 'naturalearth/lakes') - assert code == HTTPStatus.OK - - # Language settings should be ignored (return system default) - req = mock_api_request({'lang': 'fr'}) - rsp_headers, code, response = get_collection_tiles( - api_, req, 'naturalearth/lakes') - assert rsp_headers['Content-Language'] == 'en-US' - content = json.loads(response) - assert len(content['links']) > 0 - assert len(content['tilesets']) > 0 - - def test_validate_bbox(): assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] From 196da16bf642b61dfc92e425dff7bf7ee2068e4b Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Thu, 21 Mar 2024 08:55:52 +0100 Subject: [PATCH 51/71] Actually hide hidden layers in openapi --- pygeoapi/api/coverages.py | 4 ++-- pygeoapi/api/environmental_data_retrieval.py | 4 ++-- pygeoapi/api/itemtypes.py | 4 ++-- pygeoapi/api/maps.py | 4 ++-- pygeoapi/api/tiles.py | 4 ++-- pygeoapi/openapi.py | 18 ++++++++++++------ 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py index 63e52812c..8d5e8c678 100644 --- a/pygeoapi/api/coverages.py +++ b/pygeoapi/api/coverages.py @@ -201,14 +201,14 @@ def get_collection_coverage( def get_oas_30(cfg: dict, locale: str) -> dict: - from pygeoapi.openapi import OPENAPI_YAML + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections paths = {} collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') - for k, v in collections.items(): + for k, v in get_visible_collections(cfg).items(): try: load_plugin('provider', get_provider_by_type( collections[k]['providers'], 'coverage')) diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index 8c01895d9..d48ca0dba 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -207,7 +207,7 @@ def get_collection_edr_query(api: API, request: APIRequest, def get_oas_30(cfg: dict, locale: str) -> dict: - from pygeoapi.openapi import OPENAPI_YAML + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections LOGGER.debug('setting up edr endpoints') @@ -216,7 +216,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') - for k, v in collections.items(): + for k, v in get_visible_collections(cfg).items(): edr_extension = filter_providers_by_type( collections[k]['providers'], 'edr') diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 08c439002..5feadeac2 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -1376,7 +1376,7 @@ def set_content_crs_header( def get_oas_30(cfg: dict, locale: str) -> dict: - from pygeoapi.openapi import OPENAPI_YAML + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections properties = { 'name': 'properties', @@ -1399,7 +1399,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') - for k, v in collections.items(): + for k, v in get_visible_collections(cfg).items(): try: ptype = None diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index beb8b695c..20df69e68 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -244,7 +244,7 @@ def get_collection_map_legend(api: API, request: APIRequest, def get_oas_30(cfg: dict, locale: str) -> dict: - from pygeoapi.openapi import OPENAPI_YAML + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections LOGGER.debug('setting up maps endpoints') @@ -254,7 +254,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: 'type', 'collection') parameters = get_oas_30_parameters(cfg, locale) - for k, v in collections.items(): + for k, v in get_visible_collections(cfg).items(): map_extension = filter_providers_by_type( collections[k]['providers'], 'map') diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 75cc80c9d..f4d94a68a 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -437,7 +437,7 @@ def tilematrixset(api: API, def get_oas_30(cfg: dict, locale: str) -> dict: - from pygeoapi.openapi import OPENAPI_YAML + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections paths = {} @@ -445,7 +445,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') - for k, v in collections.items(): + for k, v in get_visible_collections(cfg).items(): tile_extension = filter_providers_by_type( collections[k]['providers'], 'tile') diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index b92fda87d..a701c7fb9 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -359,13 +359,8 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: items_f['schema']['enum'].append('csv') LOGGER.debug('setting up datasets') - collections = filter_dict_by_key_value(cfg['resources'], - 'type', 'collection') - for k, v in collections.items(): - if v.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {k}') - continue + for k, v in get_visible_collections(cfg).items(): name = l10n.translate(k, locale_) title = l10n.translate(v['title'], locale_) desc = l10n.translate(v['description'], locale_) @@ -610,6 +605,17 @@ def get_oas_30_parameters(cfg: dict, locale_: str): } +def get_visible_collections(cfg: dict) -> dict: + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + return { + k: v + for k, v in collections.items() + if v.get('visibility', 'default') != 'hidden' + } + + def get_config_schema(): schema_file = os.path.join(THISDIR, 'schemas', 'config', 'pygeoapi-config-0.x.yml') From fa9912ae0a81c94e073a9b73d72e02340f8270a6 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 22 Mar 2024 21:47:39 +0000 Subject: [PATCH 52/71] 1600 allow providing default value in config (#1604) --- docs/source/configuration.rst | 20 +++++++++++++++++ pygeoapi/util.py | 42 ++++++++++++++++++++++++----------- tests/test_util.py | 36 ++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index b9422d828..b11cfdd5f 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -412,6 +412,26 @@ Below is an example of how to integrate system environment variables in pygeoapi host: ${MY_HOST} port: ${MY_PORT} +Multiple environment variables are supported as follows: + +.. code-block:: yaml + + data: ${MY_HOST}:${MY_PORT} + +It is also possible to define a default value for a variable in case it does not exist in +the environment using a syntax like: ``value: ${ENV_VAR:-the default}`` + +.. code-block:: yaml + + server: + bind: + host: ${MY_HOST:-localhost} + port: ${MY_PORT:-5000} + metadata: + identification: + title: + en: This is pygeoapi host ${MY_HOST} and port ${MY_PORT:-5000}, nice to meet you! + Hierarchical collections ------------------------ diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 620b42a87..17ad7a7dd 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -163,23 +163,39 @@ def yaml_load(fh: IO) -> dict: :returns: `dict` representation of YAML """ - # support environment variables in config - # https://stackoverflow.com/a/55301129 - path_matcher = re.compile(r'.*\$\{([^}^{]+)\}.*') - - def path_constructor(loader, node): - env_var = path_matcher.match(node.value).group(1) - if env_var not in os.environ: - msg = f'Undefined environment variable {env_var} in config' - raise EnvironmentError(msg) - return get_typed_value(os.path.expandvars(node.value)) + # # support environment variables in config + # # https://stackoverflow.com/a/55301129 + + env_matcher = re.compile( + r'.*?\$\{(?P\w+)(:-(?P[^}]+))?\}') + + def env_constructor(loader, node): + result = "" + current_index = 0 + raw_value = node.value + for match_obj in env_matcher.finditer(raw_value): + groups = match_obj.groupdict() + varname_start = match_obj.span('varname')[0] + result += raw_value[current_index:(varname_start-2)] + if (var_value := os.getenv(groups['varname'])) is not None: + result += var_value + elif (default_value := groups.get('default')) is not None: + result += default_value + else: + raise EnvironmentError( + f'Could not find the {groups["varname"]!r} environment ' + f'variable' + ) + current_index = match_obj.end() + else: + result += raw_value[current_index:] + return get_typed_value(result) class EnvVarLoader(yaml.SafeLoader): pass - EnvVarLoader.add_implicit_resolver('!path', path_matcher, None) - EnvVarLoader.add_constructor('!path', path_constructor) - + EnvVarLoader.add_implicit_resolver('!env', env_matcher, None) + EnvVarLoader.add_constructor('!env', env_constructor) return yaml.load(fh, Loader=EnvVarLoader) diff --git a/tests/test_util.py b/tests/test_util.py index 98097f5ab..262cfcd8a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -31,6 +31,8 @@ from decimal import Decimal from contextlib import nullcontext as does_not_raise from copy import deepcopy +from io import StringIO +from unittest import mock import pytest from pyproj.exceptions import CRSError @@ -77,6 +79,40 @@ def test_yaml_load(config): util.yaml_load(fh) +@pytest.mark.parametrize('env,input_config,expected', [ + pytest.param({}, 'foo: something', {'foo': 'something'}, id='no-env-expansion'), # noqa E501 + pytest.param({'FOO': 'this'}, 'foo: ${FOO}', {'foo': 'this'}), # noqa E501 + pytest.param({'FOO': 'this'}, 'foo: the value is ${FOO}', {'foo': 'the value is this'}, id='no-need-for-yaml-tag'), # noqa E501 + pytest.param({}, 'foo: ${FOO:-some default}', {'foo': 'some default'}), # noqa E501 + pytest.param({'FOO': 'this', 'BAR': 'that'}, 'composite: ${FOO}:${BAR}', {'composite': 'this:that'}), # noqa E501 + pytest.param({}, 'composite: ${FOO:-default-foo}:${BAR:-default-bar}', {'composite': 'default-foo:default-bar'}), # noqa E501 + pytest.param( + { + 'HOST': 'fake-host', + 'USER': 'fake', + 'PASSWORD': 'fake-pass', + 'DB': 'fake-db' + }, + 'connection: postgres://${USER}:${PASSWORD}@${HOST}:${PORT:-5432}/${DB}', # noqa E501 + { + 'connection': 'postgres://fake:fake-pass@fake-host:5432/fake-db' + }, + id='multiple-no-need-yaml-tag' + ), +]) +def test_yaml_load_with_env_variables( + env: dict[str, str], input_config: str, expected): + + def mock_get_env(env_var_name): + result = env.get(env_var_name) + return result + + with mock.patch('pygeoapi.util.os') as mock_os: + mock_os.getenv.side_effect = mock_get_env + loaded_config = util.yaml_load(StringIO(input_config)) + assert loaded_config == expected + + def test_str2bool(): assert not util.str2bool(False) assert not util.str2bool('0') From f4878460f73e087ba0bf0a6d4021bf5eb312c4c7 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 25 Mar 2024 17:36:27 -0400 Subject: [PATCH 53/71] move coverages tests to own file --- tests/api/__init__.py | 32 +++++++ tests/api/test_coverages.py | 183 ++++++++++++++++++++++++++++++++++++ tests/test_api.py | 126 ------------------------- 3 files changed, 215 insertions(+), 126 deletions(-) create mode 100644 tests/api/test_coverages.py diff --git a/tests/api/__init__.py b/tests/api/__init__.py index e69de29bb..088240895 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -0,0 +1,32 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= diff --git a/tests/api/test_coverages.py b/tests/api/test_coverages.py new file mode 100644 index 000000000..dbbed9435 --- /dev/null +++ b/tests/api/test_coverages.py @@ -0,0 +1,183 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +import logging +from http import HTTPStatus + +import pytest + +from pygeoapi.api import FORMAT_TYPES, F_HTML +from pygeoapi.api.coverages import get_collection_coverage +from pygeoapi.api.itemtypes import get_collection_schema +from pygeoapi.util import yaml_load + +from tests.util import get_test_file_path, mock_request, mock_api_request + +LOGGER = logging.getLogger(__name__) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +def test_describe_collections(config, api_): + + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'gdps-temperature') + + collection = json.loads(response) + + assert collection['id'] == 'gdps-temperature' + assert len(collection['links']) == 10 + assert collection['extent']['spatial']['grid'][0]['cellsCount'] == 2400 + assert collection['extent']['spatial']['grid'][0]['resolution'] == 0.15000000000000002 # noqa + assert collection['extent']['spatial']['grid'][1]['cellsCount'] == 1201 + assert collection['extent']['spatial']['grid'][1]['resolution'] == 0.15 + + +def test_get_collection_schema(config, api_): + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_schema( + api_, req, 'gdps-temperature') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_schema( + api_, req, 'gdps-temperature') + + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 1 + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_schema( + api_, req, 'gdps-temperature') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 1 + print(schema['properties']) + assert schema['properties']['1']['type'] == 'number' + assert schema['properties']['1']['title'] == 'Temperature [C]' + + +def test_get_collection_coverage(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_coverage( + api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'properties': '12'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'subset': 'bad_axis(10:20)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'blah'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + assert rsp_headers['Content-Type'] == 'text/html' + + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + # NOTE: This test used to assert the code to be 200 OK, + # but it requested HTML, which is not available, + # so it should be 400 Bad Request + assert code == HTTPStatus.BAD_REQUEST + assert rsp_headers['Content-Type'] == 'text/html' + + req = mock_api_request({'subset': 'Lat(5:10),Long(5:10)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + content = json.loads(response) + + assert content['domain']['axes']['x']['num'] == 35 + assert content['domain']['axes']['y']['num'] == 35 + assert 'TMP' in content['parameters'] + assert 'TMP' in content['ranges'] + assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] + + req = mock_api_request({'bbox': '-79,45,-75,49'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + content = json.loads(response) + + assert content['domain']['axes']['x']['start'] == -79.0 + assert content['domain']['axes']['x']['stop'] == -75.0 + assert content['domain']['axes']['y']['start'] == 49.0 + assert content['domain']['axes']['y']['stop'] == 45.0 + + req = mock_api_request({ + 'subset': 'Lat(5:10),Long(5:10)', + 'f': 'GRIB' + }) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + assert isinstance(response, bytes) + + req = mock_api_request(HTTP_ACCEPT='application/x-netcdf') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'cmip5') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'application/x-netcdf' diff --git a/tests/test_api.py b/tests/test_api.py index 37c1e398a..6c4bf7dbc 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -45,7 +45,6 @@ API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__, apply_gzip, ) -from pygeoapi.api.coverages import get_collection_coverage from pygeoapi.api.itemtypes import ( get_collection_schema, get_collection_queryables, get_collection_item, get_collection_items, manage_collection_item) @@ -691,18 +690,6 @@ def test_describe_collections(config, api_): assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] assert rsp_headers['Content-Language'] == 'en-US' - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req, - 'gdps-temperature') - collection = json.loads(response) - - assert collection['id'] == 'gdps-temperature' - assert len(collection['links']) == 10 - assert collection['extent']['spatial']['grid'][0]['cellsCount'] == 2400 - assert collection['extent']['spatial']['grid'][0]['resolution'] == 0.15000000000000002 # noqa - assert collection['extent']['spatial']['grid'][1]['cellsCount'] == 1201 - assert collection['extent']['spatial']['grid'][1]['resolution'] == 0.15 - # hiearchical collections rsp_headers, code, response = api_.describe_collections( req, 'naturalearth/lakes') @@ -754,16 +741,6 @@ def test_get_collection_schema(config, api_): assert 'properties' in schema assert len(schema['properties']) == 5 - req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_schema( - api_, req, 'gdps-temperature') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 1 - assert schema['properties']['1']['type'] == 'number' - def test_get_collection_queryables(config, api_): req = mock_api_request() @@ -1404,109 +1381,6 @@ def test_get_collection_item_json_ld(config, api_): assert rsp_headers['Content-Language'] == 'fr-CA' -def test_get_collection_coverage(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_coverage( - api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'properties': '12'}) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'subset': 'bad_axis(10:20)'}) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'f': 'blah'}) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - assert rsp_headers['Content-Type'] == 'text/html' - - req = mock_api_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - # NOTE: This test used to assert the code to be 200 OK, - # but it requested HTML, which is not available, - # so it should be 400 Bad Request - assert code == HTTPStatus.BAD_REQUEST - assert rsp_headers['Content-Type'] == 'text/html' - - req = mock_api_request({'subset': 'Lat(5:10),Long(5:10)'}) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.OK - content = json.loads(response) - - assert content['domain']['axes']['x']['num'] == 35 - assert content['domain']['axes']['y']['num'] == 35 - assert 'TMP' in content['parameters'] - assert 'TMP' in content['ranges'] - assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] - - req = mock_api_request({'bbox': '-79,45,-75,49'}) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.OK - content = json.loads(response) - - assert content['domain']['axes']['x']['start'] == -79.0 - assert content['domain']['axes']['x']['stop'] == -75.0 - assert content['domain']['axes']['y']['start'] == 49.0 - assert content['domain']['axes']['y']['stop'] == 45.0 - - req = mock_api_request({ - 'subset': 'Lat(5:10),Long(5:10)', - 'f': 'GRIB' - }) - rsp_headers, code, response = get_collection_coverage( - api_, req, 'gdps-temperature') - - assert code == HTTPStatus.OK - assert isinstance(response, bytes) - - req = mock_api_request(HTTP_ACCEPT='application/x-netcdf') - rsp_headers, code, response = get_collection_coverage( - api_, req, 'cmip5') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'application/x-netcdf' - - # req = mock_api_request({ - # 'subset': 'time("2006-07-01T06:00:00":"2007-07-01T06:00:00")' - # }) - # rsp_headers, code, response = get_collection_coverage(api_, req, 'cmip5') - # - # assert code == HTTPStatus.OK - # assert isinstance(json.loads(response), dict) - - # req = mock_api_request({'subset': 'lat(1:2'}) - # rsp_headers, code, response = get_collection_coverage(api_, req, 'cmip5') - # - # assert code == HTTPStatus.BAD_REQUEST - # - # req = mock_api_request({'subset': 'lat(1:2)'}) - # rsp_headers, code, response = get_collection_coverage(api_ req, 'cmip5') - # - # assert code == HTTPStatus.NO_CONTENT - - def test_validate_bbox(): assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] From 6be32aa2962e40f3876e0a2112f8654224341c79 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 25 Mar 2024 18:21:35 -0400 Subject: [PATCH 54/71] move itemtypes to own file, move core into init test --- .github/workflows/main.yml | 1 - docs/source/development.rst | 2 +- tests/api/__init__.py | 821 +++++++++++++++++++ tests/api/test_itemtypes.py | 680 ++++++++++++++++ tests/test_api.py | 1479 ----------------------------------- 5 files changed, 1502 insertions(+), 1481 deletions(-) create mode 100644 tests/api/test_itemtypes.py delete mode 100644 tests/test_api.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index baa38a2b8..3368f985c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,7 +109,6 @@ jobs: env: POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }} run: | - pytest tests/test_api.py pytest tests/api pytest tests/test_api_ogr_provider.py pytest tests/test_config.py diff --git a/docs/source/development.rst b/docs/source/development.rst index 561f08349..59941d780 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -20,7 +20,7 @@ Tests can be run locally as part of development workflow. They are also run on `GitHub Actions setup`_ against all commits and pull requests to the code repository. To run all tests, simply run ``pytest`` in the repository. To run a specific test file, -run ``pytest tests/test_api.py``, for example. +run ``pytest tests/api/test_itemtypes.py``, for example. CQL extension lifecycle diff --git a/tests/api/__init__.py b/tests/api/__init__.py index 088240895..b2c013dc7 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -30,3 +30,824 @@ # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= + +import json +import logging +import gzip +from http import HTTPStatus + +from pyld import jsonld +import pytest + +from pygeoapi.api import ( + API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP, + __version__, validate_bbox, validate_datetime, validate_subset +) +from pygeoapi.util import yaml_load, get_api_rules, get_base_url + +from tests.util import (get_test_file_path, mock_flask, mock_starlette, + mock_request) + +LOGGER = logging.getLogger(__name__) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_with_rules() -> dict: + """ Returns a pygeoapi configuration with default API rules. """ + with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_enclosure() -> dict: + """ Returns a pygeoapi configuration with enclosure links. """ + with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_hidden_resources(): + filename = 'pygeoapi-test-config-hidden-resources.yml' + with open(get_test_file_path(filename)) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def enclosure_api(config_enclosure, openapi): + """ Returns an API instance with a collection with enclosure links. """ + return API(config_enclosure, openapi) + + +@pytest.fixture() +def rules_api(config_with_rules, openapi): + """ Returns an API instance with URL prefix and strict slashes policy. + The API version is extracted from the current version here. + """ + return API(config_with_rules, openapi) + + +@pytest.fixture() +def api_hidden_resources(config_hidden_resources, openapi): + return API(config_hidden_resources, openapi) + + +def test_apirequest(api_): + # Test without (valid) locales + with pytest.raises(ValueError): + req = mock_request() + APIRequest(req, []) + APIRequest(req, None) + APIRequest(req, ['zz']) + + # Test all supported formats from query args + for f, mt in FORMAT_TYPES.items(): + req = mock_request({'f': f}) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test all supported formats from Accept header + for f, mt in FORMAT_TYPES.items(): + req = mock_request(HTTP_ACCEPT=mt) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test nonsense format + req = mock_request({'f': 'foo'}) + apireq = APIRequest(req, api_.locales) + assert not apireq.is_valid() + assert apireq.format == 'foo' + assert apireq.is_valid(('foo',)) + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + + # Test without format + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format is None + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + assert apireq.get_linkrel(F_JSON) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Test complex format string + hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + assert apireq.get_linkrel(F_HTML) == 'self' + assert apireq.get_linkrel(F_JSON) == 'alternate' + + # Test accept header with multiple valid formats + hh = 'plain/text,application/ld+json,application/json;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_JSONLD + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSONLD] + assert apireq.get_linkrel(F_JSONLD) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Overrule HTTP content negotiation + req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + + # Test data + for d in (None, '', 'test', {'key': 'value'}): + req = mock_request(data=d) + apireq = APIRequest.with_data(req, api_.locales) + if not d: + assert apireq.data == b'' + elif isinstance(d, dict): + assert d == json.loads(apireq.data) + else: + assert apireq.data == d.encode() + + # Test multilingual + test_lang = { + 'nl': ('en', 'en-US'), # unsupported lang should return default + 'en-US': ('en', 'en-US'), + 'de_CH': ('en', 'en-US'), + 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), + 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), + } + sup_lang = ('en-US', 'fr_CA') + for lang_in, (lang_out, cl_out) in test_lang.items(): + # Using l query parameter + req = mock_request({'lang': lang_in}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Using Accept-Language header + req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Test language override + req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'fr' + assert apireq.locale.language == 'fr' + assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' + + # Test locale territory + req = mock_request({'lang': 'en-GB'}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'en-GB' + assert apireq.locale.language == 'en' + assert apireq.locale.territory == 'US' + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query parameter + # (should return default language from YAML config) + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.raw_locale is None + assert apireq.locale.language == api_.default_locale.language + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query param + # (should return first in custom list of languages) + sup_lang = ('de', 'fr', 'en') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale is None + assert apireq.locale.language == 'de' + assert apireq.get_response_headers()['Content-Language'] == 'de' + + +def test_apirules_active(config_with_rules, rules_api): + assert rules_api.config == config_with_rules + rules = get_api_rules(config_with_rules) + base_url = get_base_url(config_with_rules) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: + # Test happy path + response = flask_client.get(f'{flask_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = flask_client.get(f'{flask_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = flask_client.get(f'{flask_prefix}/') + assert response.status_code == 200 + response = flask_client.get(flask_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = flask_client.get(flask_prefix, follow_redirects=True) + assert response.status_code == 200 + assert response.is_json + links = response.json['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa + # Test happy path + response = starlette_client.get(f'{starlette_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = starlette_client.get(f'{starlette_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = starlette_client.get(f'{starlette_prefix}/') + assert response.status_code == 200 + response = starlette_client.get(starlette_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa + assert response.status_code == 200 + links = response.json()['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + +def test_apirules_inactive(config, api_): + assert api_.config == config + rules = get_api_rules(config) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + assert flask_prefix == '' + with mock_flask('pygeoapi-test-config.yml') as flask_client: + response = flask_client.get('') + assert response.status_code == 200 + response = flask_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = flask_client.get('/') + assert response.status_code == 200 + response = flask_client.get('/conformance/') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + assert starlette_prefix == '' + with mock_starlette('pygeoapi-test-config.yml') as starlette_client: + response = starlette_client.get('') + assert response.status_code == 200 + response = starlette_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert str(response.url) == f"{starlette_client.base_url}/conformance" + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = starlette_client.get('/') + assert response.status_code == 200 + response = starlette_client.get('/conformance/', follow_redirects=True) + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + +def test_api(config, api_, openapi): + assert api_.config == config + assert isinstance(api_.config, dict) + + req = mock_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + root = json.loads(response) + assert isinstance(root, dict) + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request(HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'Swagger UI' in response + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'ReDoc' in response + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + assert api_.get_collections_url() == 'http://localhost:5000/collections' + + +def test_api_exception(config, api_): + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + # When a language is set, the exception should still be English + req = mock_request({'f': 'foo', 'lang': 'fr'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + +def test_gzip(config, api_, openapi): + # Requests for each response type and gzip encoding + req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', + HTTP_ACCEPT_ENCODING=F_GZIP) + + # Responses from server config without gzip compression + rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Add gzip to server and use utf-16 encoding + config['server']['gzip'] = True + enc_16 = 'utf-16' + config['server']['encoding'] = enc_16 + api_ = API(config, openapi) + + # Responses from server with gzip compression + rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) + rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) + rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) + rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) + + # Validate compressed json response + assert rsp_json_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' + assert rsp_json_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) + assert isinstance(parsed_gzip_json, str) + parsed_gzip_json = json.loads(parsed_gzip_json) + assert isinstance(parsed_gzip_json, dict) + assert parsed_gzip_json == json.loads(rsp_json) + + # Validate compressed jsonld response + assert rsp_jsonld_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' + assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + assert isinstance(parsed_gzip_jsonld, str) + parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) + assert isinstance(parsed_gzip_jsonld, dict) + assert parsed_gzip_jsonld == json.loads(rsp_jsonld) + + # Validate compressed html response + assert rsp_html_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' + assert rsp_html_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) + assert isinstance(parsed_gzip_html, str) + assert parsed_gzip_html == rsp_html + + # Validate compressed gzip response + assert rsp_gzip_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' + assert rsp_gzip_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) + assert isinstance(parsed_gzip_gzip, str) + parsed_gzip_gzip = json.loads(parsed_gzip_gzip) + assert isinstance(parsed_gzip_gzip, dict) + + # Requests without content encoding header + req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) + req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) + req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) + + # Responses without content encoding + _, _, rsp_json_ = api_.landing_page(req_json) + _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) + _, _, rsp_html_ = api_.landing_page(req_html) + + # Confirm each request is the same when decompressed + assert rsp_json_ == rsp_json == \ + gzip.decompress(rsp_gzip_json).decode(enc_16) + + assert rsp_jsonld_ == rsp_jsonld == \ + gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + + assert rsp_html_ == rsp_html == \ + gzip.decompress(rsp_gzip_html).decode(enc_16) + + +def test_root(config, api_): + req = mock_request() + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/json' == \ + FORMAT_TYPES[F_JSON] + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + assert rsp_headers['Content-Language'] == 'en-US' + + assert isinstance(root, dict) + assert 'links' in root + assert root['links'][0]['rel'] == 'self' + assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] + assert root['links'][0]['href'].endswith('?f=json') + assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' + for link in root['links']) + assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' + for link in root['links']) + assert len(root['links']) == 11 + assert 'title' in root + assert root['title'] == 'pygeoapi default instance' + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_root_structured_data(config, api_): + req = mock_request({"f": "jsonld"}) + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/ld+json' == \ + FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + + assert isinstance(root, dict) + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + assert '@context' in root + assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' + expanded = jsonld.expand(root)[0] + assert '@type' in expanded + assert 'http://schema.org/DataCatalog' in expanded['@type'] + assert 'http://schema.org/description' in expanded + assert root['description'] == expanded['http://schema.org/description'][0][ + '@value'] + assert 'http://schema.org/keywords' in expanded + assert len(expanded['http://schema.org/keywords']) == 3 + assert '@value' in expanded['http://schema.org/keywords'][0].keys() + assert 'http://schema.org/provider' in expanded + assert expanded['http://schema.org/provider'][0]['@type'][ + 0] == 'http://schema.org/Organization' + assert expanded['http://schema.org/name'][0]['@value'] == root['name'] + + +def test_conformance(config, api_): + req = mock_request() + rsp_headers, code, response = api_.conformance(req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'conformsTo' in root + assert len(root['conformsTo']) == 37 + assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ + in root['conformsTo'] + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.conformance(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.conformance(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections(config, api_): + req = mock_request({"f": "foo"}) + rsp_headers, code, response = api_.describe_collections(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({"f": "html"}) + rsp_headers, code, response = api_.describe_collections(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request() + rsp_headers, code, response = api_.describe_collections(req) + collections = json.loads(response) + + assert len(collections) == 2 + assert len(collections['collections']) == 9 + assert len(collections['links']) == 3 + + rsp_headers, code, response = api_.describe_collections(req, 'foo') + collection = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'en-US' + assert collection['id'] == 'obs' + assert collection['title'] == 'Observations' + assert collection['description'] == 'My cool observations' + assert len(collection['links']) == 14 + assert collection['extent'] == { + 'spatial': { + 'bbox': [[-180, -90, 180, 90]], + 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + }, + 'temporal': { + 'interval': [ + ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] + ], + 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' + } + } + + # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults + assert collection['crs'] is not None + crs_set = [ + 'http://www.opengis.net/def/crs/EPSG/0/28992', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/EPSG/0/4326', + ] + for crs in crs_set: + assert crs in collection['crs'] + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert 'storageCrsCoordinateEpoch' not in collection + + # French language request + req = mock_request({'lang': 'fr'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'fr-CA' + assert collection['title'] == 'Observations' + assert collection['description'] == 'Mes belles observations' + + # Check HTML request in an unsupported language + req = mock_request({'f': 'html', 'lang': 'de'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + # hiearchical collections + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'naturalearth/lakes') + + collection = json.loads(response) + assert collection['id'] == 'naturalearth/lakes' + + # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured + assert collection['crs'] is not None + default_crs_list = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', + ] + contains_default = False + for crs in default_crs_list: + if crs in default_crs_list: + contains_default = True + assert contains_default + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert collection['storageCrsCoordinateEpoch'] == 2017.23 + + +def test_describe_collections_hidden_resources( + config_hidden_resources, api_hidden_resources): + req = mock_request({}) + rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa + assert code == HTTPStatus.OK + + assert len(config_hidden_resources['resources']) == 3 + + collections = json.loads(response) + assert len(collections['collections']) == 1 + + +def test_describe_collections_json_ld(config, api_): + req = mock_request({'f': 'jsonld'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert '@context' in collection + expanded = jsonld.expand(collection)[0] + # Metadata is about a schema:DataCollection that contains a schema:Dataset + assert not expanded['@id'].endswith('obs') + assert 'http://schema.org/dataset' in expanded + assert len(expanded['http://schema.org/dataset']) == 1 + dataset = expanded['http://schema.org/dataset'][0] + assert dataset['@type'][0] == 'http://schema.org/Dataset' + assert len(dataset['http://schema.org/distribution']) == 14 + assert all(dist['@type'][0] == 'http://schema.org/DataDownload' + for dist in dataset['http://schema.org/distribution']) + + assert 'http://schema.org/Organization' in expanded[ + 'http://schema.org/provider'][0]['@type'] + + assert 'http://schema.org/Place' in dataset[ + 'http://schema.org/spatial'][0]['@type'] + assert 'http://schema.org/GeoShape' in dataset[ + 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] + assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ + 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' + + assert 'http://schema.org/temporalCoverage' in dataset + assert dataset['http://schema.org/temporalCoverage'][0][ + '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections_enclosures(config_enclosure, enclosure_api): + original_enclosures = { + lnk['title']: lnk + for lnk in config_enclosure['resources']['objects']['links'] + if lnk['rel'] == 'enclosure' + } + + req = mock_request() + _, _, response = enclosure_api.describe_collections(req, 'objects') + features = json.loads(response) + modified_enclosures = { + lnk['title']: lnk for lnk in features['links'] + if lnk['rel'] == 'enclosure' + } + + # If type and length is set, do not verify/update link + assert original_enclosures['download link 1'] == \ + modified_enclosures['download link 1'] + # If length is missing, modify link type and length + assert original_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 2']['type'] + assert modified_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 3']['type'] + assert 'length' not in original_enclosures['download link 2'] + assert modified_enclosures['download link 2']['length'] > 0 + assert modified_enclosures['download link 2']['length'] == \ + modified_enclosures['download link 3']['length'] + assert original_enclosures['download link 3']['type'] != \ + modified_enclosures['download link 3']['type'] + + +def test_validate_bbox(): + assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] + assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] + assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] + assert (validate_bbox('-142.1,42.12,-52.22,84.4') == + [-142.1, 42.12, -52.22, 84.4]) + assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == + [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) + + assert (validate_bbox('177.0,65.0,-177.0,70.0') == + [177.0, 65.0, -177.0, 70.0]) + + with pytest.raises(ValueError): + validate_bbox('1,2,4') + + with pytest.raises(ValueError): + validate_bbox('1,2,4,5,6') + + with pytest.raises(ValueError): + validate_bbox('3,4,1,2') + + with pytest.raises(ValueError): + validate_bbox('1,2,6,4,5,3') + + +def test_validate_datetime(): + config = yaml_load(''' + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + ''') + + # test time instant + assert validate_datetime(config, '2004') == '2004' + assert validate_datetime(config, '2004-10') == '2004-10' + assert validate_datetime(config, '2001-10-30') == '2001-10-30' + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009-10-30') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-09-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-10-30T17:24:39Z') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-10-30T08:58:29Z') + + # test time envelope + assert validate_datetime(config, '2004/2005') == '2004/2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + assert validate_datetime(config, '2004/..') == '2004/..' + assert validate_datetime(config, '../2005') == '../2005' + assert validate_datetime(config, '2004/') == '2004/..' + assert validate_datetime(config, '/2005') == '../2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-11-01/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../2000-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../1999') + + +@pytest.mark.parametrize("value, expected", [ + ('time(2000-11-11)', {'time': ['2000-11-11']}), + ('time("2000-11-11")', {'time': ['2000-11-11']}), + ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), + ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa + ('lat(40)', {'lat': [40]}), + ('lat(0:40)', {'lat': [0, 40]}), + ('foo("bar")', {'foo': ['bar']}), + ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) +]) +def test_validate_subset(value, expected): + assert validate_subset(value) == expected + + with pytest.raises(ValueError): + validate_subset('foo("bar)') + + +def test_get_exception(config, api_): + d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') + assert d[0] == {} + assert d[1] == 500 + content = json.loads(d[2]) + assert content['code'] == 'NoApplicableCode' + assert content['description'] == 'oops' + + d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py new file mode 100644 index 000000000..5bb8b5de6 --- /dev/null +++ b/tests/api/test_itemtypes.py @@ -0,0 +1,680 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import copy +import gzip +import json +import logging +from http import HTTPStatus + +from pyld import jsonld +import pytest +import pyproj +from shapely.geometry import Point + +from pygeoapi.api import (API, FORMAT_TYPES, F_GZIP, F_HTML, F_JSONLD, + apply_gzip) +from pygeoapi.api.itemtypes import ( + get_collection_schema, get_collection_queryables, get_collection_item, + get_collection_items, manage_collection_item) +from pygeoapi.util import yaml_load, get_crs_from_uri + +from tests.util import get_test_file_path, mock_api_request + +LOGGER = logging.getLogger(__name__) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +def test_get_collection_schema(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_schema(api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_schema(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_schema(api_, req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 5 + + +def test_get_collection_queryables(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_queryables( + api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 5 + + # test with provider filtered properties + api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] + + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 2 + assert 'geometry' in queryables['properties'] + assert queryables['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Geometry.json' # noqa + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_get_collection_items(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'foo') + features = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'foo'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3,4c'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + # bbox-crs must be in configured values for Collection + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + # bbox-crs must be in configured values for Collection (CSV will ignore) + req = mock_api_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + # bbox-crs can be a default even if not configured + req = mock_api_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + # bbox-crs can be a default even if not configured + req = mock_api_request({'bbox': '4,52,5,53'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'f': 'html', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'fr-CA' + + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + assert len(features['features']) == 5 + + req = mock_api_request({'resulttype': 'hits'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 0 + + # Invalid limit + req = mock_api_request({'limit': 0}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'stn_id': '35'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 2 + assert features['numberMatched'] == 2 + + req = mock_api_request({'stn_id': '35', 'value': '93.9'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 1 + assert features['numberMatched'] == 1 + + req = mock_api_request({'limit': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 2 + assert features['features'][1]['properties']['stn_id'] == 35 + + links = features['links'] + assert len(links) == 4 + assert '/collections/obs/items?f=json' in links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld' in links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html' in links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs' in links[3]['href'] + assert links[3]['rel'] == 'collection' + + # Invalid offset + req = mock_api_request({'offset': -1}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'offset': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 3 + assert features['features'][1]['properties']['stn_id'] == 2147 + + links = features['links'] + assert len(links) == 5 + assert '/collections/obs/items?f=json' in links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld' in links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html' in links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0' in links[3]['href'] + assert links[3]['rel'] == 'prev' + assert '/collections/obs' in links[4]['href'] + assert links[4]['rel'] == 'collection' + + req = mock_api_request({ + 'offset': 1, + 'limit': 1, + 'bbox': '-180,90,180,90' + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 1 + + links = features['links'] + assert len(links) == 5 + assert '/collections/obs/items?f=json&limit=1&bbox=-180,90,180,90' in \ + links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld&limit=1&bbox=-180,90,180,90' in \ + links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html&limit=1&bbox=-180,90,180,90' in \ + links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0&limit=1&bbox=-180,90,180,90' \ + in links[3]['href'] + assert links[3]['rel'] == 'prev' + assert '/collections/obs' in links[4]['href'] + assert links[4]['rel'] == 'collection' + + req = mock_api_request({ + 'sortby': 'bad-property', + 'stn_id': '35' + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'sortby': 'stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'sortby': '+stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'sortby': '-stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'f': 'csv'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' + + req = mock_api_request({'datetime': '2003'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'datetime': '2010-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'datetime': '2001-11-11/2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '../2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '2001-11-11/..'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999/2005-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999/2000-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + api_.config['resources']['obs']['extents'].pop('temporal') + + req = mock_api_request({'datetime': '2002/2014-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'scalerank': 1}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') + features = json.loads(response) + + assert len(features['features']) == 10 + assert features['numberMatched'] == 11 + assert features['numberReturned'] == 10 + + req = mock_api_request({'datetime': '2005-04-22'}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'skipGeometry': 'true'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert json.loads(response)['features'][0]['geometry'] is None + + req = mock_api_request({'properties': 'foo,bar'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + +def test_collection_items_gzip_csv(config, api_, openapi): + # Add gzip to server + config['server']['gzip'] = True + api_ = API(config, openapi) + + req_csv = mock_api_request({'f': 'csv'}) + rsp_csv_headers, _, rsp_csv = get_collection_items(api_, req_csv, 'obs') + rsp_csv = apply_gzip(rsp_csv_headers, rsp_csv) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv = rsp_csv.decode('utf-8') + + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') + assert rsp_csv == rsp_csv_ + + # Use utf-16 encoding + config['server']['encoding'] = 'utf-16' + api_ = API(config, openapi) + + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') + assert rsp_csv == rsp_csv_ + + +def test_get_collection_items_crs(config, api_): + + # Invalid CRS query parameter + req = mock_api_request({'crs': '4326'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') + + assert code == HTTPStatus.BAD_REQUEST + + # Unsupported CRS + req = mock_api_request( + {'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') + + assert code == HTTPStatus.BAD_REQUEST + + # Supported CRSs + default_crs = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + storage_crs = 'http://www.opengis.net/def/crs/EPSG/0/25833' + crs_4258 = 'http://www.opengis.net/def/crs/EPSG/0/4258' + supported_crs_list = [default_crs, storage_crs, crs_4258] + + for crs in supported_crs_list: + req = mock_api_request({'crs': crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{crs}>' + + # With CRS query parameter, using storageCRS + req = mock_api_request({'crs': storage_crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' + + features_25833 = json.loads(response) + + # With CRS query parameter resulting in coordinates transformation + req = mock_api_request({'crs': crs_4258}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' + + features_4258 = json.loads(response) + transform_func = pyproj.Transformer.from_crs( + pyproj.CRS.from_epsg(25833), + pyproj.CRS.from_epsg(4258), + always_xy=False, + ).transform + for feat_orig in features_25833['features']: + id_ = feat_orig['id'] + x, y, *_ = feat_orig['geometry']['coordinates'] + loc_transf = Point(transform_func(x, y)) + for feat_out in features_4258['features']: + if id_ == feat_out['id']: + loc_out = Point(feat_out['geometry']['coordinates'][:2]) + + assert loc_out.equals_exact(loc_transf, 1e-5) + break + + # Without CRS query parameter: assume Transform to default WGS84 lon,lat + req = mock_api_request({}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{default_crs}>' + + features_wgs84 = json.loads(response) + + # With CRS query parameter resulting in coordinates transformation + transform_func = pyproj.Transformer.from_crs( + pyproj.CRS.from_epsg(4258), + get_crs_from_uri(default_crs), + always_xy=False, + ).transform + for feat_orig in features_4258['features']: + id_ = feat_orig['id'] + x, y, *_ = feat_orig['geometry']['coordinates'] + loc_transf = Point(transform_func(x, y)) + for feat_out in features_wgs84['features']: + if id_ == feat_out['id']: + loc_out = Point(feat_out['geometry']['coordinates'][:2]) + + assert loc_out.equals_exact(loc_transf, 1e-5) + break + + +def test_manage_collection_item_read_only_options_req(config, api_): + """Test OPTIONS request on a read-only items endpoint""" + req = mock_api_request() + _, code, _ = manage_collection_item(api_, req, 'options', 'foo') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET' + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET' + + +def test_manage_collection_item_editable_options_req(config, openapi): + """Test OPTIONS request on a editable items endpoint""" + config = copy.deepcopy(config) + config['resources']['obs']['providers'][0]['editable'] = True + api_ = API(config, openapi) + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET, POST' + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' + + +def test_get_collection_items_json_ld(config, api_): + req = mock_api_request({ + 'f': 'jsonld', + 'limit': 2 + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + collection = json.loads(response) + + assert '@context' in collection + assert all((f in collection['@context'][0] for + f in ('schema', 'type', 'features', 'FeatureCollection'))) + assert len(collection['@context']) > 1 + assert collection['@context'][1]['schema'] == 'https://schema.org/' + expanded = jsonld.expand(collection)[0] + featuresUri = 'https://schema.org/itemListElement' + assert len(expanded[featuresUri]) == 2 + + +def test_get_collection_item(config, api_): + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_item( + api_, req, 'gdps-temperature', '371') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'foo', '371') + + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = get_collection_item( + api_, req, 'obs', 'notfound') + + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + feature = json.loads(response) + + assert feature['properties']['stn_id'] == 35 + assert 'prev' not in feature['links'] + assert 'next' not in feature['links'] + + +def test_get_collection_item_json_ld(config, api_): + req = mock_api_request({'f': 'jsonld'}) + rsp_headers, _, response = get_collection_item(api_, req, 'objects', '3') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + feature = json.loads(response) + assert '@context' in feature + assert all((f in feature['@context'][0] for + f in ('schema', 'type', 'gsp'))) + assert len(feature['@context']) == 1 + assert 'schema' in feature['@context'][0] + assert feature['@context'][0]['schema'] == 'https://schema.org/' + assert feature['id'] == 3 + expanded = jsonld.expand(feature)[0] + + assert expanded['@id'].startswith('http://') + assert expanded['@id'].endswith('/collections/objects/items/3') + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'POINT (-85 33)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/latitude'][0][ + '@value'] == 33 + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/longitude'][0][ + '@value'] == -85 + + _, _, response = get_collection_item(api_, req, 'objects', '2') + feature = json.loads(response) + assert feature['geometry']['type'] == 'MultiPoint' + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTIPOINT (10 40, 40 30, 20 20, 30 10)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" + + _, _, response = get_collection_item(api_, req, 'objects', '1') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'LINESTRING (30 10, 10 30, 40 40)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/line'][0][ + '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' + + _, _, response = get_collection_item(api_, req, 'objects', '4') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTILINESTRING ((10 10, 20 20, 10 40), ' \ + '(40 40, 30 30, 40 20, 30 10))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/line'][0][ + '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ + '30.0,30.0 40.0,20.0 30.0,10.0' + + _, _, response = get_collection_item(api_, req, 'objects', '5') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' + + _, _, response = get_collection_item(api_, req, 'objects', '7') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '\ + '((15 5, 40 10, 10 20, 5 10, 15 5)))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ + '45.0,40.0 40.0,10.0 15.0,5.0' + + req = mock_api_request({'f': 'jsonld', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'fr-CA' diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index 6c4bf7dbc..000000000 --- a/tests/test_api.py +++ /dev/null @@ -1,1479 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# John A Stevenson -# Colin Blackburn -# -# Copyright (c) 2024 Tom Kralidis -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -import copy -import json -import logging -import gzip -from http import HTTPStatus - -from pyld import jsonld -import pytest -import pyproj -from shapely.geometry import Point - -from pygeoapi.api import ( - API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, - validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__, apply_gzip, -) -from pygeoapi.api.itemtypes import ( - get_collection_schema, get_collection_queryables, get_collection_item, - get_collection_items, manage_collection_item) -from pygeoapi.util import (yaml_load, get_crs_from_uri, - get_api_rules, get_base_url) - -from .util import (get_test_file_path, mock_request, - mock_flask, mock_starlette, mock_api_request) - -LOGGER = logging.getLogger(__name__) - - -@pytest.fixture() -def config(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_with_rules() -> dict: - """ Returns a pygeoapi configuration with default API rules. """ - with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_enclosure() -> dict: - """ Returns a pygeoapi configuration with enclosure links. """ - with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_hidden_resources(): - filename = 'pygeoapi-test-config-hidden-resources.yml' - with open(get_test_file_path(filename)) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def enclosure_api(config_enclosure, openapi): - """ Returns an API instance with a collection with enclosure links. """ - return API(config_enclosure, openapi) - - -@pytest.fixture() -def rules_api(config_with_rules, openapi): - """ Returns an API instance with URL prefix and strict slashes policy. - The API version is extracted from the current version here. - """ - return API(config_with_rules, openapi) - - -@pytest.fixture() -def api_hidden_resources(config_hidden_resources, openapi): - return API(config_hidden_resources, openapi) - - -def test_apirequest(api_): - # Test without (valid) locales - with pytest.raises(ValueError): - req = mock_request() - APIRequest(req, []) - APIRequest(req, None) - APIRequest(req, ['zz']) - - # Test all supported formats from query args - for f, mt in FORMAT_TYPES.items(): - req = mock_request({'f': f}) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test all supported formats from Accept header - for f, mt in FORMAT_TYPES.items(): - req = mock_request(HTTP_ACCEPT=mt) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test nonsense format - req = mock_request({'f': 'foo'}) - apireq = APIRequest(req, api_.locales) - assert not apireq.is_valid() - assert apireq.format == 'foo' - assert apireq.is_valid(('foo',)) - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - - # Test without format - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format is None - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - assert apireq.get_linkrel(F_JSON) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Test complex format string - hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - assert apireq.get_linkrel(F_HTML) == 'self' - assert apireq.get_linkrel(F_JSON) == 'alternate' - - # Test accept header with multiple valid formats - hh = 'plain/text,application/ld+json,application/json;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_JSONLD - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSONLD] - assert apireq.get_linkrel(F_JSONLD) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Overrule HTTP content negotiation - req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - - # Test data - for d in (None, '', 'test', {'key': 'value'}): - req = mock_request(data=d) - apireq = APIRequest.with_data(req, api_.locales) - if not d: - assert apireq.data == b'' - elif isinstance(d, dict): - assert d == json.loads(apireq.data) - else: - assert apireq.data == d.encode() - - # Test multilingual - test_lang = { - 'nl': ('en', 'en-US'), # unsupported lang should return default - 'en-US': ('en', 'en-US'), - 'de_CH': ('en', 'en-US'), - 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), - 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), - } - sup_lang = ('en-US', 'fr_CA') - for lang_in, (lang_out, cl_out) in test_lang.items(): - # Using l query parameter - req = mock_request({'lang': lang_in}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Using Accept-Language header - req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Test language override - req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'fr' - assert apireq.locale.language == 'fr' - assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' - - # Test locale territory - req = mock_request({'lang': 'en-GB'}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'en-GB' - assert apireq.locale.language == 'en' - assert apireq.locale.territory == 'US' - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query parameter - # (should return default language from YAML config) - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.raw_locale is None - assert apireq.locale.language == api_.default_locale.language - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query param - # (should return first in custom list of languages) - sup_lang = ('de', 'fr', 'en') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale is None - assert apireq.locale.language == 'de' - assert apireq.get_response_headers()['Content-Language'] == 'de' - - -def test_apirules_active(config_with_rules, rules_api): - assert rules_api.config == config_with_rules - rules = get_api_rules(config_with_rules) - base_url = get_base_url(config_with_rules) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: - # Test happy path - response = flask_client.get(f'{flask_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = flask_client.get(f'{flask_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = flask_client.get(f'{flask_prefix}/') - assert response.status_code == 200 - response = flask_client.get(flask_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = flask_client.get(flask_prefix, follow_redirects=True) - assert response.status_code == 200 - assert response.is_json - links = response.json['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa - # Test happy path - response = starlette_client.get(f'{starlette_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = starlette_client.get(f'{starlette_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = starlette_client.get(f'{starlette_prefix}/') - assert response.status_code == 200 - response = starlette_client.get(starlette_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa - assert response.status_code == 200 - links = response.json()['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - -def test_apirules_inactive(config, api_): - assert api_.config == config - rules = get_api_rules(config) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - assert flask_prefix == '' - with mock_flask('pygeoapi-test-config.yml') as flask_client: - response = flask_client.get('') - assert response.status_code == 200 - response = flask_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = flask_client.get('/') - assert response.status_code == 200 - response = flask_client.get('/conformance/') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - assert starlette_prefix == '' - with mock_starlette('pygeoapi-test-config.yml') as starlette_client: - response = starlette_client.get('') - assert response.status_code == 200 - response = starlette_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert str(response.url) == f"{starlette_client.base_url}/conformance" - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = starlette_client.get('/') - assert response.status_code == 200 - response = starlette_client.get('/conformance/', follow_redirects=True) - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - -def test_api(config, api_, openapi): - assert api_.config == config - assert isinstance(api_.config, dict) - - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - root = json.loads(response) - assert isinstance(root, dict) - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request(HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'Swagger UI' in response - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'ReDoc' in response - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - assert api_.get_collections_url() == 'http://localhost:5000/collections' - - -def test_api_exception(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - # When a language is set, the exception should still be English - req = mock_request({'f': 'foo', 'lang': 'fr'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - -def test_gzip(config, api_, openapi): - # Requests for each response type and gzip encoding - req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', - HTTP_ACCEPT_ENCODING=F_GZIP) - - # Responses from server config without gzip compression - rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Add gzip to server and use utf-16 encoding - config['server']['gzip'] = True - enc_16 = 'utf-16' - config['server']['encoding'] = enc_16 - api_ = API(config, openapi) - - # Responses from server with gzip compression - rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) - rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) - rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) - rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) - - # Validate compressed json response - assert rsp_json_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' - assert rsp_json_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) - assert isinstance(parsed_gzip_json, str) - parsed_gzip_json = json.loads(parsed_gzip_json) - assert isinstance(parsed_gzip_json, dict) - assert parsed_gzip_json == json.loads(rsp_json) - - # Validate compressed jsonld response - assert rsp_jsonld_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' - assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - assert isinstance(parsed_gzip_jsonld, str) - parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) - assert isinstance(parsed_gzip_jsonld, dict) - assert parsed_gzip_jsonld == json.loads(rsp_jsonld) - - # Validate compressed html response - assert rsp_html_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' - assert rsp_html_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) - assert isinstance(parsed_gzip_html, str) - assert parsed_gzip_html == rsp_html - - # Validate compressed gzip response - assert rsp_gzip_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' - assert rsp_gzip_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) - assert isinstance(parsed_gzip_gzip, str) - parsed_gzip_gzip = json.loads(parsed_gzip_gzip) - assert isinstance(parsed_gzip_gzip, dict) - - # Requests without content encoding header - req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) - req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) - req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) - - # Responses without content encoding - _, _, rsp_json_ = api_.landing_page(req_json) - _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) - _, _, rsp_html_ = api_.landing_page(req_html) - - # Confirm each request is the same when decompressed - assert rsp_json_ == rsp_json == \ - gzip.decompress(rsp_gzip_json).decode(enc_16) - - assert rsp_jsonld_ == rsp_jsonld == \ - gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - - assert rsp_html_ == rsp_html == \ - gzip.decompress(rsp_gzip_html).decode(enc_16) - - -def test_gzip_csv(config, api_, openapi): - # Add gzip to server - config['server']['gzip'] = True - api_ = API(config, openapi) - - req_csv = mock_api_request({'f': 'csv'}) - rsp_csv_headers, _, rsp_csv = get_collection_items(api_, req_csv, 'obs') - rsp_csv = apply_gzip(rsp_csv_headers, rsp_csv) - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv = rsp_csv.decode('utf-8') - - req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa - rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') - assert rsp_csv == rsp_csv_ - - # Use utf-16 encoding - config['server']['encoding'] = 'utf-16' - api_ = API(config, openapi) - - req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa - rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') - assert rsp_csv == rsp_csv_ - - -def test_root(config, api_): - req = mock_request() - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/json' == \ - FORMAT_TYPES[F_JSON] - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - assert rsp_headers['Content-Language'] == 'en-US' - - assert isinstance(root, dict) - assert 'links' in root - assert root['links'][0]['rel'] == 'self' - assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] - assert root['links'][0]['href'].endswith('?f=json') - assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' - for link in root['links']) - assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' - for link in root['links']) - assert len(root['links']) == 11 - assert 'title' in root - assert root['title'] == 'pygeoapi default instance' - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_root_structured_data(config, api_): - req = mock_request({"f": "jsonld"}) - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/ld+json' == \ - FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - - assert isinstance(root, dict) - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - assert '@context' in root - assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' - expanded = jsonld.expand(root)[0] - assert '@type' in expanded - assert 'http://schema.org/DataCatalog' in expanded['@type'] - assert 'http://schema.org/description' in expanded - assert root['description'] == expanded['http://schema.org/description'][0][ - '@value'] - assert 'http://schema.org/keywords' in expanded - assert len(expanded['http://schema.org/keywords']) == 3 - assert '@value' in expanded['http://schema.org/keywords'][0].keys() - assert 'http://schema.org/provider' in expanded - assert expanded['http://schema.org/provider'][0]['@type'][ - 0] == 'http://schema.org/Organization' - assert expanded['http://schema.org/name'][0]['@value'] == root['name'] - - -def test_conformance(config, api_): - req = mock_request() - rsp_headers, code, response = api_.conformance(req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'conformsTo' in root - assert len(root['conformsTo']) == 37 - assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ - in root['conformsTo'] - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.conformance(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.conformance(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections(config, api_): - req = mock_request({"f": "foo"}) - rsp_headers, code, response = api_.describe_collections(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({"f": "html"}) - rsp_headers, code, response = api_.describe_collections(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req) - collections = json.loads(response) - - assert len(collections) == 2 - assert len(collections['collections']) == 9 - assert len(collections['links']) == 3 - - rsp_headers, code, response = api_.describe_collections(req, 'foo') - collection = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'en-US' - assert collection['id'] == 'obs' - assert collection['title'] == 'Observations' - assert collection['description'] == 'My cool observations' - assert len(collection['links']) == 14 - assert collection['extent'] == { - 'spatial': { - 'bbox': [[-180, -90, 180, 90]], - 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - }, - 'temporal': { - 'interval': [ - ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] - ], - 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' - } - } - - # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults - assert collection['crs'] is not None - crs_set = [ - 'http://www.opengis.net/def/crs/EPSG/0/28992', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/EPSG/0/4326', - ] - for crs in crs_set: - assert crs in collection['crs'] - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert 'storageCrsCoordinateEpoch' not in collection - - # French language request - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'fr-CA' - assert collection['title'] == 'Observations' - assert collection['description'] == 'Mes belles observations' - - # Check HTML request in an unsupported language - req = mock_request({'f': 'html', 'lang': 'de'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - # hiearchical collections - rsp_headers, code, response = api_.describe_collections( - req, 'naturalearth/lakes') - collection = json.loads(response) - assert collection['id'] == 'naturalearth/lakes' - - # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured - assert collection['crs'] is not None - default_crs_list = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', - ] - contains_default = False - for crs in default_crs_list: - if crs in default_crs_list: - contains_default = True - assert contains_default - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert collection['storageCrsCoordinateEpoch'] == 2017.23 - - -def test_describe_collections_hidden_resources( - config_hidden_resources, api_hidden_resources): - req = mock_request({}) - rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa - assert code == HTTPStatus.OK - - assert len(config_hidden_resources['resources']) == 3 - - collections = json.loads(response) - assert len(collections['collections']) == 1 - - -def test_get_collection_schema(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_schema(api_, req, 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = get_collection_schema(api_, req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_schema(api_, req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 5 - - -def test_get_collection_queryables(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_queryables( - api_, req, 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - queryables = json.loads(response) - - assert 'properties' in queryables - assert len(queryables['properties']) == 5 - - # test with provider filtered properties - api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] - - rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') - queryables = json.loads(response) - - assert 'properties' in queryables - assert len(queryables['properties']) == 2 - assert 'geometry' in queryables['properties'] - assert queryables['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Geometry.json' # noqa - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert '@context' in collection - expanded = jsonld.expand(collection)[0] - # Metadata is about a schema:DataCollection that contains a schema:Dataset - assert not expanded['@id'].endswith('obs') - assert 'http://schema.org/dataset' in expanded - assert len(expanded['http://schema.org/dataset']) == 1 - dataset = expanded['http://schema.org/dataset'][0] - assert dataset['@type'][0] == 'http://schema.org/Dataset' - assert len(dataset['http://schema.org/distribution']) == 14 - assert all(dist['@type'][0] == 'http://schema.org/DataDownload' - for dist in dataset['http://schema.org/distribution']) - - assert 'http://schema.org/Organization' in expanded[ - 'http://schema.org/provider'][0]['@type'] - - assert 'http://schema.org/Place' in dataset[ - 'http://schema.org/spatial'][0]['@type'] - assert 'http://schema.org/GeoShape' in dataset[ - 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] - assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ - 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' - - assert 'http://schema.org/temporalCoverage' in dataset - assert dataset['http://schema.org/temporalCoverage'][0][ - '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_get_collection_items(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_items(api_, req, 'foo') - features = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request({'f': 'foo'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'bbox': '1,2,3'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'bbox': '1,2,3,4c'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'bbox-crs': 'bad_value'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - # bbox-crs must be in configured values for Collection - req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - # bbox-crs must be in configured values for Collection (CSV will ignore) - req = mock_api_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - # bbox-crs can be a default even if not configured - req = mock_api_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - # bbox-crs can be a default even if not configured - req = mock_api_request({'bbox': '4,52,5,53'}) # noqa - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'f': 'html', 'lang': 'fr'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'fr-CA' - - req = mock_api_request() - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - assert len(features['features']) == 5 - - req = mock_api_request({'resulttype': 'hits'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 0 - - # Invalid limit - req = mock_api_request({'limit': 0}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'stn_id': '35'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 2 - assert features['numberMatched'] == 2 - - req = mock_api_request({'stn_id': '35', 'value': '93.9'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 1 - assert features['numberMatched'] == 1 - - req = mock_api_request({'limit': 2}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 2 - assert features['features'][1]['properties']['stn_id'] == 35 - - links = features['links'] - assert len(links) == 4 - assert '/collections/obs/items?f=json' in links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld' in links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html' in links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs' in links[3]['href'] - assert links[3]['rel'] == 'collection' - - # Invalid offset - req = mock_api_request({'offset': -1}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'offset': 2}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 3 - assert features['features'][1]['properties']['stn_id'] == 2147 - - links = features['links'] - assert len(links) == 5 - assert '/collections/obs/items?f=json' in links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld' in links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html' in links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0' in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' - - req = mock_api_request({ - 'offset': 1, - 'limit': 1, - 'bbox': '-180,90,180,90' - }) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 1 - - links = features['links'] - assert len(links) == 5 - assert '/collections/obs/items?f=json&limit=1&bbox=-180,90,180,90' in \ - links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld&limit=1&bbox=-180,90,180,90' in \ - links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html&limit=1&bbox=-180,90,180,90' in \ - links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0&limit=1&bbox=-180,90,180,90' \ - in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' - - req = mock_api_request({ - 'sortby': 'bad-property', - 'stn_id': '35' - }) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'sortby': 'stn_id'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_api_request({'sortby': '+stn_id'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_api_request({'sortby': '-stn_id'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_api_request({'f': 'csv'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' - - req = mock_api_request({'datetime': '2003'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'datetime': '1999'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'datetime': '2010-04-22'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'datetime': '2001-11-11/2003-12-18'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'datetime': '../2003-12-18'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'datetime': '2001-11-11/..'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'datetime': '1999/2005-04-22'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'datetime': '1999/2000-04-22'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - api_.config['resources']['obs']['extents'].pop('temporal') - - req = mock_api_request({'datetime': '2002/2014-04-22'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_api_request({'scalerank': 1}) - rsp_headers, code, response = get_collection_items( - api_, req, 'naturalearth/lakes') - features = json.loads(response) - - assert len(features['features']) == 10 - assert features['numberMatched'] == 11 - assert features['numberReturned'] == 10 - - req = mock_api_request({'datetime': '2005-04-22'}) - rsp_headers, code, response = get_collection_items( - api_, req, 'naturalearth/lakes') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request({'skipGeometry': 'true'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert json.loads(response)['features'][0]['geometry'] is None - - req = mock_api_request({'properties': 'foo,bar'}) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - -def test_get_collection_items_crs(config, api_): - - # Invalid CRS query parameter - req = mock_api_request({'crs': '4326'}) - rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') - - assert code == HTTPStatus.BAD_REQUEST - - # Unsupported CRS - req = mock_api_request( - {'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) - rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') - - assert code == HTTPStatus.BAD_REQUEST - - # Supported CRSs - default_crs = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - storage_crs = 'http://www.opengis.net/def/crs/EPSG/0/25833' - crs_4258 = 'http://www.opengis.net/def/crs/EPSG/0/4258' - supported_crs_list = [default_crs, storage_crs, crs_4258] - - for crs in supported_crs_list: - req = mock_api_request({'crs': crs}) - rsp_headers, code, response = get_collection_items( - api_, req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{crs}>' - - # With CRS query parameter, using storageCRS - req = mock_api_request({'crs': storage_crs}) - rsp_headers, code, response = get_collection_items( - api_, req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' - - features_25833 = json.loads(response) - - # With CRS query parameter resulting in coordinates transformation - req = mock_api_request({'crs': crs_4258}) - rsp_headers, code, response = get_collection_items( - api_, req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' - - features_4258 = json.loads(response) - transform_func = pyproj.Transformer.from_crs( - pyproj.CRS.from_epsg(25833), - pyproj.CRS.from_epsg(4258), - always_xy=False, - ).transform - for feat_orig in features_25833['features']: - id_ = feat_orig['id'] - x, y, *_ = feat_orig['geometry']['coordinates'] - loc_transf = Point(transform_func(x, y)) - for feat_out in features_4258['features']: - if id_ == feat_out['id']: - loc_out = Point(feat_out['geometry']['coordinates'][:2]) - - assert loc_out.equals_exact(loc_transf, 1e-5) - break - - # Without CRS query parameter: assume Transform to default WGS84 lon,lat - req = mock_api_request({}) - rsp_headers, code, response = get_collection_items( - api_, req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{default_crs}>' - - features_wgs84 = json.loads(response) - - # With CRS query parameter resulting in coordinates transformation - transform_func = pyproj.Transformer.from_crs( - pyproj.CRS.from_epsg(4258), - get_crs_from_uri(default_crs), - always_xy=False, - ).transform - for feat_orig in features_4258['features']: - id_ = feat_orig['id'] - x, y, *_ = feat_orig['geometry']['coordinates'] - loc_transf = Point(transform_func(x, y)) - for feat_out in features_wgs84['features']: - if id_ == feat_out['id']: - loc_out = Point(feat_out['geometry']['coordinates'][:2]) - - assert loc_out.equals_exact(loc_transf, 1e-5) - break - - -def test_manage_collection_item_read_only_options_req(config, api_): - """Test OPTIONS request on a read-only items endpoint""" - req = mock_api_request() - _, code, _ = manage_collection_item(api_, req, 'options', 'foo') - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request() - rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET' - - req = mock_api_request() - rsp_headers, code, _ = manage_collection_item( - api_, req, 'options', 'obs', 'ressource_id') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET' - - -def test_manage_collection_item_editable_options_req(config, openapi): - """Test OPTIONS request on a editable items endpoint""" - config = copy.deepcopy(config) - config['resources']['obs']['providers'][0]['editable'] = True - api_ = API(config, openapi) - - req = mock_api_request() - rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET, POST' - - req = mock_api_request() - rsp_headers, code, _ = manage_collection_item( - api_, req, 'options', 'obs', 'ressource_id') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' - - -def test_describe_collections_enclosures(config_enclosure, enclosure_api): - original_enclosures = { - lnk['title']: lnk - for lnk in config_enclosure['resources']['objects']['links'] - if lnk['rel'] == 'enclosure' - } - - req = mock_request() - _, _, response = enclosure_api.describe_collections(req, 'objects') - features = json.loads(response) - modified_enclosures = { - lnk['title']: lnk for lnk in features['links'] - if lnk['rel'] == 'enclosure' - } - - # If type and length is set, do not verify/update link - assert original_enclosures['download link 1'] == \ - modified_enclosures['download link 1'] - # If length is missing, modify link type and length - assert original_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 2']['type'] - assert modified_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 3']['type'] - assert 'length' not in original_enclosures['download link 2'] - assert modified_enclosures['download link 2']['length'] > 0 - assert modified_enclosures['download link 2']['length'] == \ - modified_enclosures['download link 3']['length'] - assert original_enclosures['download link 3']['type'] != \ - modified_enclosures['download link 3']['type'] - - -def test_get_collection_items_json_ld(config, api_): - req = mock_api_request({ - 'f': 'jsonld', - 'limit': 2 - }) - rsp_headers, code, response = get_collection_items(api_, req, 'obs') - - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - collection = json.loads(response) - - assert '@context' in collection - assert all((f in collection['@context'][0] for - f in ('schema', 'type', 'features', 'FeatureCollection'))) - assert len(collection['@context']) > 1 - assert collection['@context'][1]['schema'] == 'https://schema.org/' - expanded = jsonld.expand(collection)[0] - featuresUri = 'https://schema.org/itemListElement' - assert len(expanded[featuresUri]) == 2 - - -def test_get_collection_item(config, api_): - req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_item( - api_, req, 'gdps-temperature', '371') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_api_request() - rsp_headers, code, response = get_collection_item(api_, req, 'foo', '371') - - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = get_collection_item( - api_, req, 'obs', 'notfound') - - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') - - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - req = mock_api_request() - rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') - feature = json.loads(response) - - assert feature['properties']['stn_id'] == 35 - assert 'prev' not in feature['links'] - assert 'next' not in feature['links'] - - -def test_get_collection_item_json_ld(config, api_): - req = mock_api_request({'f': 'jsonld'}) - rsp_headers, _, response = get_collection_item(api_, req, 'objects', '3') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - feature = json.loads(response) - assert '@context' in feature - assert all((f in feature['@context'][0] for - f in ('schema', 'type', 'gsp'))) - assert len(feature['@context']) == 1 - assert 'schema' in feature['@context'][0] - assert feature['@context'][0]['schema'] == 'https://schema.org/' - assert feature['id'] == 3 - expanded = jsonld.expand(feature)[0] - - assert expanded['@id'].startswith('http://') - assert expanded['@id'].endswith('/collections/objects/items/3') - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'POINT (-85 33)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/latitude'][0][ - '@value'] == 33 - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/longitude'][0][ - '@value'] == -85 - - _, _, response = get_collection_item(api_, req, 'objects', '2') - feature = json.loads(response) - assert feature['geometry']['type'] == 'MultiPoint' - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTIPOINT (10 40, 40 30, 20 20, 30 10)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" - - _, _, response = get_collection_item(api_, req, 'objects', '1') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'LINESTRING (30 10, 10 30, 40 40)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/line'][0][ - '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' - - _, _, response = get_collection_item(api_, req, 'objects', '4') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTILINESTRING ((10 10, 20 20, 10 40), ' \ - '(40 40, 30 30, 40 20, 30 10))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/line'][0][ - '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ - '30.0,30.0 40.0,20.0 30.0,10.0' - - _, _, response = get_collection_item(api_, req, 'objects', '5') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' - - _, _, response = get_collection_item(api_, req, 'objects', '7') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '\ - '((15 5, 40 10, 10 20, 5 10, 15 5)))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ - '45.0,40.0 40.0,10.0 15.0,5.0' - - req = mock_api_request({'f': 'jsonld', 'lang': 'fr'}) - rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'fr-CA' - - -def test_validate_bbox(): - assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] - assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] - assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] - assert (validate_bbox('-142.1,42.12,-52.22,84.4') == - [-142.1, 42.12, -52.22, 84.4]) - assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == - [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) - - assert (validate_bbox('177.0,65.0,-177.0,70.0') == - [177.0, 65.0, -177.0, 70.0]) - - with pytest.raises(ValueError): - validate_bbox('1,2,4') - - with pytest.raises(ValueError): - validate_bbox('1,2,4,5,6') - - with pytest.raises(ValueError): - validate_bbox('3,4,1,2') - - with pytest.raises(ValueError): - validate_bbox('1,2,6,4,5,3') - - -def test_validate_datetime(): - config = yaml_load(''' - temporal: - begin: 2000-10-30T18:24:39Z - end: 2007-10-30T08:57:29Z - ''') - - # test time instant - assert validate_datetime(config, '2004') == '2004' - assert validate_datetime(config, '2004-10') == '2004-10' - assert validate_datetime(config, '2001-10-30') == '2001-10-30' - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009-10-30') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-09-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-10-30T17:24:39Z') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-10-30T08:58:29Z') - - # test time envelope - assert validate_datetime(config, '2004/2005') == '2004/2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - assert validate_datetime(config, '2004/..') == '2004/..' - assert validate_datetime(config, '../2005') == '../2005' - assert validate_datetime(config, '2004/') == '2004/..' - assert validate_datetime(config, '/2005') == '../2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-11-01/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../2000-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../1999') - - -@pytest.mark.parametrize("value, expected", [ - ('time(2000-11-11)', {'time': ['2000-11-11']}), - ('time("2000-11-11")', {'time': ['2000-11-11']}), - ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), - ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa - ('lat(40)', {'lat': [40]}), - ('lat(0:40)', {'lat': [0, 40]}), - ('foo("bar")', {'foo': ['bar']}), - ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) -]) -def test_validate_subset(value, expected): - assert validate_subset(value) == expected - - with pytest.raises(ValueError): - validate_subset('foo("bar)') - - -def test_get_exception(config, api_): - d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') - assert d[0] == {} - assert d[1] == 500 - content = json.loads(d[2]) - assert content['code'] == 'NoApplicableCode' - assert content['description'] == 'oops' - - d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') - assert d[0] == {'Content-Type': 'text/html'} From a911b166c4dc1b5a8621cf71c4558e059c8d398d Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 25 Mar 2024 20:33:13 -0400 Subject: [PATCH 55/71] fix OpenAPI output --- pygeoapi/api/itemtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 5feadeac2..d4511f4cb 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -1436,7 +1436,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa {'$ref': '#/components/parameters/crs'}, # noqa {'$ref': '#/components/parameters/bbox-crs'}, - {'properties': coll_properties}, + coll_properties, {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa {'$ref': '#/components/parameters/skipGeometry'}, {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa @@ -1503,7 +1503,7 @@ def get_oas_30(cfg: dict, locale: str) -> dict: 'summary': f'Get {title} schema', 'description': description, 'tags': [k], - 'operationId': f'get{k.capitalize()}Queryables', + 'operationId': f'get{k.capitalize()}Schema', 'parameters': [ {'$ref': '#/components/parameters/f'}, {'$ref': '#/components/parameters/lang'} From c4982fe63c2cdf00e4361ea04a0ff1e13f6c54af Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 25 Mar 2024 20:53:15 -0400 Subject: [PATCH 56/71] update tests --- tests/test_api_ogr_provider.py | 4 +- tests/test_postgresql_provider.py | 68 +++++++++++++++---------------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/tests/test_api_ogr_provider.py b/tests/test_api_ogr_provider.py index cbf2e44ce..c29fc629e 100644 --- a/tests/test_api_ogr_provider.py +++ b/tests/test_api_ogr_provider.py @@ -4,7 +4,7 @@ # Authors: Tom Kralidis # # Copyright (c) 2019 Just van den Broecke -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -35,8 +35,8 @@ import pytest from pygeoapi.api import API -from pygeoapi.util import yaml_load, geojson_to_geom from pygeoapi.api.itemtypes import get_collection_item, get_collection_items +from pygeoapi.util import yaml_load, geojson_to_geom from .util import get_test_file_path, mock_api_request diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index 650a6096c..bd56aed0c 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -7,7 +7,7 @@ # Francesco Bartoli # # Copyright (c) 2019 Just van den Broecke -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Francesco Bartoli # @@ -47,7 +47,9 @@ from pygeofilter.parsers.ecql import parse from pygeoapi.api import API - +from pygeoapi.api.itemtypes import ( + get_collection_items, get_collection_item, post_collection_items +) from pygeoapi.provider.base import ( ProviderConnectionError, ProviderItemNotFoundError, @@ -433,8 +435,8 @@ def test_get_collection_items_postgresql_cql(pg_api_): 'filter-lang': 'cql-text', 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -446,8 +448,8 @@ def test_get_collection_items_postgresql_cql(pg_api_): req = mock_request({ 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -471,8 +473,8 @@ def test_get_collection_items_postgresql_cql_invalid_filter_language(pg_api_): 'filter-lang': 'cql-json', # Only cql-text is valid for GET 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -497,8 +499,8 @@ def test_get_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): req = mock_request({ 'filter': bad_cql }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -527,8 +529,8 @@ def test_post_collection_items_postgresql_cql(pg_api_): req = mock_request({ 'filter-lang': 'cql-json' }, data=cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -553,8 +555,8 @@ def test_post_collection_items_postgresql_cql_invalid_filter_language(pg_api_): req = mock_request({ 'filter-lang': 'cql-text' # Only cql-json is valid for POST }, data=cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -583,8 +585,8 @@ def test_post_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): req = mock_request({ 'filter-lang': 'cql-json' }, data=bad_cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -602,9 +604,8 @@ def test_get_collection_items_postgresql_crs(pg_api_): # Without CRS query parameter -> no coordinates transformation req = mock_request({'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK @@ -614,9 +615,8 @@ def test_get_collection_items_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') req = mock_request({'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -625,9 +625,8 @@ def test_get_collection_items_postgresql_crs(pg_api_): # With CRS query parameter resulting in coordinates transformation req = mock_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_32735}>' @@ -689,9 +688,8 @@ def test_get_collection_item_postgresql_crs(pg_api_): for fid in fid_list: # Without CRS query parameter -> no coordinates transformation req = mock_request({'f': 'json'}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{DEFAULT_CRS}>' @@ -702,9 +700,8 @@ def test_get_collection_item_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') req = mock_request({'f': 'json', 'crs': storage_crs}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -717,9 +714,8 @@ def test_get_collection_item_postgresql_crs(pg_api_): # With CRS query parameter resulting in coordinates transformation req = mock_request({'f': 'json', 'crs': crs_32735}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_32735}>' @@ -742,8 +738,8 @@ def test_get_collection_items_postgresql_automap_naming_conflicts(pg_api_): classes and relationships from database schema. """ req = mock_request() - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'dummy_naming_conflicts') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'dummy_naming_conflicts') assert code == HTTPStatus.OK features = json.loads(response).get('features') From d63b10ea7c7f2360ecb85a904263cd5d3d24d8ba Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 25 Mar 2024 21:03:13 -0400 Subject: [PATCH 57/71] add missing descriptions to OpenAPI admin responses --- pygeoapi/openapi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index a701c7fb9..88a3bd501 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -642,6 +642,7 @@ def get_admin(): ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict @@ -703,6 +704,7 @@ def get_admin(): ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -745,6 +747,7 @@ def get_admin(): ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa From d79f3671af2336bf27227d87023ba054d3bc441b Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 25 Mar 2024 21:59:14 -0400 Subject: [PATCH 58/71] update tests --- tests/test_postgresql_provider.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index bd56aed0c..b027d0849 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -61,7 +61,7 @@ from pygeoapi.util import (yaml_load, geojson_to_geom, get_transform_from_crs, get_crs_from_uri) -from .util import get_test_file_path, mock_request +from .util import get_test_file_path, mock_api_request PASSWORD = os.environ.get('POSTGRESQL_PASSWORD', 'postgres') DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' @@ -431,7 +431,7 @@ def test_get_collection_items_postgresql_cql(pg_api_): expected_ids = [80835474, 80835483] # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-text', 'filter': cql_query }) @@ -445,7 +445,7 @@ def test_get_collection_items_postgresql_cql(pg_api_): assert ids == expected_ids # Act, no filter-lang - req = mock_request({ + req = mock_api_request({ 'filter': cql_query }) rsp_headers, code, response = get_collection_items( @@ -469,7 +469,7 @@ def test_get_collection_items_postgresql_cql_invalid_filter_language(pg_api_): cql_query = 'osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL' # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json', # Only cql-text is valid for GET 'filter': cql_query }) @@ -496,7 +496,7 @@ def test_get_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): Test for bad cql """ # Act - req = mock_request({ + req = mock_api_request({ 'filter': bad_cql }) rsp_headers, code, response = get_collection_items( @@ -526,7 +526,7 @@ def test_post_collection_items_postgresql_cql(pg_api_): expected_ids = [80835474, 80835483] # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json' }, data=cql, **headers) rsp_headers, code, response = post_collection_items( @@ -552,7 +552,7 @@ def test_post_collection_items_postgresql_cql_invalid_filter_language(pg_api_): headers = {'CONTENT_TYPE': 'application/query-cql-json'} # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-text' # Only cql-json is valid for POST }, data=cql, **headers) rsp_headers, code, response = post_collection_items( @@ -582,7 +582,7 @@ def test_post_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): headers = {'CONTENT_TYPE': 'application/query-cql-json'} # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json' }, data=bad_cql, **headers) rsp_headers, code, response = post_collection_items( @@ -603,7 +603,7 @@ def test_get_collection_items_postgresql_crs(pg_api_): crs_32735 = 'http://www.opengis.net/def/crs/EPSG/0/32735' # Without CRS query parameter -> no coordinates transformation - req = mock_request({'bbox': '29.0,-2.85,29.05,-2.8'}) + req = mock_api_request({'bbox': '29.0,-2.85,29.05,-2.8'}) rsp_headers, code, response = get_collection_items( pg_api_, req, 'hot_osm_waterways') @@ -614,7 +614,8 @@ def test_get_collection_items_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') - req = mock_request({'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) + req = mock_api_request( + {'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) rsp_headers, code, response = get_collection_items( pg_api_, req, 'hot_osm_waterways') @@ -624,7 +625,7 @@ def test_get_collection_items_postgresql_crs(pg_api_): features_storage_crs = json.loads(response) # With CRS query parameter resulting in coordinates transformation - req = mock_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) + req = mock_api_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) rsp_headers, code, response = get_collection_items( pg_api_, req, 'hot_osm_waterways') @@ -687,7 +688,7 @@ def test_get_collection_item_postgresql_crs(pg_api_): ] for fid in fid_list: # Without CRS query parameter -> no coordinates transformation - req = mock_request({'f': 'json'}) + req = mock_api_request({'f': 'json'}) rsp_headers, code, response = get_collection_item( pg_api_, req, 'hot_osm_waterways', fid) @@ -699,7 +700,7 @@ def test_get_collection_item_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') - req = mock_request({'f': 'json', 'crs': storage_crs}) + req = mock_api_request({'f': 'json', 'crs': storage_crs}) rsp_headers, code, response = get_collection_item( pg_api_, req, 'hot_osm_waterways', fid) @@ -713,7 +714,7 @@ def test_get_collection_item_postgresql_crs(pg_api_): assert feat_orig['geometry'] == feat_storage_crs['geometry'] # With CRS query parameter resulting in coordinates transformation - req = mock_request({'f': 'json', 'crs': crs_32735}) + req = mock_api_request({'f': 'json', 'crs': crs_32735}) rsp_headers, code, response = get_collection_item( pg_api_, req, 'hot_osm_waterways', fid) @@ -737,7 +738,7 @@ def test_get_collection_items_postgresql_automap_naming_conflicts(pg_api_): Test that PostgreSQLProvider can handle naming conflicts when automapping classes and relationships from database schema. """ - req = mock_request() + req = mock_api_request() rsp_headers, code, response = get_collection_items( pg_api_, req, 'dummy_naming_conflicts') From af38787eea7da4be25376b078f8044763513cb5e Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 26 Mar 2024 06:14:13 -0400 Subject: [PATCH 59/71] fix tests autodiscovery --- tests/api/__init__.py | 822 ---------------------------------------- tests/api/test_api.py | 853 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 853 insertions(+), 822 deletions(-) create mode 100644 tests/api/test_api.py diff --git a/tests/api/__init__.py b/tests/api/__init__.py index b2c013dc7..ce0b0bef8 100644 --- a/tests/api/__init__.py +++ b/tests/api/__init__.py @@ -3,7 +3,6 @@ # Authors: Tom Kralidis # John A Stevenson # Colin Blackburn -# Bernhard Mallinger # # Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 John A Stevenson and Colin Blackburn @@ -30,824 +29,3 @@ # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= - -import json -import logging -import gzip -from http import HTTPStatus - -from pyld import jsonld -import pytest - -from pygeoapi.api import ( - API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP, - __version__, validate_bbox, validate_datetime, validate_subset -) -from pygeoapi.util import yaml_load, get_api_rules, get_base_url - -from tests.util import (get_test_file_path, mock_flask, mock_starlette, - mock_request) - -LOGGER = logging.getLogger(__name__) - - -@pytest.fixture() -def config(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_with_rules() -> dict: - """ Returns a pygeoapi configuration with default API rules. """ - with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_enclosure() -> dict: - """ Returns a pygeoapi configuration with enclosure links. """ - with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_hidden_resources(): - filename = 'pygeoapi-test-config-hidden-resources.yml' - with open(get_test_file_path(filename)) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def enclosure_api(config_enclosure, openapi): - """ Returns an API instance with a collection with enclosure links. """ - return API(config_enclosure, openapi) - - -@pytest.fixture() -def rules_api(config_with_rules, openapi): - """ Returns an API instance with URL prefix and strict slashes policy. - The API version is extracted from the current version here. - """ - return API(config_with_rules, openapi) - - -@pytest.fixture() -def api_hidden_resources(config_hidden_resources, openapi): - return API(config_hidden_resources, openapi) - - -def test_apirequest(api_): - # Test without (valid) locales - with pytest.raises(ValueError): - req = mock_request() - APIRequest(req, []) - APIRequest(req, None) - APIRequest(req, ['zz']) - - # Test all supported formats from query args - for f, mt in FORMAT_TYPES.items(): - req = mock_request({'f': f}) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test all supported formats from Accept header - for f, mt in FORMAT_TYPES.items(): - req = mock_request(HTTP_ACCEPT=mt) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test nonsense format - req = mock_request({'f': 'foo'}) - apireq = APIRequest(req, api_.locales) - assert not apireq.is_valid() - assert apireq.format == 'foo' - assert apireq.is_valid(('foo',)) - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - - # Test without format - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format is None - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - assert apireq.get_linkrel(F_JSON) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Test complex format string - hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - assert apireq.get_linkrel(F_HTML) == 'self' - assert apireq.get_linkrel(F_JSON) == 'alternate' - - # Test accept header with multiple valid formats - hh = 'plain/text,application/ld+json,application/json;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_JSONLD - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSONLD] - assert apireq.get_linkrel(F_JSONLD) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Overrule HTTP content negotiation - req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - - # Test data - for d in (None, '', 'test', {'key': 'value'}): - req = mock_request(data=d) - apireq = APIRequest.with_data(req, api_.locales) - if not d: - assert apireq.data == b'' - elif isinstance(d, dict): - assert d == json.loads(apireq.data) - else: - assert apireq.data == d.encode() - - # Test multilingual - test_lang = { - 'nl': ('en', 'en-US'), # unsupported lang should return default - 'en-US': ('en', 'en-US'), - 'de_CH': ('en', 'en-US'), - 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), - 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), - } - sup_lang = ('en-US', 'fr_CA') - for lang_in, (lang_out, cl_out) in test_lang.items(): - # Using l query parameter - req = mock_request({'lang': lang_in}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Using Accept-Language header - req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Test language override - req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'fr' - assert apireq.locale.language == 'fr' - assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' - - # Test locale territory - req = mock_request({'lang': 'en-GB'}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'en-GB' - assert apireq.locale.language == 'en' - assert apireq.locale.territory == 'US' - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query parameter - # (should return default language from YAML config) - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.raw_locale is None - assert apireq.locale.language == api_.default_locale.language - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query param - # (should return first in custom list of languages) - sup_lang = ('de', 'fr', 'en') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale is None - assert apireq.locale.language == 'de' - assert apireq.get_response_headers()['Content-Language'] == 'de' - - -def test_apirules_active(config_with_rules, rules_api): - assert rules_api.config == config_with_rules - rules = get_api_rules(config_with_rules) - base_url = get_base_url(config_with_rules) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: - # Test happy path - response = flask_client.get(f'{flask_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = flask_client.get(f'{flask_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = flask_client.get(f'{flask_prefix}/') - assert response.status_code == 200 - response = flask_client.get(flask_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = flask_client.get(flask_prefix, follow_redirects=True) - assert response.status_code == 200 - assert response.is_json - links = response.json['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa - # Test happy path - response = starlette_client.get(f'{starlette_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = starlette_client.get(f'{starlette_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = starlette_client.get(f'{starlette_prefix}/') - assert response.status_code == 200 - response = starlette_client.get(starlette_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa - assert response.status_code == 200 - links = response.json()['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - -def test_apirules_inactive(config, api_): - assert api_.config == config - rules = get_api_rules(config) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - assert flask_prefix == '' - with mock_flask('pygeoapi-test-config.yml') as flask_client: - response = flask_client.get('') - assert response.status_code == 200 - response = flask_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = flask_client.get('/') - assert response.status_code == 200 - response = flask_client.get('/conformance/') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - assert starlette_prefix == '' - with mock_starlette('pygeoapi-test-config.yml') as starlette_client: - response = starlette_client.get('') - assert response.status_code == 200 - response = starlette_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert str(response.url) == f"{starlette_client.base_url}/conformance" - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = starlette_client.get('/') - assert response.status_code == 200 - response = starlette_client.get('/conformance/', follow_redirects=True) - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - -def test_api(config, api_, openapi): - assert api_.config == config - assert isinstance(api_.config, dict) - - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - root = json.loads(response) - assert isinstance(root, dict) - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request(HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'Swagger UI' in response - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'ReDoc' in response - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - assert api_.get_collections_url() == 'http://localhost:5000/collections' - - -def test_api_exception(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - # When a language is set, the exception should still be English - req = mock_request({'f': 'foo', 'lang': 'fr'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - -def test_gzip(config, api_, openapi): - # Requests for each response type and gzip encoding - req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', - HTTP_ACCEPT_ENCODING=F_GZIP) - - # Responses from server config without gzip compression - rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Add gzip to server and use utf-16 encoding - config['server']['gzip'] = True - enc_16 = 'utf-16' - config['server']['encoding'] = enc_16 - api_ = API(config, openapi) - - # Responses from server with gzip compression - rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) - rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) - rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) - rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) - - # Validate compressed json response - assert rsp_json_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' - assert rsp_json_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) - assert isinstance(parsed_gzip_json, str) - parsed_gzip_json = json.loads(parsed_gzip_json) - assert isinstance(parsed_gzip_json, dict) - assert parsed_gzip_json == json.loads(rsp_json) - - # Validate compressed jsonld response - assert rsp_jsonld_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' - assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - assert isinstance(parsed_gzip_jsonld, str) - parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) - assert isinstance(parsed_gzip_jsonld, dict) - assert parsed_gzip_jsonld == json.loads(rsp_jsonld) - - # Validate compressed html response - assert rsp_html_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' - assert rsp_html_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) - assert isinstance(parsed_gzip_html, str) - assert parsed_gzip_html == rsp_html - - # Validate compressed gzip response - assert rsp_gzip_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' - assert rsp_gzip_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) - assert isinstance(parsed_gzip_gzip, str) - parsed_gzip_gzip = json.loads(parsed_gzip_gzip) - assert isinstance(parsed_gzip_gzip, dict) - - # Requests without content encoding header - req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) - req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) - req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) - - # Responses without content encoding - _, _, rsp_json_ = api_.landing_page(req_json) - _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) - _, _, rsp_html_ = api_.landing_page(req_html) - - # Confirm each request is the same when decompressed - assert rsp_json_ == rsp_json == \ - gzip.decompress(rsp_gzip_json).decode(enc_16) - - assert rsp_jsonld_ == rsp_jsonld == \ - gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - - assert rsp_html_ == rsp_html == \ - gzip.decompress(rsp_gzip_html).decode(enc_16) - - -def test_root(config, api_): - req = mock_request() - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/json' == \ - FORMAT_TYPES[F_JSON] - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - assert rsp_headers['Content-Language'] == 'en-US' - - assert isinstance(root, dict) - assert 'links' in root - assert root['links'][0]['rel'] == 'self' - assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] - assert root['links'][0]['href'].endswith('?f=json') - assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' - for link in root['links']) - assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' - for link in root['links']) - assert len(root['links']) == 11 - assert 'title' in root - assert root['title'] == 'pygeoapi default instance' - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_root_structured_data(config, api_): - req = mock_request({"f": "jsonld"}) - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/ld+json' == \ - FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - - assert isinstance(root, dict) - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - assert '@context' in root - assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' - expanded = jsonld.expand(root)[0] - assert '@type' in expanded - assert 'http://schema.org/DataCatalog' in expanded['@type'] - assert 'http://schema.org/description' in expanded - assert root['description'] == expanded['http://schema.org/description'][0][ - '@value'] - assert 'http://schema.org/keywords' in expanded - assert len(expanded['http://schema.org/keywords']) == 3 - assert '@value' in expanded['http://schema.org/keywords'][0].keys() - assert 'http://schema.org/provider' in expanded - assert expanded['http://schema.org/provider'][0]['@type'][ - 0] == 'http://schema.org/Organization' - assert expanded['http://schema.org/name'][0]['@value'] == root['name'] - - -def test_conformance(config, api_): - req = mock_request() - rsp_headers, code, response = api_.conformance(req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'conformsTo' in root - assert len(root['conformsTo']) == 37 - assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ - in root['conformsTo'] - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.conformance(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.conformance(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections(config, api_): - req = mock_request({"f": "foo"}) - rsp_headers, code, response = api_.describe_collections(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({"f": "html"}) - rsp_headers, code, response = api_.describe_collections(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req) - collections = json.loads(response) - - assert len(collections) == 2 - assert len(collections['collections']) == 9 - assert len(collections['links']) == 3 - - rsp_headers, code, response = api_.describe_collections(req, 'foo') - collection = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'en-US' - assert collection['id'] == 'obs' - assert collection['title'] == 'Observations' - assert collection['description'] == 'My cool observations' - assert len(collection['links']) == 14 - assert collection['extent'] == { - 'spatial': { - 'bbox': [[-180, -90, 180, 90]], - 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - }, - 'temporal': { - 'interval': [ - ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] - ], - 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' - } - } - - # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults - assert collection['crs'] is not None - crs_set = [ - 'http://www.opengis.net/def/crs/EPSG/0/28992', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/EPSG/0/4326', - ] - for crs in crs_set: - assert crs in collection['crs'] - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert 'storageCrsCoordinateEpoch' not in collection - - # French language request - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'fr-CA' - assert collection['title'] == 'Observations' - assert collection['description'] == 'Mes belles observations' - - # Check HTML request in an unsupported language - req = mock_request({'f': 'html', 'lang': 'de'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - # hiearchical collections - req = mock_request() - rsp_headers, code, response = api_.describe_collections( - req, 'naturalearth/lakes') - - collection = json.loads(response) - assert collection['id'] == 'naturalearth/lakes' - - # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured - assert collection['crs'] is not None - default_crs_list = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', - ] - contains_default = False - for crs in default_crs_list: - if crs in default_crs_list: - contains_default = True - assert contains_default - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert collection['storageCrsCoordinateEpoch'] == 2017.23 - - -def test_describe_collections_hidden_resources( - config_hidden_resources, api_hidden_resources): - req = mock_request({}) - rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa - assert code == HTTPStatus.OK - - assert len(config_hidden_resources['resources']) == 3 - - collections = json.loads(response) - assert len(collections['collections']) == 1 - - -def test_describe_collections_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert '@context' in collection - expanded = jsonld.expand(collection)[0] - # Metadata is about a schema:DataCollection that contains a schema:Dataset - assert not expanded['@id'].endswith('obs') - assert 'http://schema.org/dataset' in expanded - assert len(expanded['http://schema.org/dataset']) == 1 - dataset = expanded['http://schema.org/dataset'][0] - assert dataset['@type'][0] == 'http://schema.org/Dataset' - assert len(dataset['http://schema.org/distribution']) == 14 - assert all(dist['@type'][0] == 'http://schema.org/DataDownload' - for dist in dataset['http://schema.org/distribution']) - - assert 'http://schema.org/Organization' in expanded[ - 'http://schema.org/provider'][0]['@type'] - - assert 'http://schema.org/Place' in dataset[ - 'http://schema.org/spatial'][0]['@type'] - assert 'http://schema.org/GeoShape' in dataset[ - 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] - assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ - 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' - - assert 'http://schema.org/temporalCoverage' in dataset - assert dataset['http://schema.org/temporalCoverage'][0][ - '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections_enclosures(config_enclosure, enclosure_api): - original_enclosures = { - lnk['title']: lnk - for lnk in config_enclosure['resources']['objects']['links'] - if lnk['rel'] == 'enclosure' - } - - req = mock_request() - _, _, response = enclosure_api.describe_collections(req, 'objects') - features = json.loads(response) - modified_enclosures = { - lnk['title']: lnk for lnk in features['links'] - if lnk['rel'] == 'enclosure' - } - - # If type and length is set, do not verify/update link - assert original_enclosures['download link 1'] == \ - modified_enclosures['download link 1'] - # If length is missing, modify link type and length - assert original_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 2']['type'] - assert modified_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 3']['type'] - assert 'length' not in original_enclosures['download link 2'] - assert modified_enclosures['download link 2']['length'] > 0 - assert modified_enclosures['download link 2']['length'] == \ - modified_enclosures['download link 3']['length'] - assert original_enclosures['download link 3']['type'] != \ - modified_enclosures['download link 3']['type'] - - -def test_validate_bbox(): - assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] - assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] - assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] - assert (validate_bbox('-142.1,42.12,-52.22,84.4') == - [-142.1, 42.12, -52.22, 84.4]) - assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == - [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) - - assert (validate_bbox('177.0,65.0,-177.0,70.0') == - [177.0, 65.0, -177.0, 70.0]) - - with pytest.raises(ValueError): - validate_bbox('1,2,4') - - with pytest.raises(ValueError): - validate_bbox('1,2,4,5,6') - - with pytest.raises(ValueError): - validate_bbox('3,4,1,2') - - with pytest.raises(ValueError): - validate_bbox('1,2,6,4,5,3') - - -def test_validate_datetime(): - config = yaml_load(''' - temporal: - begin: 2000-10-30T18:24:39Z - end: 2007-10-30T08:57:29Z - ''') - - # test time instant - assert validate_datetime(config, '2004') == '2004' - assert validate_datetime(config, '2004-10') == '2004-10' - assert validate_datetime(config, '2001-10-30') == '2001-10-30' - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009-10-30') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-09-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-10-30T17:24:39Z') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-10-30T08:58:29Z') - - # test time envelope - assert validate_datetime(config, '2004/2005') == '2004/2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - assert validate_datetime(config, '2004/..') == '2004/..' - assert validate_datetime(config, '../2005') == '../2005' - assert validate_datetime(config, '2004/') == '2004/..' - assert validate_datetime(config, '/2005') == '../2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-11-01/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../2000-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../1999') - - -@pytest.mark.parametrize("value, expected", [ - ('time(2000-11-11)', {'time': ['2000-11-11']}), - ('time("2000-11-11")', {'time': ['2000-11-11']}), - ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), - ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa - ('lat(40)', {'lat': [40]}), - ('lat(0:40)', {'lat': [0, 40]}), - ('foo("bar")', {'foo': ['bar']}), - ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) -]) -def test_validate_subset(value, expected): - assert validate_subset(value) == expected - - with pytest.raises(ValueError): - validate_subset('foo("bar)') - - -def test_get_exception(config, api_): - d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') - assert d[0] == {} - assert d[1] == 500 - content = json.loads(d[2]) - assert content['code'] == 'NoApplicableCode' - assert content['description'] == 'oops' - - d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') diff --git a/tests/api/test_api.py b/tests/api/test_api.py new file mode 100644 index 000000000..b2c013dc7 --- /dev/null +++ b/tests/api/test_api.py @@ -0,0 +1,853 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +import logging +import gzip +from http import HTTPStatus + +from pyld import jsonld +import pytest + +from pygeoapi.api import ( + API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP, + __version__, validate_bbox, validate_datetime, validate_subset +) +from pygeoapi.util import yaml_load, get_api_rules, get_base_url + +from tests.util import (get_test_file_path, mock_flask, mock_starlette, + mock_request) + +LOGGER = logging.getLogger(__name__) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_with_rules() -> dict: + """ Returns a pygeoapi configuration with default API rules. """ + with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_enclosure() -> dict: + """ Returns a pygeoapi configuration with enclosure links. """ + with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_hidden_resources(): + filename = 'pygeoapi-test-config-hidden-resources.yml' + with open(get_test_file_path(filename)) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def enclosure_api(config_enclosure, openapi): + """ Returns an API instance with a collection with enclosure links. """ + return API(config_enclosure, openapi) + + +@pytest.fixture() +def rules_api(config_with_rules, openapi): + """ Returns an API instance with URL prefix and strict slashes policy. + The API version is extracted from the current version here. + """ + return API(config_with_rules, openapi) + + +@pytest.fixture() +def api_hidden_resources(config_hidden_resources, openapi): + return API(config_hidden_resources, openapi) + + +def test_apirequest(api_): + # Test without (valid) locales + with pytest.raises(ValueError): + req = mock_request() + APIRequest(req, []) + APIRequest(req, None) + APIRequest(req, ['zz']) + + # Test all supported formats from query args + for f, mt in FORMAT_TYPES.items(): + req = mock_request({'f': f}) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test all supported formats from Accept header + for f, mt in FORMAT_TYPES.items(): + req = mock_request(HTTP_ACCEPT=mt) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test nonsense format + req = mock_request({'f': 'foo'}) + apireq = APIRequest(req, api_.locales) + assert not apireq.is_valid() + assert apireq.format == 'foo' + assert apireq.is_valid(('foo',)) + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + + # Test without format + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format is None + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + assert apireq.get_linkrel(F_JSON) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Test complex format string + hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + assert apireq.get_linkrel(F_HTML) == 'self' + assert apireq.get_linkrel(F_JSON) == 'alternate' + + # Test accept header with multiple valid formats + hh = 'plain/text,application/ld+json,application/json;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_JSONLD + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSONLD] + assert apireq.get_linkrel(F_JSONLD) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Overrule HTTP content negotiation + req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + + # Test data + for d in (None, '', 'test', {'key': 'value'}): + req = mock_request(data=d) + apireq = APIRequest.with_data(req, api_.locales) + if not d: + assert apireq.data == b'' + elif isinstance(d, dict): + assert d == json.loads(apireq.data) + else: + assert apireq.data == d.encode() + + # Test multilingual + test_lang = { + 'nl': ('en', 'en-US'), # unsupported lang should return default + 'en-US': ('en', 'en-US'), + 'de_CH': ('en', 'en-US'), + 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), + 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), + } + sup_lang = ('en-US', 'fr_CA') + for lang_in, (lang_out, cl_out) in test_lang.items(): + # Using l query parameter + req = mock_request({'lang': lang_in}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Using Accept-Language header + req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Test language override + req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'fr' + assert apireq.locale.language == 'fr' + assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' + + # Test locale territory + req = mock_request({'lang': 'en-GB'}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'en-GB' + assert apireq.locale.language == 'en' + assert apireq.locale.territory == 'US' + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query parameter + # (should return default language from YAML config) + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.raw_locale is None + assert apireq.locale.language == api_.default_locale.language + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query param + # (should return first in custom list of languages) + sup_lang = ('de', 'fr', 'en') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale is None + assert apireq.locale.language == 'de' + assert apireq.get_response_headers()['Content-Language'] == 'de' + + +def test_apirules_active(config_with_rules, rules_api): + assert rules_api.config == config_with_rules + rules = get_api_rules(config_with_rules) + base_url = get_base_url(config_with_rules) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: + # Test happy path + response = flask_client.get(f'{flask_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = flask_client.get(f'{flask_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = flask_client.get(f'{flask_prefix}/') + assert response.status_code == 200 + response = flask_client.get(flask_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = flask_client.get(flask_prefix, follow_redirects=True) + assert response.status_code == 200 + assert response.is_json + links = response.json['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa + # Test happy path + response = starlette_client.get(f'{starlette_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = starlette_client.get(f'{starlette_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = starlette_client.get(f'{starlette_prefix}/') + assert response.status_code == 200 + response = starlette_client.get(starlette_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa + assert response.status_code == 200 + links = response.json()['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + +def test_apirules_inactive(config, api_): + assert api_.config == config + rules = get_api_rules(config) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + assert flask_prefix == '' + with mock_flask('pygeoapi-test-config.yml') as flask_client: + response = flask_client.get('') + assert response.status_code == 200 + response = flask_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = flask_client.get('/') + assert response.status_code == 200 + response = flask_client.get('/conformance/') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + assert starlette_prefix == '' + with mock_starlette('pygeoapi-test-config.yml') as starlette_client: + response = starlette_client.get('') + assert response.status_code == 200 + response = starlette_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert str(response.url) == f"{starlette_client.base_url}/conformance" + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = starlette_client.get('/') + assert response.status_code == 200 + response = starlette_client.get('/conformance/', follow_redirects=True) + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + +def test_api(config, api_, openapi): + assert api_.config == config + assert isinstance(api_.config, dict) + + req = mock_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + root = json.loads(response) + assert isinstance(root, dict) + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request(HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'Swagger UI' in response + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'ReDoc' in response + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + assert api_.get_collections_url() == 'http://localhost:5000/collections' + + +def test_api_exception(config, api_): + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + # When a language is set, the exception should still be English + req = mock_request({'f': 'foo', 'lang': 'fr'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + +def test_gzip(config, api_, openapi): + # Requests for each response type and gzip encoding + req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', + HTTP_ACCEPT_ENCODING=F_GZIP) + + # Responses from server config without gzip compression + rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Add gzip to server and use utf-16 encoding + config['server']['gzip'] = True + enc_16 = 'utf-16' + config['server']['encoding'] = enc_16 + api_ = API(config, openapi) + + # Responses from server with gzip compression + rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) + rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) + rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) + rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) + + # Validate compressed json response + assert rsp_json_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' + assert rsp_json_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) + assert isinstance(parsed_gzip_json, str) + parsed_gzip_json = json.loads(parsed_gzip_json) + assert isinstance(parsed_gzip_json, dict) + assert parsed_gzip_json == json.loads(rsp_json) + + # Validate compressed jsonld response + assert rsp_jsonld_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' + assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + assert isinstance(parsed_gzip_jsonld, str) + parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) + assert isinstance(parsed_gzip_jsonld, dict) + assert parsed_gzip_jsonld == json.loads(rsp_jsonld) + + # Validate compressed html response + assert rsp_html_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' + assert rsp_html_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) + assert isinstance(parsed_gzip_html, str) + assert parsed_gzip_html == rsp_html + + # Validate compressed gzip response + assert rsp_gzip_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' + assert rsp_gzip_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) + assert isinstance(parsed_gzip_gzip, str) + parsed_gzip_gzip = json.loads(parsed_gzip_gzip) + assert isinstance(parsed_gzip_gzip, dict) + + # Requests without content encoding header + req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) + req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) + req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) + + # Responses without content encoding + _, _, rsp_json_ = api_.landing_page(req_json) + _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) + _, _, rsp_html_ = api_.landing_page(req_html) + + # Confirm each request is the same when decompressed + assert rsp_json_ == rsp_json == \ + gzip.decompress(rsp_gzip_json).decode(enc_16) + + assert rsp_jsonld_ == rsp_jsonld == \ + gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + + assert rsp_html_ == rsp_html == \ + gzip.decompress(rsp_gzip_html).decode(enc_16) + + +def test_root(config, api_): + req = mock_request() + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/json' == \ + FORMAT_TYPES[F_JSON] + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + assert rsp_headers['Content-Language'] == 'en-US' + + assert isinstance(root, dict) + assert 'links' in root + assert root['links'][0]['rel'] == 'self' + assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] + assert root['links'][0]['href'].endswith('?f=json') + assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' + for link in root['links']) + assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' + for link in root['links']) + assert len(root['links']) == 11 + assert 'title' in root + assert root['title'] == 'pygeoapi default instance' + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_root_structured_data(config, api_): + req = mock_request({"f": "jsonld"}) + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/ld+json' == \ + FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + + assert isinstance(root, dict) + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + assert '@context' in root + assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' + expanded = jsonld.expand(root)[0] + assert '@type' in expanded + assert 'http://schema.org/DataCatalog' in expanded['@type'] + assert 'http://schema.org/description' in expanded + assert root['description'] == expanded['http://schema.org/description'][0][ + '@value'] + assert 'http://schema.org/keywords' in expanded + assert len(expanded['http://schema.org/keywords']) == 3 + assert '@value' in expanded['http://schema.org/keywords'][0].keys() + assert 'http://schema.org/provider' in expanded + assert expanded['http://schema.org/provider'][0]['@type'][ + 0] == 'http://schema.org/Organization' + assert expanded['http://schema.org/name'][0]['@value'] == root['name'] + + +def test_conformance(config, api_): + req = mock_request() + rsp_headers, code, response = api_.conformance(req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'conformsTo' in root + assert len(root['conformsTo']) == 37 + assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ + in root['conformsTo'] + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.conformance(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.conformance(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections(config, api_): + req = mock_request({"f": "foo"}) + rsp_headers, code, response = api_.describe_collections(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({"f": "html"}) + rsp_headers, code, response = api_.describe_collections(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request() + rsp_headers, code, response = api_.describe_collections(req) + collections = json.loads(response) + + assert len(collections) == 2 + assert len(collections['collections']) == 9 + assert len(collections['links']) == 3 + + rsp_headers, code, response = api_.describe_collections(req, 'foo') + collection = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'en-US' + assert collection['id'] == 'obs' + assert collection['title'] == 'Observations' + assert collection['description'] == 'My cool observations' + assert len(collection['links']) == 14 + assert collection['extent'] == { + 'spatial': { + 'bbox': [[-180, -90, 180, 90]], + 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + }, + 'temporal': { + 'interval': [ + ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] + ], + 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' + } + } + + # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults + assert collection['crs'] is not None + crs_set = [ + 'http://www.opengis.net/def/crs/EPSG/0/28992', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/EPSG/0/4326', + ] + for crs in crs_set: + assert crs in collection['crs'] + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert 'storageCrsCoordinateEpoch' not in collection + + # French language request + req = mock_request({'lang': 'fr'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'fr-CA' + assert collection['title'] == 'Observations' + assert collection['description'] == 'Mes belles observations' + + # Check HTML request in an unsupported language + req = mock_request({'f': 'html', 'lang': 'de'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + # hiearchical collections + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'naturalearth/lakes') + + collection = json.loads(response) + assert collection['id'] == 'naturalearth/lakes' + + # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured + assert collection['crs'] is not None + default_crs_list = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', + ] + contains_default = False + for crs in default_crs_list: + if crs in default_crs_list: + contains_default = True + assert contains_default + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert collection['storageCrsCoordinateEpoch'] == 2017.23 + + +def test_describe_collections_hidden_resources( + config_hidden_resources, api_hidden_resources): + req = mock_request({}) + rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa + assert code == HTTPStatus.OK + + assert len(config_hidden_resources['resources']) == 3 + + collections = json.loads(response) + assert len(collections['collections']) == 1 + + +def test_describe_collections_json_ld(config, api_): + req = mock_request({'f': 'jsonld'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert '@context' in collection + expanded = jsonld.expand(collection)[0] + # Metadata is about a schema:DataCollection that contains a schema:Dataset + assert not expanded['@id'].endswith('obs') + assert 'http://schema.org/dataset' in expanded + assert len(expanded['http://schema.org/dataset']) == 1 + dataset = expanded['http://schema.org/dataset'][0] + assert dataset['@type'][0] == 'http://schema.org/Dataset' + assert len(dataset['http://schema.org/distribution']) == 14 + assert all(dist['@type'][0] == 'http://schema.org/DataDownload' + for dist in dataset['http://schema.org/distribution']) + + assert 'http://schema.org/Organization' in expanded[ + 'http://schema.org/provider'][0]['@type'] + + assert 'http://schema.org/Place' in dataset[ + 'http://schema.org/spatial'][0]['@type'] + assert 'http://schema.org/GeoShape' in dataset[ + 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] + assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ + 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' + + assert 'http://schema.org/temporalCoverage' in dataset + assert dataset['http://schema.org/temporalCoverage'][0][ + '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections_enclosures(config_enclosure, enclosure_api): + original_enclosures = { + lnk['title']: lnk + for lnk in config_enclosure['resources']['objects']['links'] + if lnk['rel'] == 'enclosure' + } + + req = mock_request() + _, _, response = enclosure_api.describe_collections(req, 'objects') + features = json.loads(response) + modified_enclosures = { + lnk['title']: lnk for lnk in features['links'] + if lnk['rel'] == 'enclosure' + } + + # If type and length is set, do not verify/update link + assert original_enclosures['download link 1'] == \ + modified_enclosures['download link 1'] + # If length is missing, modify link type and length + assert original_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 2']['type'] + assert modified_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 3']['type'] + assert 'length' not in original_enclosures['download link 2'] + assert modified_enclosures['download link 2']['length'] > 0 + assert modified_enclosures['download link 2']['length'] == \ + modified_enclosures['download link 3']['length'] + assert original_enclosures['download link 3']['type'] != \ + modified_enclosures['download link 3']['type'] + + +def test_validate_bbox(): + assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] + assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] + assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] + assert (validate_bbox('-142.1,42.12,-52.22,84.4') == + [-142.1, 42.12, -52.22, 84.4]) + assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == + [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) + + assert (validate_bbox('177.0,65.0,-177.0,70.0') == + [177.0, 65.0, -177.0, 70.0]) + + with pytest.raises(ValueError): + validate_bbox('1,2,4') + + with pytest.raises(ValueError): + validate_bbox('1,2,4,5,6') + + with pytest.raises(ValueError): + validate_bbox('3,4,1,2') + + with pytest.raises(ValueError): + validate_bbox('1,2,6,4,5,3') + + +def test_validate_datetime(): + config = yaml_load(''' + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + ''') + + # test time instant + assert validate_datetime(config, '2004') == '2004' + assert validate_datetime(config, '2004-10') == '2004-10' + assert validate_datetime(config, '2001-10-30') == '2001-10-30' + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009-10-30') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-09-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-10-30T17:24:39Z') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-10-30T08:58:29Z') + + # test time envelope + assert validate_datetime(config, '2004/2005') == '2004/2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + assert validate_datetime(config, '2004/..') == '2004/..' + assert validate_datetime(config, '../2005') == '../2005' + assert validate_datetime(config, '2004/') == '2004/..' + assert validate_datetime(config, '/2005') == '../2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-11-01/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../2000-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../1999') + + +@pytest.mark.parametrize("value, expected", [ + ('time(2000-11-11)', {'time': ['2000-11-11']}), + ('time("2000-11-11")', {'time': ['2000-11-11']}), + ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), + ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa + ('lat(40)', {'lat': [40]}), + ('lat(0:40)', {'lat': [0, 40]}), + ('foo("bar")', {'foo': ['bar']}), + ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) +]) +def test_validate_subset(value, expected): + assert validate_subset(value) == expected + + with pytest.raises(ValueError): + validate_subset('foo("bar)') + + +def test_get_exception(config, api_): + d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') + assert d[0] == {} + assert d[1] == 500 + content = json.loads(d[2]) + assert content['code'] == 'NoApplicableCode' + assert content['description'] == 'oops' + + d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') From 38fdd57960bbc5799540c686b7674050434e5e12 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 27 Mar 2024 14:45:33 -0400 Subject: [PATCH 60/71] remove unused logging in tests --- tests/api/test_api.py | 3 --- tests/api/test_coverages.py | 3 --- tests/api/test_itemtypes.py | 3 --- 3 files changed, 9 deletions(-) diff --git a/tests/api/test_api.py b/tests/api/test_api.py index b2c013dc7..d43a4aae9 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -32,7 +32,6 @@ # ================================================================= import json -import logging import gzip from http import HTTPStatus @@ -48,8 +47,6 @@ from tests.util import (get_test_file_path, mock_flask, mock_starlette, mock_request) -LOGGER = logging.getLogger(__name__) - @pytest.fixture() def config(): diff --git a/tests/api/test_coverages.py b/tests/api/test_coverages.py index dbbed9435..50e5166d6 100644 --- a/tests/api/test_coverages.py +++ b/tests/api/test_coverages.py @@ -31,7 +31,6 @@ # ================================================================= import json -import logging from http import HTTPStatus import pytest @@ -43,8 +42,6 @@ from tests.util import get_test_file_path, mock_request, mock_api_request -LOGGER = logging.getLogger(__name__) - @pytest.fixture() def config(): diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py index 5bb8b5de6..47adb2f32 100644 --- a/tests/api/test_itemtypes.py +++ b/tests/api/test_itemtypes.py @@ -33,7 +33,6 @@ import copy import gzip import json -import logging from http import HTTPStatus from pyld import jsonld @@ -50,8 +49,6 @@ from tests.util import get_test_file_path, mock_api_request -LOGGER = logging.getLogger(__name__) - @pytest.fixture() def config(): From cb7df7c46d5ec729d319843d8c812113b3425512 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 2 Apr 2024 19:05:33 -0400 Subject: [PATCH 61/71] address PR comments --- pygeoapi/api/__init__.py | 84 +++++++++++++++++- pygeoapi/api/coverages.py | 13 ++- pygeoapi/api/environmental_data_retrieval.py | 13 ++- pygeoapi/api/itemtypes.py | 93 +++----------------- pygeoapi/api/maps.py | 13 ++- pygeoapi/api/processes.py | 29 +++--- pygeoapi/api/stac.py | 20 +++-- pygeoapi/api/tiles.py | 13 ++- pygeoapi/openapi.py | 8 +- pygeoapi/process/manager/base.py | 2 - pygeoapi/util.py | 5 +- tests/api/test_api.py | 22 ++++- tests/api/test_coverages.py | 16 ++-- tests/api/test_itemtypes.py | 20 +---- 14 files changed, 201 insertions(+), 150 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index 2fd768e1f..d284b67ba 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -59,7 +59,8 @@ from pygeoapi.log import setup_logger from pygeoapi.plugin import load_plugin from pygeoapi.process.manager.base import get_manager -from pygeoapi.provider.base import ProviderConnectionError, ProviderTypeError +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderGenericError, ProviderTypeError) from pygeoapi.util import ( CrsTransformSpec, TEMPLATES, UrlPrefetcher, dategetter, @@ -1283,6 +1284,87 @@ def describe_collections(self, request: Union[APIRequest, Any], return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) + @gzip + @pre_process + def get_collection_schema(self, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Returns a collection schema + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**self.api_headers) + + if any([dataset is None, + dataset not in self.config['resources'].keys()]): + + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection schema') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + schema = { + 'type': 'object', + 'title': l10n.translate( + self.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{self.get_collections_url()}/{dataset}/schema' + } + + if p.type != 'coverage': + schema['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + schema['properties'][k] = v + + if k == p.id_field: + schema['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + schema['properties'][k]['x-ogc-role'] = 'primary-instant' + + if request.format == F_HTML: # render + schema['title'] = l10n.translate( + self.config['resources'][dataset]['title'], request.locale) + + schema['collections_path'] = self.get_collections_url() + + content = render_j2_template(self.tpl_config, + 'collections/schema.html', + schema, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(schema, self.pretty_print) + def get_exception(self, status, headers, format_, code, description) -> Tuple[dict, int, str]: """ diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py index 8d5e8c678..7761bd583 100644 --- a/pygeoapi/api/coverages.py +++ b/pygeoapi/api/coverages.py @@ -200,7 +200,16 @@ def get_collection_coverage( return api.get_format_exception(request) -def get_oas_30(cfg: dict, locale: str) -> dict: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags and `dict` of path objects + """ + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections paths = {} @@ -241,4 +250,4 @@ def get_oas_30(cfg: dict, locale: str) -> dict: } } - return {'tags': [], 'paths': paths} + return [], {'paths': paths} diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index d48ca0dba..9ffe0c116 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -206,7 +206,16 @@ def get_collection_edr_query(api: API, request: APIRequest, return headers, HTTPStatus.OK, content -def get_oas_30(cfg: dict, locale: str) -> dict: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags and `dict` of path objects + """ + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections LOGGER.debug('setting up edr endpoints') @@ -320,4 +329,4 @@ def get_oas_30(cfg: dict, locale: str) -> dict: } } - return {'tags': [], 'paths': paths} + return [], {'paths': paths} diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index d4511f4cb..fb5e730f4 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -100,86 +100,6 @@ ] -def get_collection_schema( - api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: - """ - Returns a collection schema - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - headers = request.get_response_headers(**api.api_headers) - - if any([dataset is None, - dataset not in api.config['resources'].keys()]): - - msg = 'Collection not found' - return api.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection schema') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - api.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - api.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - api.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return api.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - schema = { - 'type': 'object', - 'title': l10n.translate( - api.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{api.get_collections_url()}/{dataset}/schema' - } - - if p.type != 'coverage': - schema['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - schema['properties'][k] = v - - if k == p.id_field: - schema['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - schema['properties'][k]['x-ogc-role'] = 'primary-instant' - - if request.format == F_HTML: # render - schema['title'] = l10n.translate( - api.config['resources'][dataset]['title'], request.locale) - - schema['collections_path'] = api.get_collections_url() - - content = render_j2_template(api.tpl_config, - 'collections/schema.html', - schema, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(schema, api.pretty_print) - - def get_collection_queryables(api: API, request: Union[APIRequest, Any], dataset=None) -> Tuple[dict, int, str]: """ @@ -1375,7 +1295,16 @@ def set_content_crs_header( headers['Content-Crs'] = f'<{content_crs_uri}>' -def get_oas_30(cfg: dict, locale: str) -> dict: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags, and `dict` of path objects + """ + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections properties = { @@ -1678,4 +1607,4 @@ def get_oas_30(cfg: dict, locale: str) -> dict: except ProviderTypeError: LOGGER.debug('collection is not feature/item based') - return {'tags': [], 'paths': paths} + return [], {'paths': paths} diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index 20df69e68..a402ce63e 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -243,7 +243,16 @@ def get_collection_map_legend(api: API, request: APIRequest, data, api.pretty_print) -def get_oas_30(cfg: dict, locale: str) -> dict: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags, and `dict` of path objects + """ + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections LOGGER.debug('setting up maps endpoints') @@ -328,4 +337,4 @@ def get_oas_30(cfg: dict, locale: str) -> dict: paths[pth]['get']['parameters'].append( {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - return {'tags': [], 'paths': paths} + return [], {'paths': paths} diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index d8eee308c..a29e2b082 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -554,7 +554,16 @@ def delete_job( return {}, http_status, response -def get_oas_30(cfg: dict, locale_: str): +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags, and `dict` of path objects + """ + from pygeoapi.openapi import OPENAPI_YAML LOGGER.debug('setting up processes endpoints') @@ -588,9 +597,9 @@ def get_oas_30(cfg: dict, locale_: str): if k.startswith('_'): LOGGER.debug(f'Skipping hidden layer: {k}') continue - name = l10n.translate(k, locale_) + name = l10n.translate(k, locale) p = process_manager.get_processor(k) - md_desc = l10n.translate(p.metadata['description'], locale_) + md_desc = l10n.translate(p.metadata['description'], locale) process_name_path = f'/processes/{name}' tag = { 'name': name, @@ -599,7 +608,7 @@ def get_oas_30(cfg: dict, locale_: str): } for link in p.metadata.get('links', []): if link['type'] == 'information': - translated_link = l10n.translate(link, locale_) + translated_link = l10n.translate(link, locale) tag['externalDocs']['description'] = translated_link[ 'type'] tag['externalDocs']['url'] = translated_link['url'] @@ -627,7 +636,7 @@ def get_oas_30(cfg: dict, locale_: str): paths[f'{process_name_path}/execution'] = { 'post': { - 'summary': f"Process {l10n.translate(p.metadata['title'], locale_)} execution", # noqa + 'summary': f"Process {l10n.translate(p.metadata['title'], locale)} execution", # noqa 'description': md_desc, 'tags': [name], 'operationId': f'execute{name.capitalize()}Job', @@ -728,12 +737,4 @@ def get_oas_30(cfg: dict, locale_: str): } } - oas['paths'] = paths - - tag = { - 'name': 'jobs', - 'description': 'Process jobs', - } - oas['tags'].insert(1, tag) - - return oas + return ['jobs'], {'paths': paths} diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index e2b4fb623..3ffd0da0c 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -224,7 +224,16 @@ def get_stac_path(api: API, request: APIRequest, return headers, HTTPStatus.OK, stac_data -def get_oas_30(cfg: dict, locale: str) -> dict: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags, and `dict` of path objects + """ + LOGGER.debug('setting up STAC') stac_collections = filter_dict_by_key_value(cfg['resources'], 'type', 'stac-collection') @@ -243,11 +252,4 @@ def get_oas_30(cfg: dict, locale: str) -> dict: } } } - return { - 'tags': [{ - 'name': 'stac', - 'description': 'SpatioTemporal Asset Catalog' - }], - 'paths': paths, - - } + return ['stac'], {'paths': paths} diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index f4d94a68a..1613bd0a1 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -436,7 +436,16 @@ def tilematrixset(api: API, return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) -def get_oas_30(cfg: dict, locale: str) -> dict: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tags, and `dict` of path objects + """ + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections paths = {} @@ -522,4 +531,4 @@ def get_oas_30(cfg: dict, locale: str) -> dict: } } - return {'tags': [], 'paths': paths} + return [], {'paths': paths} diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 88a3bd501..52664079f 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -65,7 +65,7 @@ THISDIR = os.path.dirname(os.path.realpath(__file__)) -def get_ogc_schemas_location(server_config) -> dict: +def get_ogc_schemas_location(server_config: dict) -> str: """ Determine OGC schemas location @@ -454,9 +454,9 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: LOGGER.debug(f'Adding OpenAPI definitions for {api_name}') try: - sub_oas = api_module.get_oas_30(cfg, locale_) - oas['paths'].update(sub_oas['paths']) - oas['tags'].extend(sub_oas['tags']) + sub_tags, sub_paths = api_module.get_oas_30(cfg, locale_) + oas['paths'].update(sub_paths) + oas['tags'].extend(sub_tags) except Exception as err: if fail_on_invalid_collection: raise diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index 11c1a802f..d4d8f23b4 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -320,8 +320,6 @@ def _execute_handler_sync(self, p: BaseProcessor, job_id: str, self.update_job(job_id, job_metadata) self._send_failed_notification(subscriber) - self._send_failed_notification(subscriber) - return jfmt, outputs, current_status def execute_process( diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 17ad7a7dd..7e1f64e22 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -609,9 +609,12 @@ class JobStatus(Enum): @dataclass(frozen=True) class Subscriber: - """Store subscriber urls as defined in: + """ + Store subscriber URLs as defined in: + https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/subscriber.yaml # noqa """ + success_uri: str in_progress_uri: Optional[str] failed_uri: Optional[str] diff --git a/tests/api/test_api.py b/tests/api/test_api.py index d43a4aae9..1f2c2f344 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -40,7 +40,8 @@ from pygeoapi.api import ( API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP, - __version__, validate_bbox, validate_datetime, validate_subset + __version__, validate_bbox, validate_datetime, + validate_subset ) from pygeoapi.util import yaml_load, get_api_rules, get_base_url @@ -594,6 +595,7 @@ def test_describe_collections(config, api_): rsp_headers, code, response = api_.describe_collections(req) collections = json.loads(response) + print(collections) assert len(collections) == 2 assert len(collections['collections']) == 9 assert len(collections['links']) == 3 @@ -753,6 +755,24 @@ def test_describe_collections_enclosures(config_enclosure, enclosure_api): modified_enclosures['download link 3']['type'] +def test_get_collection_schema(config, api_): + req = mock_request() + rsp_headers, code, response = api_.get_collection_schema(req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 5 + + def test_validate_bbox(): assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] diff --git a/tests/api/test_coverages.py b/tests/api/test_coverages.py index 50e5166d6..f8cad2c4c 100644 --- a/tests/api/test_coverages.py +++ b/tests/api/test_coverages.py @@ -35,9 +35,7 @@ import pytest -from pygeoapi.api import FORMAT_TYPES, F_HTML from pygeoapi.api.coverages import get_collection_coverage -from pygeoapi.api.itemtypes import get_collection_schema from pygeoapi.util import yaml_load from tests.util import get_test_file_path, mock_request, mock_api_request @@ -67,14 +65,14 @@ def test_describe_collections(config, api_): def test_get_collection_schema(config, api_): req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = get_collection_schema( - api_, req, 'gdps-temperature') + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Type'] == 'application/schema+json' req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_schema( - api_, req, 'gdps-temperature') + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') assert rsp_headers['Content-Type'] == 'application/schema+json' schema = json.loads(response) @@ -83,8 +81,8 @@ def test_get_collection_schema(config, api_): assert len(schema['properties']) == 1 req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_schema( - api_, req, 'gdps-temperature') + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') assert rsp_headers['Content-Type'] == 'application/schema+json' schema = json.loads(response) diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py index 47adb2f32..1ced8ea45 100644 --- a/tests/api/test_itemtypes.py +++ b/tests/api/test_itemtypes.py @@ -43,7 +43,7 @@ from pygeoapi.api import (API, FORMAT_TYPES, F_GZIP, F_HTML, F_JSONLD, apply_gzip) from pygeoapi.api.itemtypes import ( - get_collection_schema, get_collection_queryables, get_collection_item, + get_collection_queryables, get_collection_item, get_collection_items, manage_collection_item) from pygeoapi.util import yaml_load, get_crs_from_uri @@ -56,24 +56,6 @@ def config(): return yaml_load(fh) -def test_get_collection_schema(config, api_): - req = mock_api_request() - rsp_headers, code, response = get_collection_schema(api_, req, 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_api_request({'f': 'html'}) - rsp_headers, code, response = get_collection_schema(api_, req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_api_request({'f': 'json'}) - rsp_headers, code, response = get_collection_schema(api_, req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 5 - - def test_get_collection_queryables(config, api_): req = mock_api_request() rsp_headers, code, response = get_collection_queryables( From f005fefb52f9a426bbeea0e2a1431d352886265b Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 2 Apr 2024 20:51:23 -0400 Subject: [PATCH 62/71] test with xarray 2024.2.0 --- requirements-provider.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-provider.txt b/requirements-provider.txt index 9dec1fec5..4c8e91c70 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -15,6 +15,6 @@ pygeometa pymongo==3.10.1 scipy sodapy -xarray +xarray==2024.2.0 zarr s3fs<=2023.6.0 From 284af73012109811260e68761d09c0ed67a4269d Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 2 Apr 2024 21:47:44 -0400 Subject: [PATCH 63/71] remove unneeded file --- tests/api/__init__.py | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 tests/api/__init__.py diff --git a/tests/api/__init__.py b/tests/api/__init__.py deleted file mode 100644 index ce0b0bef8..000000000 --- a/tests/api/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# John A Stevenson -# Colin Blackburn -# -# Copyright (c) 2024 Tom Kralidis -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= From eeb997c7207a052a140e02972f196f3b625fb0ab Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 2 Apr 2024 22:02:29 -0400 Subject: [PATCH 64/71] safeguard xarray error --- pygeoapi/provider/xarray_.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index d1595c4d3..2609e2947 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -398,12 +398,15 @@ def _get_coverage_properties(self): } if 'crs' in self._data.variables.keys(): - properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa + try: + properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa - properties['inverse_flattening'] = self._data.crs.\ - inverse_flattening + properties['inverse_flattening'] = self._data.crs.\ + inverse_flattening - properties['crs_type'] = 'ProjectedCRS' + properties['crs_type'] = 'ProjectedCRS' + except AttributeError: + pass properties['axes'] = [ properties['x_axis_label'], From 5bd9027645a9665160be1b46bd8075b13dead375 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 2 Apr 2024 22:05:31 -0400 Subject: [PATCH 65/71] unpin xarray --- requirements-provider.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-provider.txt b/requirements-provider.txt index 4c8e91c70..9dec1fec5 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -15,6 +15,6 @@ pygeometa pymongo==3.10.1 scipy sodapy -xarray==2024.2.0 +xarray zarr s3fs<=2023.6.0 From df05295629435ff358e6bb5631ac523c52d796a3 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 2 Apr 2024 22:35:51 -0400 Subject: [PATCH 66/71] fix OpenAPI generation --- pygeoapi/api/coverages.py | 6 +++--- pygeoapi/api/environmental_data_retrieval.py | 6 +++--- pygeoapi/api/itemtypes.py | 6 +++--- pygeoapi/api/maps.py | 6 +++--- pygeoapi/api/processes.py | 6 +++--- pygeoapi/api/stac.py | 6 +++--- pygeoapi/api/tiles.py | 6 +++--- pygeoapi/openapi.py | 4 ++-- tests/api/test_api.py | 1 - tests/api/test_coverages.py | 1 - 10 files changed, 23 insertions(+), 25 deletions(-) diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py index 7761bd583..01e5638a6 100644 --- a/pygeoapi/api/coverages.py +++ b/pygeoapi/api/coverages.py @@ -200,14 +200,14 @@ def get_collection_coverage( return api.get_format_exception(request) -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections @@ -250,4 +250,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: } } - return [], {'paths': paths} + return [{'name': 'coverages'}], {'paths': paths} diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py index 9ffe0c116..f659cc835 100644 --- a/pygeoapi/api/environmental_data_retrieval.py +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -206,14 +206,14 @@ def get_collection_edr_query(api: API, request: APIRequest, return headers, HTTPStatus.OK, content -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections @@ -329,4 +329,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: } } - return [], {'paths': paths} + return [{'name': 'edr'}], {'paths': paths} diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index fb5e730f4..4a746067b 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -1295,14 +1295,14 @@ def set_content_crs_header( headers['Content-Crs'] = f'<{content_crs_uri}>' -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags, and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections @@ -1607,4 +1607,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: except ProviderTypeError: LOGGER.debug('collection is not feature/item based') - return [], {'paths': paths} + return [{'name': 'records'}, {'name': 'features'}], {'paths': paths} diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py index a402ce63e..03866fb25 100644 --- a/pygeoapi/api/maps.py +++ b/pygeoapi/api/maps.py @@ -243,14 +243,14 @@ def get_collection_map_legend(api: API, request: APIRequest, data, api.pretty_print) -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags, and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections @@ -337,4 +337,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: paths[pth]['get']['parameters'].append( {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - return [], {'paths': paths} + return [{'name': 'maps'}], {'paths': paths} diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py index a29e2b082..6cc7a7180 100644 --- a/pygeoapi/api/processes.py +++ b/pygeoapi/api/processes.py @@ -554,14 +554,14 @@ def delete_job( return {}, http_status, response -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags, and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ from pygeoapi.openapi import OPENAPI_YAML @@ -737,4 +737,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: } } - return ['jobs'], {'paths': paths} + return [{'name': 'proceses'}, {'name': 'jobs'}], {'paths': paths} diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index 3ffd0da0c..dc07a806b 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -224,14 +224,14 @@ def get_stac_path(api: API, request: APIRequest, return headers, HTTPStatus.OK, stac_data -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags, and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ LOGGER.debug('setting up STAC') @@ -252,4 +252,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: } } } - return ['stac'], {'paths': paths} + return [{'name': 'stac'}], {'paths': paths} diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 1613bd0a1..5f6dd5fec 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -436,14 +436,14 @@ def tilematrixset(api: API, return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) -def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa """ Get OpenAPI fragments :param cfg: `dict` of configuration :param locale: `str` of locale - :returns: `tuple` of `list` of tags, and `dict` of path objects + :returns: `tuple` of `list` of tag objects, and `dict` of path objects """ from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections @@ -531,4 +531,4 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[str], dict[str, dict]]: } } - return [], {'paths': paths} + return [{'name': 'tiles'}], {'paths': paths} diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 52664079f..b089f39c1 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -167,7 +167,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'x-keywords': l10n.translate(cfg['metadata']['identification']['keywords'], locale_), # noqa 'termsOfService': cfg['metadata']['identification']['terms_of_service'], - 'contact': { + 'contact': { 'name': cfg['metadata']['provider']['name'], 'url': cfg['metadata']['provider']['url'], 'email': cfg['metadata']['contact']['email'] @@ -455,7 +455,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: try: sub_tags, sub_paths = api_module.get_oas_30(cfg, locale_) - oas['paths'].update(sub_paths) + oas['paths'].update(sub_paths['paths']) oas['tags'].extend(sub_tags) except Exception as err: if fail_on_invalid_collection: diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 1f2c2f344..c11afca28 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -595,7 +595,6 @@ def test_describe_collections(config, api_): rsp_headers, code, response = api_.describe_collections(req) collections = json.loads(response) - print(collections) assert len(collections) == 2 assert len(collections['collections']) == 9 assert len(collections['links']) == 3 diff --git a/tests/api/test_coverages.py b/tests/api/test_coverages.py index f8cad2c4c..39b275228 100644 --- a/tests/api/test_coverages.py +++ b/tests/api/test_coverages.py @@ -88,7 +88,6 @@ def test_get_collection_schema(config, api_): assert 'properties' in schema assert len(schema['properties']) == 1 - print(schema['properties']) assert schema['properties']['1']['type'] == 'number' assert schema['properties']['1']['title'] == 'Temperature [C]' From e72d4ba3a5ba3b8621ca839e7814429beeeb8f01 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 3 Apr 2024 06:21:56 -0400 Subject: [PATCH 67/71] fix schema endpoint in Flask and Starlette --- pygeoapi/flask_app.py | 3 +-- pygeoapi/starlette_app.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 7e77b75dd..e4d8e82d1 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -245,8 +245,7 @@ def collection_schema(collection_id): :returns: HTTP response """ - return execute_from_flask(itemtypes_api.get_collection_schema, request, - collection_id) + return get_response(api_.get_collection_schema(request, collection_id)) @BLUEPRINT.route('/collections//queryables') diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 71cefe09d..6d6b7054c 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -229,9 +229,8 @@ async def collection_schema(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await execute_from_starlette( - itemtypes_api.get_collection_schema, request, collection_id - ) + return await get_response(api_.get_collection_schema, request, + collection_id) async def collection_queryables(request: Request, collection_id=None): From 122666cc6799e1ea1217d30f90341368c241ae58 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Wed, 3 Apr 2024 06:39:00 -0400 Subject: [PATCH 68/71] Safely serialize configuration JSON (#1605) * Safely serialize configuration JSON Co-Authored-By: Tom Kralidis * Revert "Safely serialize configuration JSON" This reverts commit 36feb067ee6f87e61955852dc48994f075806370. * Add test for datetime with Admin API * Safely serialize configuration JSON --------- Co-authored-by: Tom Kralidis --- pygeoapi/admin.py | 4 ++-- tests/data/admin/resource-post.json | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pygeoapi/admin.py b/pygeoapi/admin.py index ff98b611f..215414408 100644 --- a/pygeoapi/admin.py +++ b/pygeoapi/admin.py @@ -128,8 +128,8 @@ def write_config(self, config): # Preserve env variables LOGGER.debug('Reading env variables in configuration') - raw_conf = get_config(raw=True) - conf = get_config() + raw_conf = json.loads(to_json(get_config(raw=True))) + conf = json.loads(to_json(get_config())) patch = make_patch(conf, raw_conf) LOGGER.debug('Merging env variables') diff --git a/tests/data/admin/resource-post.json b/tests/data/admin/resource-post.json index 031d8d76e..fb7804da1 100644 --- a/tests/data/admin/resource-post.json +++ b/tests/data/admin/resource-post.json @@ -32,6 +32,9 @@ 90 ], "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + }, + "temporal": { + "begin": "2018-02-14T11:11:11Z" } }, "providers": [ From 07c1c375313dd22bdfa03d3c8502c4a28344eea7 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 3 Apr 2024 12:30:39 -0400 Subject: [PATCH 69/71] backport #1611 --- pygeoapi/api/stac.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py index dc07a806b..948b68857 100644 --- a/pygeoapi/api/stac.py +++ b/pygeoapi/api/stac.py @@ -182,7 +182,8 @@ def get_stac_path(api: API, request: APIRequest, if isinstance(stac_data, dict): content.update(stac_data) - content['links'].extend(stac_collections[dataset]['links']) + content['links'].extend( + stac_collections[dataset].get('links', [])) if request.format == F_HTML: # render content['path'] = path From 74d709d2098c7f6a542993692ab626b253287564 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Thu, 4 Apr 2024 10:15:45 +0200 Subject: [PATCH 70/71] Also fix schema endpoint for django Fix is analogous to e72d4ba3a5ba3b8621ca839e7814429beeeb8f01 --- pygeoapi/django_/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 84c50bae3..fe841b2fd 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -145,8 +145,8 @@ def collection_schema(request: HttpRequest, :returns: Django HTTP Response """ - response_ = execute_from_django( - itemtypes_api.get_collection_schema, request, collection_id + response_ = _feed_response( + request, 'get_collection_schema', collection_id ) response = _to_django_response(*response_) From c3c3b2f3cf314aadf6d5fa62a3fd500408bdb831 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 4 Apr 2024 19:14:38 -0400 Subject: [PATCH 71/71] address additional PR comments --- pygeoapi/api/__init__.py | 3 +-- pygeoapi/openapi.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py index d284b67ba..0d11c3550 100644 --- a/pygeoapi/api/__init__.py +++ b/pygeoapi/api/__init__.py @@ -354,8 +354,7 @@ def with_data(cls, request, supported_locales) -> 'APIRequest': return api_req @classmethod - def from_flask(cls, request, supported_locales - ) -> 'APIRequest': + def from_flask(cls, request, supported_locales) -> 'APIRequest': """Factory class similar to with_data, but only for flask requests""" api_req = cls(request, supported_locales) api_req._data = request.data diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index b089f39c1..274e19dad 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -167,7 +167,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'x-keywords': l10n.translate(cfg['metadata']['identification']['keywords'], locale_), # noqa 'termsOfService': cfg['metadata']['identification']['terms_of_service'], - 'contact': { + 'contact': { 'name': cfg['metadata']['provider']['name'], 'url': cfg['metadata']['provider']['url'], 'email': cfg['metadata']['contact']['email']