diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 13689fbbf..9f2705546 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -40,7 +40,7 @@ jobs: docker logs wis2box-management - name: setup wis2box-management ⚙️ run: | - sleep 10 + sleep 30 python3 wis2box-ctl.py execute wis2box environment show - name: add Malawi data 🇲🇼 env: diff --git a/default.env b/default.env index c18eb08f4..b424fd494 100644 --- a/default.env +++ b/default.env @@ -25,6 +25,9 @@ WIS2BOX_BROKER_PUBLIC=mqtt://wis2box:wis2box@mosquitto:1883 WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png WIS2BOX_BASEMAP_ATTRIBUTION=OpenStreetMap contributors +# Admin UI +WIS2BOX_UI_ADMIN_BASEURL=/admin + # other WIS2BOX_URL=http://localhost diff --git a/docker-compose.yml b/docker-compose.yml index afbbf90ee..15185bf8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: wis2box-ui: container_name: wis2box-ui - image: ghcr.io/wmo-im/wis2box-ui:latest + image: ghcr.io/wmo-im/wis2box-ui:0.5.3 restart: always env_file: - default.env @@ -21,9 +21,19 @@ services: depends_on: - wis2box-api +# wis2box-ui-admin: +# container_name: wis2box-ui-admin +# image: ghcr.io/wmo-im/wis2box-ui-admin:latest +# restart: always +# env_file: +# - default.env +# - dev.env +# depends_on: +# - wis2box-api + wis2box-api: container_name: wis2box-api - image: ghcr.io/wmo-im/wis2box-api:0.5.1 + image: ghcr.io/wmo-im/wis2box-api:latest restart: always env_file: - default.env @@ -58,12 +68,12 @@ services: elasticsearch: container_name: elasticsearch - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0 + image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2 restart: always environment: - discovery.type=single-node + - discovery.seed_hosts=[] - node.name=elasticsearch-01 - - discovery.seed_hosts=elasticsearch-01 - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - cluster.name=es-wis2box diff --git a/docs/source/reference/data-access/r-api.ipynb b/docs/source/reference/data-access/r-api.ipynb index ae12f58c2..1e0e05702 100644 --- a/docs/source/reference/data-access/r-api.ipynb +++ b/docs/source/reference/data-access/r-api.ipynb @@ -152,7 +152,7 @@ " \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \n", "\u001b[90m1\u001b[39m data.core… \u001b[90m\"\u001b[39m[ { \\\"sc… Surf… Surface we… \u001b[90m\"\u001b[39m[ { … \u001b[90m\"\u001b[39m[ { \\\"n… en data… \u001b[90m\"\u001b[39m{ \\\"…\n", "\u001b[90m# … with 5 more variables: created , rights ,\u001b[39m\n", - "\u001b[90m# X_metadata.anytext , id , geometry \u001b[39m\n" + "\u001b[90m# id , geometry \u001b[39m\n" ] } ], diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index e9351d352..6b8a7ea44 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -8,7 +8,7 @@ It is the minimal runtime configuration profile as used in wis2box GitHub CI/CD: .. note:: wis2box web components are run on port 80 by default. When using wis2box from source, the default port for web components is 8999, to be used for development. -To download the wis2box from source: +To download wis2box from source: .. code-block:: bash diff --git a/docs/source/reference/running/api-publishing.rst b/docs/source/reference/running/api-publishing.rst index e09e64db8..738b5069c 100644 --- a/docs/source/reference/running/api-publishing.rst +++ b/docs/source/reference/running/api-publishing.rst @@ -25,14 +25,12 @@ will generate local station collection GeoJSON for API publication. wis2box metadata station publish-collection -.. note:: run the command ``wis2box metadata station publish-collection`` to - publish your stations as a collection to the wis2box API - +.. note:: This command also runs automatically at startup and thereafter every 10 minutes + to keep your stations up to date. .. seealso:: :ref:`station-metadata` - Discovery metadata API publishing --------------------------------- diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf index 4d9545c76..2d5b3bc17 100644 --- a/nginx/nginx-ssl.conf +++ b/nginx/nginx-ssl.conf @@ -67,7 +67,9 @@ location / { proxy_pass http://wis2box-ui:80; } - +# location /admin/ { +# proxy_pass http://wis2box-ui-admin:80/; +# } location /auth { internal; proxy_pass http://wis2box-auth:80/authorize; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c6a42bec7..0538bcdaf 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -64,7 +64,9 @@ location / { proxy_pass http://wis2box-ui:80; } - +# location /admin/ { +# proxy_pass http://wis2box-ui-admin:80/; +# } location /auth { internal; proxy_pass http://wis2box-auth:80/authorize; diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh index 671faf1d6..cb5382f46 100755 --- a/wis2box-management/docker/entrypoint.sh +++ b/wis2box-management/docker/entrypoint.sh @@ -43,6 +43,7 @@ cd ~/.pywcmp/wis2-topic-hierarchy && unzip -j /tmp/all.json.zip wis2box environment create wis2box environment show wis2box api setup +wis2box metadata discovery setup wis2box metadata station publish-collection echo "END /entrypoint.sh" exec "$@" diff --git a/wis2box-management/requirements.txt b/wis2box-management/requirements.txt index ab4cc938f..8cf5570ef 100644 --- a/wis2box-management/requirements.txt +++ b/wis2box-management/requirements.txt @@ -1,4 +1,4 @@ -elasticsearch<8 +elasticsearch isodate minio OWSLib diff --git a/wis2box-management/wis2box/api/backend/elastic.py b/wis2box-management/wis2box/api/backend/elastic.py index 937bbc792..b0a9fe0b8 100644 --- a/wis2box-management/wis2box/api/backend/elastic.py +++ b/wis2box-management/wis2box/api/backend/elastic.py @@ -19,7 +19,6 @@ # ############################################################################### -from copy import deepcopy import logging from elasticsearch import Elasticsearch, helpers @@ -33,60 +32,59 @@ # default index settings SETTINGS = { - 'settings': { - 'number_of_shards': 1, - 'number_of_replicas': 0 - }, - 'mappings': { - 'properties': { - 'geometry': { - 'type': 'geo_shape' - }, - 'reportId': { - 'type': 'text', - 'fields': { - 'raw': { - 'type': 'keyword' - } + 'number_of_shards': 1, + 'number_of_replicas': 0 +} + +MAPPINGS = { + 'properties': { + 'geometry': { + 'type': 'geo_shape' + }, + 'reportId': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' } - }, + } + }, + 'properties': { 'properties': { - 'properties': { - 'resultTime': { - 'type': 'date', - 'fields': { - 'raw': { - 'type': 'keyword' - } + 'resultTime': { + 'type': 'date', + 'fields': { + 'raw': { + 'type': 'keyword' } - }, - 'pubTime': { - 'type': 'date', - 'fields': { - 'raw': { - 'type': 'keyword' - } - } - }, - 'phenomenonTime': { - 'type': 'text' - }, - 'wigos_station_identifier': { - 'type': 'text', - 'fields': { - 'raw': {'type': 'keyword'} + } + }, + 'pubTime': { + 'type': 'date', + 'fields': { + 'raw': { + 'type': 'keyword' } - }, - 'value': { - 'type': 'float', - 'coerce': True - }, - 'metadata': { - 'properties': { - 'value': { - 'type': 'float', - 'coerce': True - } + } + }, + 'phenomenonTime': { + 'type': 'text' + }, + 'wigos_station_identifier': { + 'type': 'text', + 'fields': { + 'raw': {'type': 'keyword'} + } + }, + 'value': { + 'type': 'float', + 'coerce': True + }, + 'metadata': { + 'properties': { + 'value': { + 'type': 'float', + 'coerce': True } } } @@ -111,7 +109,7 @@ def __init__(self, defs: dict) -> None: self.type = 'Elasticsearch' self.url = defs.get('url').rstrip('/') - self.conn = Elasticsearch([self.url], timeout=30, + self.conn = Elasticsearch(self.url, timeout=30, max_retries=10, retry_on_timeout=True) @staticmethod @@ -140,10 +138,9 @@ def add_collection(self, collection_id: str) -> dict: LOGGER.error(msg) raise RuntimeError(msg) - settings = deepcopy(SETTINGS) - LOGGER.debug('Creating index') - self.conn.indices.create(index=es_index, body=settings) + self.conn.options().indices.create(index=es_index, mappings=MAPPINGS, + settings=SETTINGS) return self.has_collection(collection_id) @@ -162,7 +159,7 @@ def delete_collection(self, collection_id: str) -> bool: LOGGER.error(msg) raise RuntimeError(msg) - if self.conn.indices.exists(es_index): + if self.conn.indices.exists(index=es_index): self.conn.indices.delete(index=es_index) return not self.has_collection(collection_id) @@ -178,7 +175,7 @@ def has_collection(self, collection_id: str) -> bool: es_index = self.es_id(collection_id) indices = self.conn.indices - return indices.exists(es_index) + return indices.exists(index=es_index) def upsert_collection_items(self, collection_id: str, items: list) -> str: """ @@ -267,7 +264,7 @@ def delete_collections_by_retention(self, days: int) -> bool: for index in indices: LOGGER.debug(f'deleting documents older than {days} days ({before})') # noqa - self.conn.delete_by_query(index=index, body=query_by_date) + self.conn.delete_by_query(index=index, **query_by_date) return diff --git a/wis2box-management/wis2box/api/config/pygeoapi.py b/wis2box-management/wis2box/api/config/pygeoapi.py index c9e1370f2..646736d4c 100644 --- a/wis2box-management/wis2box/api/config/pygeoapi.py +++ b/wis2box-management/wis2box/api/config/pygeoapi.py @@ -104,11 +104,17 @@ def prepare_collection(self, meta: dict) -> bool: :returns: `dict` of collection configuration """ + editable = False + if meta['id'] in ['discovery-metadata', 'messages', 'stations']: resource_id = meta['id'] else: resource_id = meta['topic_hierarchy'] + # TODO: uncomment once admin UI is implemented + # if meta['id'] in ['discovery-metadata', 'stations']: + # editable = True + LOGGER.debug(f'Resource id: {resource_id}') type_ = meta.get('type', 'feature') @@ -131,6 +137,7 @@ def prepare_collection(self, meta: dict) -> bool: }, 'providers': [{ 'type': type_, + 'editable': editable, 'name': provider_name, 'data': f'{API_BACKEND_URL}/{resource_id}', 'id_field': meta.get('id_field'), diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py index 614d9ef67..f4310f5c7 100644 --- a/wis2box-management/wis2box/metadata/discovery.py +++ b/wis2box-management/wis2box/metadata/discovery.py @@ -98,16 +98,6 @@ def generate(self, mcf: dict) -> str: LOGGER.debug('Generating OARec discovery metadata') record = WMOWCMP2OutputSchema().write(md, stringify=False) - anytext_bag = [ - md['identification']['title'], - md['identification']['abstract'] - ] - - for k, v in md['identification']['keywords'].items(): - anytext_bag.extend(v['keywords']) - - record['properties']['_metadata-anytext'] = ' '.join(anytext_bag) - return record @@ -163,6 +153,16 @@ def discovery_metadata(): pass +@click.command() +@click.pass_context +@cli_helpers.OPTION_VERBOSITY +def setup(ctx, verbosity): + """Initializes metadata repository""" + + click.echo('Setting up discovery metadata repository') + setup_collection(meta=gcm()) + + @click.command() @click.pass_context @cli_helpers.ARGUMENT_FILEPATH @@ -198,4 +198,5 @@ def unpublish(ctx, identifier, verbosity): discovery_metadata.add_command(publish) +discovery_metadata.add_command(setup) discovery_metadata.add_command(unpublish)