diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
index 13689fbbf..9f2705546 100644
--- a/.github/workflows/tests-docker.yml
+++ b/.github/workflows/tests-docker.yml
@@ -40,7 +40,7 @@ jobs:
docker logs wis2box-management
- name: setup wis2box-management ⚙️
run: |
- sleep 10
+ sleep 30
python3 wis2box-ctl.py execute wis2box environment show
- name: add Malawi data 🇲🇼
env:
diff --git a/default.env b/default.env
index c18eb08f4..b424fd494 100644
--- a/default.env
+++ b/default.env
@@ -25,6 +25,9 @@ WIS2BOX_BROKER_PUBLIC=mqtt://wis2box:wis2box@mosquitto:1883
WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png
WIS2BOX_BASEMAP_ATTRIBUTION=OpenStreetMap contributors
+# Admin UI
+WIS2BOX_UI_ADMIN_BASEURL=/admin
+
# other
WIS2BOX_URL=http://localhost
diff --git a/docker-compose.yml b/docker-compose.yml
index afbbf90ee..15185bf8f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,7 +13,7 @@ services:
wis2box-ui:
container_name: wis2box-ui
- image: ghcr.io/wmo-im/wis2box-ui:latest
+ image: ghcr.io/wmo-im/wis2box-ui:0.5.3
restart: always
env_file:
- default.env
@@ -21,9 +21,19 @@ services:
depends_on:
- wis2box-api
+# wis2box-ui-admin:
+# container_name: wis2box-ui-admin
+# image: ghcr.io/wmo-im/wis2box-ui-admin:latest
+# restart: always
+# env_file:
+# - default.env
+# - dev.env
+# depends_on:
+# - wis2box-api
+
wis2box-api:
container_name: wis2box-api
- image: ghcr.io/wmo-im/wis2box-api:0.5.1
+ image: ghcr.io/wmo-im/wis2box-api:latest
restart: always
env_file:
- default.env
@@ -58,12 +68,12 @@ services:
elasticsearch:
container_name: elasticsearch
- image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+ image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2
restart: always
environment:
- discovery.type=single-node
+ - discovery.seed_hosts=[]
- node.name=elasticsearch-01
- - discovery.seed_hosts=elasticsearch-01
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- cluster.name=es-wis2box
diff --git a/docs/source/reference/data-access/r-api.ipynb b/docs/source/reference/data-access/r-api.ipynb
index ae12f58c2..1e0e05702 100644
--- a/docs/source/reference/data-access/r-api.ipynb
+++ b/docs/source/reference/data-access/r-api.ipynb
@@ -152,7 +152,7 @@
" \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \u001b[3m\u001b[90m\u001b[39m\u001b[23m \n",
"\u001b[90m1\u001b[39m data.core… \u001b[90m\"\u001b[39m[ { \\\"sc… Surf… Surface we… \u001b[90m\"\u001b[39m[ { … \u001b[90m\"\u001b[39m[ { \\\"n… en data… \u001b[90m\"\u001b[39m{ \\\"…\n",
"\u001b[90m# … with 5 more variables: created , rights ,\u001b[39m\n",
- "\u001b[90m# X_metadata.anytext , id , geometry \u001b[39m\n"
+ "\u001b[90m# id , geometry \u001b[39m\n"
]
}
],
diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst
index e9351d352..6b8a7ea44 100644
--- a/docs/source/reference/quickstart.rst
+++ b/docs/source/reference/quickstart.rst
@@ -8,7 +8,7 @@ It is the minimal runtime configuration profile as used in wis2box GitHub CI/CD:
.. note:: wis2box web components are run on port 80 by default. When using wis2box from source, the default port for web components is 8999, to be used for development.
-To download the wis2box from source:
+To download wis2box from source:
.. code-block:: bash
diff --git a/docs/source/reference/running/api-publishing.rst b/docs/source/reference/running/api-publishing.rst
index e09e64db8..738b5069c 100644
--- a/docs/source/reference/running/api-publishing.rst
+++ b/docs/source/reference/running/api-publishing.rst
@@ -25,14 +25,12 @@ will generate local station collection GeoJSON for API publication.
wis2box metadata station publish-collection
-.. note:: run the command ``wis2box metadata station publish-collection`` to
- publish your stations as a collection to the wis2box API
-
+.. note:: This command also runs automatically at startup and thereafter every 10 minutes
+ to keep your stations up to date.
.. seealso:: :ref:`station-metadata`
-
Discovery metadata API publishing
---------------------------------
diff --git a/nginx/nginx-ssl.conf b/nginx/nginx-ssl.conf
index 4d9545c76..2d5b3bc17 100644
--- a/nginx/nginx-ssl.conf
+++ b/nginx/nginx-ssl.conf
@@ -67,7 +67,9 @@
location / {
proxy_pass http://wis2box-ui:80;
}
-
+# location /admin/ {
+# proxy_pass http://wis2box-ui-admin:80/;
+# }
location /auth {
internal;
proxy_pass http://wis2box-auth:80/authorize;
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index c6a42bec7..0538bcdaf 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -64,7 +64,9 @@
location / {
proxy_pass http://wis2box-ui:80;
}
-
+# location /admin/ {
+# proxy_pass http://wis2box-ui-admin:80/;
+# }
location /auth {
internal;
proxy_pass http://wis2box-auth:80/authorize;
diff --git a/wis2box-management/docker/entrypoint.sh b/wis2box-management/docker/entrypoint.sh
index 671faf1d6..cb5382f46 100755
--- a/wis2box-management/docker/entrypoint.sh
+++ b/wis2box-management/docker/entrypoint.sh
@@ -43,6 +43,7 @@ cd ~/.pywcmp/wis2-topic-hierarchy && unzip -j /tmp/all.json.zip
wis2box environment create
wis2box environment show
wis2box api setup
+wis2box metadata discovery setup
wis2box metadata station publish-collection
echo "END /entrypoint.sh"
exec "$@"
diff --git a/wis2box-management/requirements.txt b/wis2box-management/requirements.txt
index ab4cc938f..8cf5570ef 100644
--- a/wis2box-management/requirements.txt
+++ b/wis2box-management/requirements.txt
@@ -1,4 +1,4 @@
-elasticsearch<8
+elasticsearch
isodate
minio
OWSLib
diff --git a/wis2box-management/wis2box/api/backend/elastic.py b/wis2box-management/wis2box/api/backend/elastic.py
index 937bbc792..b0a9fe0b8 100644
--- a/wis2box-management/wis2box/api/backend/elastic.py
+++ b/wis2box-management/wis2box/api/backend/elastic.py
@@ -19,7 +19,6 @@
#
###############################################################################
-from copy import deepcopy
import logging
from elasticsearch import Elasticsearch, helpers
@@ -33,60 +32,59 @@
# default index settings
SETTINGS = {
- 'settings': {
- 'number_of_shards': 1,
- 'number_of_replicas': 0
- },
- 'mappings': {
- 'properties': {
- 'geometry': {
- 'type': 'geo_shape'
- },
- 'reportId': {
- 'type': 'text',
- 'fields': {
- 'raw': {
- 'type': 'keyword'
- }
+ 'number_of_shards': 1,
+ 'number_of_replicas': 0
+}
+
+MAPPINGS = {
+ 'properties': {
+ 'geometry': {
+ 'type': 'geo_shape'
+ },
+ 'reportId': {
+ 'type': 'text',
+ 'fields': {
+ 'raw': {
+ 'type': 'keyword'
}
- },
+ }
+ },
+ 'properties': {
'properties': {
- 'properties': {
- 'resultTime': {
- 'type': 'date',
- 'fields': {
- 'raw': {
- 'type': 'keyword'
- }
+ 'resultTime': {
+ 'type': 'date',
+ 'fields': {
+ 'raw': {
+ 'type': 'keyword'
}
- },
- 'pubTime': {
- 'type': 'date',
- 'fields': {
- 'raw': {
- 'type': 'keyword'
- }
- }
- },
- 'phenomenonTime': {
- 'type': 'text'
- },
- 'wigos_station_identifier': {
- 'type': 'text',
- 'fields': {
- 'raw': {'type': 'keyword'}
+ }
+ },
+ 'pubTime': {
+ 'type': 'date',
+ 'fields': {
+ 'raw': {
+ 'type': 'keyword'
}
- },
- 'value': {
- 'type': 'float',
- 'coerce': True
- },
- 'metadata': {
- 'properties': {
- 'value': {
- 'type': 'float',
- 'coerce': True
- }
+ }
+ },
+ 'phenomenonTime': {
+ 'type': 'text'
+ },
+ 'wigos_station_identifier': {
+ 'type': 'text',
+ 'fields': {
+ 'raw': {'type': 'keyword'}
+ }
+ },
+ 'value': {
+ 'type': 'float',
+ 'coerce': True
+ },
+ 'metadata': {
+ 'properties': {
+ 'value': {
+ 'type': 'float',
+ 'coerce': True
}
}
}
@@ -111,7 +109,7 @@ def __init__(self, defs: dict) -> None:
self.type = 'Elasticsearch'
self.url = defs.get('url').rstrip('/')
- self.conn = Elasticsearch([self.url], timeout=30,
+ self.conn = Elasticsearch(self.url, timeout=30,
max_retries=10, retry_on_timeout=True)
@staticmethod
@@ -140,10 +138,9 @@ def add_collection(self, collection_id: str) -> dict:
LOGGER.error(msg)
raise RuntimeError(msg)
- settings = deepcopy(SETTINGS)
-
LOGGER.debug('Creating index')
- self.conn.indices.create(index=es_index, body=settings)
+ self.conn.options().indices.create(index=es_index, mappings=MAPPINGS,
+ settings=SETTINGS)
return self.has_collection(collection_id)
@@ -162,7 +159,7 @@ def delete_collection(self, collection_id: str) -> bool:
LOGGER.error(msg)
raise RuntimeError(msg)
- if self.conn.indices.exists(es_index):
+ if self.conn.indices.exists(index=es_index):
self.conn.indices.delete(index=es_index)
return not self.has_collection(collection_id)
@@ -178,7 +175,7 @@ def has_collection(self, collection_id: str) -> bool:
es_index = self.es_id(collection_id)
indices = self.conn.indices
- return indices.exists(es_index)
+ return indices.exists(index=es_index)
def upsert_collection_items(self, collection_id: str, items: list) -> str:
"""
@@ -267,7 +264,7 @@ def delete_collections_by_retention(self, days: int) -> bool:
for index in indices:
LOGGER.debug(f'deleting documents older than {days} days ({before})') # noqa
- self.conn.delete_by_query(index=index, body=query_by_date)
+ self.conn.delete_by_query(index=index, **query_by_date)
return
diff --git a/wis2box-management/wis2box/api/config/pygeoapi.py b/wis2box-management/wis2box/api/config/pygeoapi.py
index c9e1370f2..646736d4c 100644
--- a/wis2box-management/wis2box/api/config/pygeoapi.py
+++ b/wis2box-management/wis2box/api/config/pygeoapi.py
@@ -104,11 +104,17 @@ def prepare_collection(self, meta: dict) -> bool:
:returns: `dict` of collection configuration
"""
+ editable = False
+
if meta['id'] in ['discovery-metadata', 'messages', 'stations']:
resource_id = meta['id']
else:
resource_id = meta['topic_hierarchy']
+ # TODO: uncomment once admin UI is implemented
+ # if meta['id'] in ['discovery-metadata', 'stations']:
+ # editable = True
+
LOGGER.debug(f'Resource id: {resource_id}')
type_ = meta.get('type', 'feature')
@@ -131,6 +137,7 @@ def prepare_collection(self, meta: dict) -> bool:
},
'providers': [{
'type': type_,
+ 'editable': editable,
'name': provider_name,
'data': f'{API_BACKEND_URL}/{resource_id}',
'id_field': meta.get('id_field'),
diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py
index 614d9ef67..f4310f5c7 100644
--- a/wis2box-management/wis2box/metadata/discovery.py
+++ b/wis2box-management/wis2box/metadata/discovery.py
@@ -98,16 +98,6 @@ def generate(self, mcf: dict) -> str:
LOGGER.debug('Generating OARec discovery metadata')
record = WMOWCMP2OutputSchema().write(md, stringify=False)
- anytext_bag = [
- md['identification']['title'],
- md['identification']['abstract']
- ]
-
- for k, v in md['identification']['keywords'].items():
- anytext_bag.extend(v['keywords'])
-
- record['properties']['_metadata-anytext'] = ' '.join(anytext_bag)
-
return record
@@ -163,6 +153,16 @@ def discovery_metadata():
pass
+@click.command()
+@click.pass_context
+@cli_helpers.OPTION_VERBOSITY
+def setup(ctx, verbosity):
+ """Initializes metadata repository"""
+
+ click.echo('Setting up discovery metadata repository')
+ setup_collection(meta=gcm())
+
+
@click.command()
@click.pass_context
@cli_helpers.ARGUMENT_FILEPATH
@@ -198,4 +198,5 @@ def unpublish(ctx, identifier, verbosity):
discovery_metadata.add_command(publish)
+discovery_metadata.add_command(setup)
discovery_metadata.add_command(unpublish)