update to Elasticsearch 8 and latest pygeoapi, add wis2box-ui-admin

.github/workflows/tests-docker.yml

@@ -40,7 +40,7 @@ jobs:
         docker logs wis2box-management
     - name: setup wis2box-management ⚙️
       run: |
-        sleep 10
+        sleep 30
         python3 wis2box-ctl.py execute wis2box environment show
     - name: add Malawi data 🇲🇼
       env:

default.env

@@ -25,6 +25,9 @@ WIS2BOX_BROKER_PUBLIC=mqtt://wis2box:wis2box@mosquitto:1883
 WIS2BOX_BASEMAP_URL=https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png
 WIS2BOX_BASEMAP_ATTRIBUTION=<a href="https://osm.org/copyright">OpenStreetMap</a> contributors
 
+# Admin UI
+WIS2BOX_UI_ADMIN_BASEURL=/admin
+
 # other
 WIS2BOX_URL=http://localhost
 

docker-compose.yml

@@ -13,17 +13,27 @@ services:
 
   wis2box-ui:
     container_name: wis2box-ui
-    image: ghcr.io/wmo-im/wis2box-ui:latest
+    image: ghcr.io/wmo-im/wis2box-ui:0.5.3
     restart: always
     env_file:
       - default.env
       - dev.env
     depends_on:
       - wis2box-api
 
+#  wis2box-ui-admin:
+#    container_name: wis2box-ui-admin
+#    image: ghcr.io/wmo-im/wis2box-ui-admin:latest
+#    restart: always
+#    env_file:
+#      - default.env
+#      - dev.env
+#    depends_on:
+#      - wis2box-api
+
   wis2box-api:
     container_name: wis2box-api
-    image: ghcr.io/wmo-im/wis2box-api:0.5.1
+    image: ghcr.io/wmo-im/wis2box-api:latest
     restart: always
     env_file:
       - default.env
@@ -58,12 +68,12 @@ services:
 
   elasticsearch:
     container_name: elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.2
     restart: always
     environment:
       - discovery.type=single-node
+      - discovery.seed_hosts=[]
       - node.name=elasticsearch-01
-      - discovery.seed_hosts=elasticsearch-01
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
       - cluster.name=es-wis2box

docs/source/reference/data-access/r-api.ipynb

@@ -152,7 +152,7 @@
       "  \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m      \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m      \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m       \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m  \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m     \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m    \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m \u001b[3m\u001b[90m<chr>\u001b[39m\u001b[23m \n",
       "\u001b[90m1\u001b[39m data.core… \u001b[90m\"\u001b[39m[ { \\\"sc… Surf… Surface we… \u001b[90m\"\u001b[39m[ { … \u001b[90m\"\u001b[39m[ { \\\"n… en       data… \u001b[90m\"\u001b[39m{ \\\"…\n",
       "\u001b[90m# … with 5 more variables: created <date>, rights <chr>,\u001b[39m\n",
-      "\u001b[90m#   X_metadata.anytext <chr>, id <chr>, geometry <POLYGON [°]>\u001b[39m\n"
+      "\u001b[90m#   id <chr>, geometry <POLYGON [°]>\u001b[39m\n"
      ]
     }
    ],

docs/source/reference/quickstart.rst

@@ -8,7 +8,7 @@ It is the minimal runtime configuration profile as used in wis2box GitHub CI/CD:
 
 .. note:: wis2box web components are run on port 80 by default.  When using wis2box from source, the default port for web components is 8999, to be used for development.
 
-To download the wis2box from source: 
+To download wis2box from source:
 
 .. code-block:: bash
 

docs/source/reference/running/api-publishing.rst

@@ -25,14 +25,12 @@ will generate local station collection GeoJSON for API publication.
    wis2box metadata station publish-collection
 
 
-.. note:: run the command ``wis2box metadata station publish-collection`` to
-          publish your stations as a collection to the wis2box API
-
+.. note:: This command also runs automatically at startup and thereafter every 10 minutes
+          to keep your stations up to date.
 
 .. seealso:: :ref:`station-metadata`
 
 
-
 Discovery metadata API publishing
 ---------------------------------
 

nginx/nginx-ssl.conf

@@ -67,7 +67,9 @@
         location / {
             proxy_pass http://wis2box-ui:80;
         }
-
+#        location /admin/ {
+#            proxy_pass http://wis2box-ui-admin:80/;
+#        }
         location /auth {
             internal;
             proxy_pass http://wis2box-auth:80/authorize;

nginx/nginx.conf

@@ -64,7 +64,9 @@
         location / {
             proxy_pass http://wis2box-ui:80;
         }
-
+#        location /admin/ {
+#            proxy_pass http://wis2box-ui-admin:80/;
+#        }
         location /auth {
             internal;
             proxy_pass http://wis2box-auth:80/authorize;

wis2box-management/docker/entrypoint.sh

@@ -43,6 +43,7 @@ cd ~/.pywcmp/wis2-topic-hierarchy && unzip -j /tmp/all.json.zip
 wis2box environment create
 wis2box environment show
 wis2box api setup
+wis2box metadata discovery setup
 wis2box metadata station publish-collection
 echo "END /entrypoint.sh"
 exec "$@"

wis2box-management/requirements.txt

@@ -1,4 +1,4 @@
-elasticsearch<8
+elasticsearch
 isodate
 minio
 OWSLib

wis2box-management/wis2box/api/backend/elastic.py

@@ -19,7 +19,6 @@
 #
 ###############################################################################
 
-from copy import deepcopy
 import logging
 
 from elasticsearch import Elasticsearch, helpers
@@ -33,60 +32,59 @@
 
 # default index settings
 SETTINGS = {
-    'settings': {
-        'number_of_shards': 1,
-        'number_of_replicas': 0
-    },
-    'mappings': {
-        'properties': {
-            'geometry': {
-                'type': 'geo_shape'
-            },
-            'reportId': {
-                'type': 'text',
-                'fields': {
-                    'raw': {
-                        'type': 'keyword'
-                    }
+    'number_of_shards': 1,
+    'number_of_replicas': 0
+}
+
+MAPPINGS = {
+    'properties': {
+        'geometry': {
+            'type': 'geo_shape'
+        },
+        'reportId': {
+            'type': 'text',
+            'fields': {
+                'raw': {
+                    'type': 'keyword'
                 }
-            },
+            }
+        },
+        'properties': {
             'properties': {
-                'properties': {
-                    'resultTime': {
-                        'type': 'date',
-                        'fields': {
-                            'raw': {
-                                'type': 'keyword'
-                            }
+                'resultTime': {
+                    'type': 'date',
+                    'fields': {
+                        'raw': {
+                            'type': 'keyword'
                         }
-                    },
-                    'pubTime': {
-                        'type': 'date',
-                        'fields': {
-                            'raw': {
-                                'type': 'keyword'
-                            }
-                        }
-                    },
-                    'phenomenonTime': {
-                        'type': 'text'
-                    },
-                    'wigos_station_identifier': {
-                        'type': 'text',
-                        'fields': {
-                            'raw': {'type': 'keyword'}
+                    }
+                },
+                'pubTime': {
+                    'type': 'date',
+                    'fields': {
+                        'raw': {
+                            'type': 'keyword'
                         }
-                    },
-                    'value': {
-                        'type': 'float',
-                        'coerce': True
-                    },
-                    'metadata': {
-                        'properties': {
-                            'value': {
-                                'type': 'float',
-                                'coerce': True
-                            }
+                    }
+                },
+                'phenomenonTime': {
+                    'type': 'text'
+                },
+                'wigos_station_identifier': {
+                    'type': 'text',
+                    'fields': {
+                        'raw': {'type': 'keyword'}
+                    }
+                },
+                'value': {
+                    'type': 'float',
+                    'coerce': True
+                },
+                'metadata': {
+                    'properties': {
+                        'value': {
+                            'type': 'float',
+                            'coerce': True
                         }
                     }
                 }
@@ -111,7 +109,7 @@ def __init__(self, defs: dict) -> None:
         self.type = 'Elasticsearch'
         self.url = defs.get('url').rstrip('/')
 
-        self.conn = Elasticsearch([self.url], timeout=30,
+        self.conn = Elasticsearch(self.url, timeout=30,
                                   max_retries=10, retry_on_timeout=True)
 
     @staticmethod
@@ -140,10 +138,9 @@ def add_collection(self, collection_id: str) -> dict:
             LOGGER.error(msg)
             raise RuntimeError(msg)
 
-        settings = deepcopy(SETTINGS)
-
         LOGGER.debug('Creating index')
-        self.conn.indices.create(index=es_index, body=settings)
+        self.conn.options().indices.create(index=es_index, mappings=MAPPINGS,
+                                           settings=SETTINGS)
 
         return self.has_collection(collection_id)
 
@@ -162,7 +159,7 @@ def delete_collection(self, collection_id: str) -> bool:
             LOGGER.error(msg)
             raise RuntimeError(msg)
 
-        if self.conn.indices.exists(es_index):
+        if self.conn.indices.exists(index=es_index):
             self.conn.indices.delete(index=es_index)
 
         return not self.has_collection(collection_id)
@@ -178,7 +175,7 @@ def has_collection(self, collection_id: str) -> bool:
         es_index = self.es_id(collection_id)
         indices = self.conn.indices
 
-        return indices.exists(es_index)
+        return indices.exists(index=es_index)
 
     def upsert_collection_items(self, collection_id: str, items: list) -> str:
         """
@@ -267,7 +264,7 @@ def delete_collections_by_retention(self, days: int) -> bool:
 
         for index in indices:
             LOGGER.debug(f'deleting documents older than {days} days ({before})')  # noqa
-            self.conn.delete_by_query(index=index, body=query_by_date)
+            self.conn.delete_by_query(index=index, **query_by_date)
 
         return
 

wis2box-management/wis2box/api/config/pygeoapi.py

@@ -104,11 +104,17 @@ def prepare_collection(self, meta: dict) -> bool:
         :returns: `dict` of collection configuration
         """
 
+        editable = False
+
         if meta['id'] in ['discovery-metadata', 'messages', 'stations']:
             resource_id = meta['id']
         else:
             resource_id = meta['topic_hierarchy']
 
+        # TODO: uncomment once admin UI is implemented
+        # if meta['id'] in ['discovery-metadata', 'stations']:
+        #     editable = True
+
         LOGGER.debug(f'Resource id: {resource_id}')
 
         type_ = meta.get('type', 'feature')
@@ -131,6 +137,7 @@ def prepare_collection(self, meta: dict) -> bool:
             },
             'providers': [{
                 'type': type_,
+                'editable': editable,
                 'name': provider_name,
                 'data': f'{API_BACKEND_URL}/{resource_id}',
                 'id_field': meta.get('id_field'),

wis2box-management/wis2box/metadata/discovery.py

@@ -98,16 +98,6 @@ def generate(self, mcf: dict) -> str:
         LOGGER.debug('Generating OARec discovery metadata')
         record = WMOWCMP2OutputSchema().write(md, stringify=False)
 
-        anytext_bag = [
-            md['identification']['title'],
-            md['identification']['abstract']
-        ]
-
-        for k, v in md['identification']['keywords'].items():
-            anytext_bag.extend(v['keywords'])
-
-        record['properties']['_metadata-anytext'] = ' '.join(anytext_bag)
-
         return record
 
 
@@ -163,6 +153,16 @@ def discovery_metadata():
     pass
 
 
+@click.command()
+@click.pass_context
+@cli_helpers.OPTION_VERBOSITY
+def setup(ctx, verbosity):
+    """Initializes metadata repository"""
+
+    click.echo('Setting up discovery metadata repository')
+    setup_collection(meta=gcm())
+
+
 @click.command()
 @click.pass_context
 @cli_helpers.ARGUMENT_FILEPATH
@@ -198,4 +198,5 @@ def unpublish(ctx, identifier, verbosity):
 
 
 discovery_metadata.add_command(publish)
+discovery_metadata.add_command(setup)
 discovery_metadata.add_command(unpublish)