From 107c3e28c1a8b5dc6ce3972d23828116037682ee Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 12:07:11 +0100 Subject: [PATCH 1/8] Update to Nominatim 4.4 --- 4.4/Dockerfile | 156 ++++++++++++++++ 4.4/README.md | 246 ++++++++++++++++++++++++++ 4.4/conf.d/apache.conf | 13 ++ 4.4/conf.d/env | 6 + 4.4/conf.d/postgres-import.conf | 2 + 4.4/conf.d/postgres-tuning.conf | 10 ++ 4.4/config.sh | 62 +++++++ 4.4/contrib/docker-compose-planet.yml | 31 ++++ 4.4/contrib/docker-compose.yml | 19 ++ 4.4/example.md | 79 +++++++++ 4.4/init.sh | 141 +++++++++++++++ 4.4/start.sh | 77 ++++++++ 4.4/startapache.sh | 5 + 4.4/startpostgres.sh | 4 + 14 files changed, 851 insertions(+) create mode 100644 4.4/Dockerfile create mode 100644 4.4/README.md create mode 100644 4.4/conf.d/apache.conf create mode 100644 4.4/conf.d/env create mode 100644 4.4/conf.d/postgres-import.conf create mode 100644 4.4/conf.d/postgres-tuning.conf create mode 100755 4.4/config.sh create mode 100644 4.4/contrib/docker-compose-planet.yml create mode 100644 4.4/contrib/docker-compose.yml create mode 100644 4.4/example.md create mode 100755 4.4/init.sh create mode 100755 4.4/start.sh create mode 100755 4.4/startapache.sh create mode 100755 4.4/startpostgres.sh diff --git a/4.4/Dockerfile b/4.4/Dockerfile new file mode 100644 index 00000000..fc11891f --- /dev/null +++ b/4.4/Dockerfile @@ -0,0 +1,156 @@ +ARG NOMINATIM_VERSION=4.4.0 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} + +FROM ubuntu:jammy AS build + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 + +WORKDIR /app + +RUN true \ + # Do not start daemons after installation. + && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. + build-essential \ + g++ \ + cmake \ + libpq-dev \ + zlib1g-dev \ + libbz2-dev \ + libproj-dev \ + libexpat1-dev \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + liblua5.4-dev \ + nlohmann-json3-dev \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + # PHP and Apache 2. + php \ + php-intl \ + php-pgsql \ + php-cgi \ + apache2 \ + libapache2-mod-php \ + # Python 3. + python3-dev \ + python3-pip \ + python3-tidylib \ + python3-psycopg2 \ + python3-setuptools \ + python3-dotenv \ + python3-psutil \ + python3-jinja2 \ + python3-sqlalchemy \ + python3-asyncpg \ + python3-datrie \ + python3-icu \ + python3-argparse-manpage \ + # Misc. + git \ + curl \ + sudo \ + sshpass \ + openssh-client + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Osmium install to run continuous updates. +RUN pip3 install osmium + +# Nominatim install. +ARG NOMINATIM_VERSION +ARG USER_AGENT + +RUN true \ + && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && tar xf nominatim.tar.bz2 \ + && mkdir build \ + && cd build \ + && cmake ../Nominatim-$NOMINATIM_VERSION \ + && make -j`nproc` \ + && make install + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge \ + cpp-9 \ + gcc-9* \ + g++ \ + git \ + make \ + cmake* \ + llvm-10* \ + libc6-dev \ + linux-libc-dev \ + libclang-*-dev \ + build-essential \ + liblua*-dev \ + postgresql-server-dev-14 \ + nlohmann-json3-dev \ + && apt-get clean \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /app/src/.git \ + /var/lib/apt/lists/* \ + # Remove nominatim source and build directories + && rm /app/*.tar.bz2 \ + && rm -rf /app/build \ + && rm -rf /app/Nominatim-$NOMINATIM_VERSION + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled +COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 + +ENV PROJECT_DIR=/nominatim + +ARG USER_AGENT +ENV USER_AGENT=${USER_AGENT} + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD ["/app/start.sh"] diff --git a/4.4/README.md b/4.4/README.md new file mode 100644 index 00000000..ac5f4d7e --- /dev/null +++ b/4.4/README.md @@ -0,0 +1,246 @@ +# Nominatim Docker (Nominatim version 4.3) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Configuration Example](#configuration-example) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. + Check [https://download.geofabrik.de](https://download.geofabrik.de) + Since the download speed is restricted at Geofabrik, there is a recommended list of mirrors for importing the full planet at [OSM Wiki](https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors). + At the mirror sites you can find the folder /planet which contains the planet-latest.osm.pbf + and often a `/replication` folder for the `REPLICATION_URL`. +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. For example Geofabrik's update for the Europe extract are available at `https://download.geofabrik.de/europe-updates/` +Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CONTINENT/$COUNTRY-updates/` + +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: number of processing units available to the current process via `nproc`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.4.0/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +```sh +docker run -it \ + -v nominatim-flatnode:/nominatim/flatnode \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +### Configuration Example + +Here you can find a [configuration example](example.md) for all flags you can use for the container creation. + + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/14/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.0/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.3 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.0/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) +- [Upgrading Nominatim](https://github.com/mediagis/nominatim-docker/discussions/317) +- [Using Nominatim UI](https://github.com/mediagis/nominatim-docker/discussions/486#discussioncomment-7239861) + diff --git a/4.4/conf.d/apache.conf b/4.4/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.4/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.4/conf.d/env b/4.4/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.4/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.4/conf.d/postgres-import.conf b/4.4/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.4/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.4/conf.d/postgres-tuning.conf b/4.4/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..2b3c97f5 --- /dev/null +++ b/4.4/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.4/config.sh b/4.4/config.sh new file mode 100755 index 00000000..483354c9 --- /dev/null +++ b/4.4/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.4/contrib/docker-compose-planet.yml b/4.4/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..86455146 --- /dev/null +++ b/4.4/contrib/docker-compose-planet.yml @@ -0,0 +1,31 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.3 + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/14/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb diff --git a/4.4/contrib/docker-compose.yml b/4.4/contrib/docker-compose.yml new file mode 100644 index 00000000..7f3bc6c8 --- /dev/null +++ b/4.4/contrib/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.3 + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.3#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/14/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.4/example.md b/4.4/example.md new file mode 100644 index 00000000..bdf4f108 --- /dev/null +++ b/4.4/example.md @@ -0,0 +1,79 @@ +# Configuration Example + +```sh +docker run -it \ + #Sets the flatnode file, which is to reduce the load on the database when you plan to use multiple countrys together bigger than 6GB + #and highly recommended if you want to import the World! + -v nominatim-flatnode:/nominatim/flatnode \ + + #PostgreSQL Tuning, without the need to edit the .conf after the setup (Nominatim default recommended values) + -e POSTGRES_SHARED_BUFFERS=2GB \ + -e POSTGRES_MAINTAINENCE_WORK_MEM=10GB \ + -e POSTGRES_AUTOVACUUM_WORK_MEM=2GB \ + -e POSTGRES_WORK_MEM=50MB \ + -e POSTGRES_EFFECTIVE_CACHE_SIZE=24GB \ + -e POSTGRES_SYNCHRONOUS_COMMIT=off \ + -e POSTGRES_MAX_WAL_SIZE=1GB \ + -e POSTGRES_CHECKPOINT_TIMEOUT=10min \ + -e POSTGRES_CHECKPOINT_COMPLETITION_TARGET=0.9 \ + + #Sets the target for the initial file for the import. If the file is already on the local system you use: + #-e PBF_PATH=/path/to/your/planet-latest.osm.pbf PBF_URL cannot be used together with PBF_PATH! + -e PBF_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf \ + + #Sets the Path, where Nominatim gets the map updates - the REPLICATION_URL is never a file. + -e REPLICATION_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ \ + + #How often upstream publishes diffs (in seconds, default: 86400). Requires REPLICATION_URL to be set. + -e REPLICATION_UPDATE_INTERVAL=43200 + + #How long to sleep if no update found yet (in seconds, default: 900). Requires REPLICATION_URL to be set. + -e REPLICATION_RECHECK_INTERVAL=450 + + #Configures the way the map files will be updated (default: none) + -e UPDATE_MODE=continuous/once/catch-up/none + + #Disables the updates to save space for example (default: false) + -e FREEZE=true/false + + #If you only want to use the Nominatim database for reverse lookups. (default: false) + -e REVERSE_ONLY=true/false + + #When enabled additional Wikipedia Data will be loaded (default off) + -e IMPORT_WIKIPEDIA=true/false + + #Whether to download and import the US postcode dump (true) or path to US postcode dump in the container. (default: false) + -e IMPORT_US_POSTCODES=true/false/path + + #Whether to download and import the GB postcode dump (true) or path to GB postcode dump in the container. (default: false) + -e IMPORT_GB_POSTCODES=true/false/path + + #Sets either an importfilter for a reduced data import or the full set and the full set with additional data (default: full): + #admin: Only import administrative boundaries and places. + #street: Like the admin style but also adds streets. + #address: Import all data necessary to compute addresses down to house number level. + #full: Default style that also includes points of interest. + #extratags: Like the full style but also adds most of the OSM tags into the extratags column. + -e IMPORT_STYLE=admin/street/address/full/extratags + + #Whether to download and import the Tiger address data (true) or path to a preprocessed Tiger address set in the container. (default: false) + -e IMPORT_TIGER_ADDRESSES=true/false/path + + #Sets the used threads at the import (default 16) + -e THREADS=10 \ + + #Sets the Docker tmpfs. Highly recommended for bigger imports like Europe. At least 1GB - ideally half of the available RAM. + --shm-size=60g \ + + #The password to connect to the database with (default: qaIACxO6wMR3) + -e NOMINATIM_PASSWORD=supersafepassword + + #Sets the ports of the container guest:host + -p 8080:8080 \ + + #Sets the name of the container + --name nominatim \ + + #Here you choose the Docker image and version + mediagis/nominatim:4.3 +``` diff --git a/4.4/init.sh b/4.4/init.sh new file mode 100755 index 00000000..e3e32f30 --- /dev/null +++ b/4.4/init.sh @@ -0,0 +1,141 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") + +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + +# Check if THREADS is not set or is empty +if [ -z "$THREADS" ]; then + THREADS=$(nproc) +fi + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then + chown postgres /var/lib/postgresql/14/main + sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main +fi + +# temporarily enable unsafe import optimization config +cp /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled /etc/postgresql/14/main/conf.d/postgres-import.conf + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -H -E -u nominatim nominatim admin --warm --reverse +else + sudo -H -E -u nominatim nominatim admin --warm +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/14/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.4/start.sh b/4.4/start.sh new file mode 100755 index 00000000..c59a4c85 --- /dev/null +++ b/4.4/start.sh @@ -0,0 +1,77 @@ +#!/bin/bash -ex + +tailpid=0 +replicationpid=0 + +stopServices() { + service apache2 stop + service postgresql stop + kill $replicationpid + kill $tailpid +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + +# fork a process and wait for it +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -H -E -u nominatim nominatim admin --warm > /dev/null +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 +echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + +wait diff --git a/4.4/startapache.sh b/4.4/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.4/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.4/startpostgres.sh b/4.4/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.4/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log From 3603d20a86954953975b0d5b6bf41ff32d27a6bf Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 12:08:04 +0100 Subject: [PATCH 2/8] Upgrade versions in ci config --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 523da39b..011262f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,14 @@ jobs: strategy: matrix: nominatim: - - version: "4.2" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 - user_agent: mediagis/nominatim-docker-action:4.2 - version: "4.3" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 user_agent: mediagis/nominatim-docker-action:4.3 + - version: "4.4" + update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + postgres_version: 14 + user_agent: mediagis/nominatim-docker-action:4.4 runs-on: ubuntu-latest steps: From 3cf67a80bd8451f016b341ec055cb28561a30050 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 12:11:35 +0100 Subject: [PATCH 3/8] Add 4.4.0 to overview --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 22ab0ea6..d8dc3eda 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ See relevant installation and usage instructions for each version in the ``` Date: Mon, 18 Mar 2024 12:23:59 +0100 Subject: [PATCH 4/8] Reduce sleep time a bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 011262f3..7136f964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 180 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -139,7 +139,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 180 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log From 576c1efca63f0baae582d899696578607e4cc230 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 13:42:29 +0100 Subject: [PATCH 5/8] Lower sleep even more --- .github/workflows/assert-empty-json | 2 +- .github/workflows/assert-reverse-only | 2 +- .github/workflows/ci.yml | 24 +++++++++++------------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/assert-empty-json b/.github/workflows/assert-empty-json index a0b810f6..a43cbeb2 100755 --- a/.github/workflows/assert-empty-json +++ b/.github/workflows/assert-empty-json @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=7, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) +retries = Retry(total=18, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/assert-reverse-only b/.github/workflows/assert-reverse-only index 1d14168d..48876aae 100755 --- a/.github/workflows/assert-reverse-only +++ b/.github/workflows/assert-reverse-only @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=7, backoff_factor=2, status_forcelist=[502, 503, 504]) +retries = Retry(total=18, backoff_factor=2, status_forcelist=[502, 503, 504]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7136f964..fd7af116 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -64,7 +64,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount @@ -80,7 +80,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim @@ -104,7 +104,6 @@ jobs: docker stop nominatim - name: Check UPDATE_MODE=once with volume - if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from four days ago to make sure there really are updates to apply @@ -118,14 +117,13 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log docker stop nominatim - name: Check UPDATE_MODE=continuous with bind-mount - if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from few days ago to make sure there really are updates to apply @@ -139,7 +137,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -155,7 +153,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8005:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -168,7 +166,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8006:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -184,7 +182,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -197,7 +195,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8008:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -209,7 +207,7 @@ jobs: -e FREEZE="true" \ -p 8009:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: Check GB postcode import @@ -235,7 +233,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8011:8080 \ nominatim & - sleep 120 + sleep 60 ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" From 1a0c50174905355713eb901d4a5c6ecb3bb9b45f Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 13:45:23 +0100 Subject: [PATCH 6/8] Update documenation --- 4.4/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/4.4/README.md b/4.4/README.md index ac5f4d7e..5b8123c5 100644 --- a/4.4/README.md +++ b/4.4/README.md @@ -30,7 +30,7 @@ docker run -it \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. @@ -102,7 +102,7 @@ See https://nominatim.org/release-docs/4.4.0/admin/Import/#filtering-imported-da ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.3.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. ```sh docker run -it \ @@ -111,7 +111,7 @@ docker run -it \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` ### Configuration Example @@ -137,7 +137,7 @@ docker run -it --shm-size=1g \ -v nominatim-data:/var/lib/postgresql/14/main \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` ## OpenStreetMap Data Extracts @@ -160,7 +160,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ @@ -191,7 +191,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ @@ -207,7 +207,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/extras:/nominatim/extras \ --name nominatim \ - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. From eb0b99115be931330e7b6da0927cccb3bbfa3d5a Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 13:51:13 +0100 Subject: [PATCH 7/8] Apply suggestions from code review Co-authored-by: Philip Kozeny --- 4.4/README.md | 2 +- 4.4/contrib/docker-compose-planet.yml | 2 +- 4.4/contrib/docker-compose.yml | 2 +- 4.4/example.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/4.4/README.md b/4.4/README.md index 5b8123c5..0ac5b5bc 100644 --- a/4.4/README.md +++ b/4.4/README.md @@ -1,4 +1,4 @@ -# Nominatim Docker (Nominatim version 4.3) +# Nominatim Docker (Nominatim version 4.4) ## Table of contents diff --git a/4.4/contrib/docker-compose-planet.yml b/4.4/contrib/docker-compose-planet.yml index 86455146..8603adbd 100644 --- a/4.4/contrib/docker-compose-planet.yml +++ b/4.4/contrib/docker-compose-planet.yml @@ -10,7 +10,7 @@ version: "3" services: nominatim: container_name: nominatim - image: mediagis/nominatim:4.3 + image: mediagis/nominatim:4.4 ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.4/contrib/docker-compose.yml b/4.4/contrib/docker-compose.yml index 7f3bc6c8..cf648e18 100644 --- a/4.4/contrib/docker-compose.yml +++ b/4.4/contrib/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: nominatim: container_name: nominatim - image: mediagis/nominatim:4.3 + image: mediagis/nominatim:4.4 ports: - "8080:8080" environment: diff --git a/4.4/example.md b/4.4/example.md index bdf4f108..370b64a7 100644 --- a/4.4/example.md +++ b/4.4/example.md @@ -75,5 +75,5 @@ docker run -it \ --name nominatim \ #Here you choose the Docker image and version - mediagis/nominatim:4.3 + mediagis/nominatim:4.4 ``` From e6de6db37d4205dd2344c6fe7843b48d01365289 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 18 Mar 2024 14:43:33 +0100 Subject: [PATCH 8/8] Revert "Lower sleep even more" This reverts commit 576c1efca63f0baae582d899696578607e4cc230. --- .github/workflows/assert-empty-json | 2 +- .github/workflows/assert-reverse-only | 2 +- .github/workflows/ci.yml | 24 +++++++++++++----------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/assert-empty-json b/.github/workflows/assert-empty-json index a43cbeb2..a0b810f6 100755 --- a/.github/workflows/assert-empty-json +++ b/.github/workflows/assert-empty-json @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=18, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) +retries = Retry(total=7, backoff_factor=2, status_forcelist=[ 502, 503, 504 ]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/assert-reverse-only b/.github/workflows/assert-reverse-only index 48876aae..1d14168d 100755 --- a/.github/workflows/assert-reverse-only +++ b/.github/workflows/assert-reverse-only @@ -16,7 +16,7 @@ from requests.packages.urllib3.util.retry import Retry logging.basicConfig(level=logging.WARNING) s = requests.Session() -retries = Retry(total=18, backoff_factor=2, status_forcelist=[502, 503, 504]) +retries = Retry(total=7, backoff_factor=2, status_forcelist=[502, 503, 504]) s.mount('http://', HTTPAdapter(max_retries=retries)) resp = s.get(sys.argv[1]) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd7af116..7136f964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: -p 8001:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" ${{ matrix.nominatim.update_command }} ./assert-non-empty-json "http://localhost:8001/search.php?q=avenue%20pasteur" @@ -64,7 +64,7 @@ jobs: -v nominatim-data:/var/lib/postgresql/${{ matrix.nominatim.postgres_version }}/main \ -p 8002:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8002/search.php?q=avenue%20pasteur" - name: Check import with bind-mount @@ -80,7 +80,7 @@ jobs: -p 8003:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8003/search.php?q=avenue%20pasteur" docker stop nominatim @@ -104,6 +104,7 @@ jobs: docker stop nominatim - name: Check UPDATE_MODE=once with volume + if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from four days ago to make sure there really are updates to apply @@ -117,13 +118,14 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log docker stop nominatim - name: Check UPDATE_MODE=continuous with bind-mount + if: matrix.nominatim.version != '4.0' working-directory: .github/workflows run: |- # get the data from few days ago to make sure there really are updates to apply @@ -137,7 +139,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -153,7 +155,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8005:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8005/search.php?q=hotel%20de%20paris" - name: Check import admin style @@ -166,7 +168,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8006:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-empty-json "http://localhost:8006/search.php?q=hotel%20de%20paris" - name: Check import with PBF_PATH @@ -182,7 +184,7 @@ jobs: -p 8007:8080 \ --name nominatim \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8007/search.php?q=avenue%20pasteur" docker stop nominatim docker volume rm nominatim7-data @@ -195,7 +197,7 @@ jobs: -e USER_AGENT=${{matrix.nominatim.user_agent}} \ -p 8008:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8008/search.php?q=avenue%20pasteur" - name: Check when using FREEZE @@ -207,7 +209,7 @@ jobs: -e FREEZE="true" \ -p 8009:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-non-empty-json "http://localhost:8009/search.php?q=avenue%20pasteur" - name: Check GB postcode import @@ -233,7 +235,7 @@ jobs: -e REVERSE_ONLY="true" \ -p 8011:8080 \ nominatim & - sleep 60 + sleep 120 ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2"