diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 523da39b..7136f964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,14 @@ jobs: strategy: matrix: nominatim: - - version: "4.2" - update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once - postgres_version: 14 - user_agent: mediagis/nominatim-docker-action:4.2 - version: "4.3" update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once postgres_version: 14 user_agent: mediagis/nominatim-docker-action:4.3 + - version: "4.4" + update_command: docker exec -i nominatim sudo -u nominatim nominatim replication --project-dir /nominatim --once + postgres_version: 14 + user_agent: mediagis/nominatim-docker-action:4.4 runs-on: ubuntu-latest steps: @@ -118,7 +118,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 180 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log @@ -139,7 +139,7 @@ jobs: -p 8004:8080 \ --name nominatim \ nominatim & - sleep 180 + sleep 120 ./assert-non-empty-json "http://localhost:8004/search.php?q=avenue%20pasteur" echo "check replication log for Update completed. Count:" docker exec -i nominatim grep -c 'Update completed.' /var/log/replication.log diff --git a/4.4/Dockerfile b/4.4/Dockerfile new file mode 100644 index 00000000..fc11891f --- /dev/null +++ b/4.4/Dockerfile @@ -0,0 +1,156 @@ +ARG NOMINATIM_VERSION=4.4.0 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} + +FROM ubuntu:jammy AS build + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 + +WORKDIR /app + +RUN true \ + # Do not start daemons after installation. + && echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. + build-essential \ + g++ \ + cmake \ + libpq-dev \ + zlib1g-dev \ + libbz2-dev \ + libproj-dev \ + libexpat1-dev \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + liblua5.4-dev \ + nlohmann-json3-dev \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + # PHP and Apache 2. + php \ + php-intl \ + php-pgsql \ + php-cgi \ + apache2 \ + libapache2-mod-php \ + # Python 3. + python3-dev \ + python3-pip \ + python3-tidylib \ + python3-psycopg2 \ + python3-setuptools \ + python3-dotenv \ + python3-psutil \ + python3-jinja2 \ + python3-sqlalchemy \ + python3-asyncpg \ + python3-datrie \ + python3-icu \ + python3-argparse-manpage \ + # Misc. + git \ + curl \ + sudo \ + sshpass \ + openssh-client + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Osmium install to run continuous updates. +RUN pip3 install osmium + +# Nominatim install. +ARG NOMINATIM_VERSION +ARG USER_AGENT + +RUN true \ + && curl -A $USER_AGENT https://nominatim.org/release/Nominatim-$NOMINATIM_VERSION.tar.bz2 -o nominatim.tar.bz2 \ + && tar xf nominatim.tar.bz2 \ + && mkdir build \ + && cd build \ + && cmake ../Nominatim-$NOMINATIM_VERSION \ + && make -j`nproc` \ + && make install + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge \ + cpp-9 \ + gcc-9* \ + g++ \ + git \ + make \ + cmake* \ + llvm-10* \ + libc6-dev \ + linux-libc-dev \ + libclang-*-dev \ + build-essential \ + liblua*-dev \ + postgresql-server-dev-14 \ + nlohmann-json3-dev \ + && apt-get clean \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* \ + /root/.cache \ + /app/src/.git \ + /var/lib/apt/lists/* \ + # Remove nominatim source and build directories + && rm /app/*.tar.bz2 \ + && rm -rf /app/build \ + && rm -rf /app/Nominatim-$NOMINATIM_VERSION + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled +COPY conf.d/postgres-tuning.conf /etc/postgresql/14/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 + +ENV PROJECT_DIR=/nominatim + +ARG USER_AGENT +ENV USER_AGENT=${USER_AGENT} + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD ["/app/start.sh"] diff --git a/4.4/README.md b/4.4/README.md new file mode 100644 index 00000000..0ac5b5bc --- /dev/null +++ b/4.4/README.md @@ -0,0 +1,246 @@ +# Nominatim Docker (Nominatim version 4.4) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Configuration Example](#configuration-example) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. + Check [https://download.geofabrik.de](https://download.geofabrik.de) + Since the download speed is restricted at Geofabrik, there is a recommended list of mirrors for importing the full planet at [OSM Wiki](https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors). + At the mirror sites you can find the folder /planet which contains the planet-latest.osm.pbf + and often a `/replication` folder for the `REPLICATION_URL`. +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. For example Geofabrik's update for the Europe extract are available at `https://download.geofabrik.de/europe-updates/` +Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CONTINENT/$COUNTRY-updates/` + +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: number of processing units available to the current process via `nproc`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.4.0/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +```sh +docker run -it \ + -v nominatim-flatnode:/nominatim/flatnode \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +### Configuration Example + +Here you can find a [configuration example](example.md) for all flags you can use for the container creation. + + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/14/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/14/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.0/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.0/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) +- [Upgrading Nominatim](https://github.com/mediagis/nominatim-docker/discussions/317) +- [Using Nominatim UI](https://github.com/mediagis/nominatim-docker/discussions/486#discussioncomment-7239861) + diff --git a/4.4/conf.d/apache.conf b/4.4/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.4/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.4/conf.d/env b/4.4/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.4/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.4/conf.d/postgres-import.conf b/4.4/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.4/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.4/conf.d/postgres-tuning.conf b/4.4/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..2b3c97f5 --- /dev/null +++ b/4.4/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.4.0/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.4/config.sh b/4.4/config.sh new file mode 100755 index 00000000..483354c9 --- /dev/null +++ b/4.4/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/14/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.4/contrib/docker-compose-planet.yml b/4.4/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..8603adbd --- /dev/null +++ b/4.4/contrib/docker-compose-planet.yml @@ -0,0 +1,31 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.4 + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/14/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb diff --git a/4.4/contrib/docker-compose.yml b/4.4/contrib/docker-compose.yml new file mode 100644 index 00000000..cf648e18 --- /dev/null +++ b/4.4/contrib/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.4 + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.3#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/14/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.4/example.md b/4.4/example.md new file mode 100644 index 00000000..370b64a7 --- /dev/null +++ b/4.4/example.md @@ -0,0 +1,79 @@ +# Configuration Example + +```sh +docker run -it \ + #Sets the flatnode file, which is to reduce the load on the database when you plan to use multiple countrys together bigger than 6GB + #and highly recommended if you want to import the World! + -v nominatim-flatnode:/nominatim/flatnode \ + + #PostgreSQL Tuning, without the need to edit the .conf after the setup (Nominatim default recommended values) + -e POSTGRES_SHARED_BUFFERS=2GB \ + -e POSTGRES_MAINTAINENCE_WORK_MEM=10GB \ + -e POSTGRES_AUTOVACUUM_WORK_MEM=2GB \ + -e POSTGRES_WORK_MEM=50MB \ + -e POSTGRES_EFFECTIVE_CACHE_SIZE=24GB \ + -e POSTGRES_SYNCHRONOUS_COMMIT=off \ + -e POSTGRES_MAX_WAL_SIZE=1GB \ + -e POSTGRES_CHECKPOINT_TIMEOUT=10min \ + -e POSTGRES_CHECKPOINT_COMPLETITION_TARGET=0.9 \ + + #Sets the target for the initial file for the import. If the file is already on the local system you use: + #-e PBF_PATH=/path/to/your/planet-latest.osm.pbf PBF_URL cannot be used together with PBF_PATH! + -e PBF_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf \ + + #Sets the Path, where Nominatim gets the map updates - the REPLICATION_URL is never a file. + -e REPLICATION_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ \ + + #How often upstream publishes diffs (in seconds, default: 86400). Requires REPLICATION_URL to be set. + -e REPLICATION_UPDATE_INTERVAL=43200 + + #How long to sleep if no update found yet (in seconds, default: 900). Requires REPLICATION_URL to be set. + -e REPLICATION_RECHECK_INTERVAL=450 + + #Configures the way the map files will be updated (default: none) + -e UPDATE_MODE=continuous/once/catch-up/none + + #Disables the updates to save space for example (default: false) + -e FREEZE=true/false + + #If you only want to use the Nominatim database for reverse lookups. (default: false) + -e REVERSE_ONLY=true/false + + #When enabled additional Wikipedia Data will be loaded (default off) + -e IMPORT_WIKIPEDIA=true/false + + #Whether to download and import the US postcode dump (true) or path to US postcode dump in the container. (default: false) + -e IMPORT_US_POSTCODES=true/false/path + + #Whether to download and import the GB postcode dump (true) or path to GB postcode dump in the container. (default: false) + -e IMPORT_GB_POSTCODES=true/false/path + + #Sets either an importfilter for a reduced data import or the full set and the full set with additional data (default: full): + #admin: Only import administrative boundaries and places. + #street: Like the admin style but also adds streets. + #address: Import all data necessary to compute addresses down to house number level. + #full: Default style that also includes points of interest. + #extratags: Like the full style but also adds most of the OSM tags into the extratags column. + -e IMPORT_STYLE=admin/street/address/full/extratags + + #Whether to download and import the Tiger address data (true) or path to a preprocessed Tiger address set in the container. (default: false) + -e IMPORT_TIGER_ADDRESSES=true/false/path + + #Sets the used threads at the import (default 16) + -e THREADS=10 \ + + #Sets the Docker tmpfs. Highly recommended for bigger imports like Europe. At least 1GB - ideally half of the available RAM. + --shm-size=60g \ + + #The password to connect to the database with (default: qaIACxO6wMR3) + -e NOMINATIM_PASSWORD=supersafepassword + + #Sets the ports of the container guest:host + -p 8080:8080 \ + + #Sets the name of the container + --name nominatim \ + + #Here you choose the Docker image and version + mediagis/nominatim:4.4 +``` diff --git a/4.4/init.sh b/4.4/init.sh new file mode 100755 index 00000000..e3e32f30 --- /dev/null +++ b/4.4/init.sh @@ -0,0 +1,141 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") + +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + +# Check if THREADS is not set or is empty +if [ -z "$THREADS" ]; then + THREADS=$(nproc) +fi + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + ${SCP}:tiger2021-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then + chown postgres /var/lib/postgresql/14/main + sudo -u postgres /usr/lib/postgresql/14/bin/initdb -D /var/lib/postgresql/14/main +fi + +# temporarily enable unsafe import optimization config +cp /etc/postgresql/14/main/conf.d/postgres-import.conf.disabled /etc/postgresql/14/main/conf.d/postgres-import.conf + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -H -E -u nominatim nominatim admin --warm --reverse +else + sudo -H -E -u nominatim nominatim admin --warm +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/14/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.4/start.sh b/4.4/start.sh new file mode 100755 index 00000000..c59a4c85 --- /dev/null +++ b/4.4/start.sh @@ -0,0 +1,77 @@ +#!/bin/bash -ex + +tailpid=0 +replicationpid=0 + +stopServices() { + service apache2 stop + service postgresql stop + kill $replicationpid + kill $tailpid +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/14/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + +# fork a process and wait for it +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -H -E -u nominatim nominatim admin --warm > /dev/null +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 +echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + +wait diff --git a/4.4/startapache.sh b/4.4/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.4/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.4/startpostgres.sh b/4.4/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.4/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log diff --git a/README.md b/README.md index 22ab0ea6..d8dc3eda 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ See relevant installation and usage instructions for each version in the ```