From 86f1d3494b29b98d7486ae70b3c79c9448cf44e0 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 14:55:59 +0100 Subject: [PATCH 01/26] Update build process for Nominatim 4.5 --- 4.5/Dockerfile | 111 ++++++++++++ 4.5/README.md | 246 ++++++++++++++++++++++++++ 4.5/conf.d/apache.conf | 13 ++ 4.5/conf.d/env | 6 + 4.5/conf.d/postgres-import.conf | 2 + 4.5/conf.d/postgres-tuning.conf | 10 ++ 4.5/config.sh | 62 +++++++ 4.5/contrib/docker-compose-planet.yml | 31 ++++ 4.5/contrib/docker-compose.yml | 19 ++ 4.5/example.md | 79 +++++++++ 4.5/init.sh | 145 +++++++++++++++ 4.5/start.sh | 83 +++++++++ 4.5/startapache.sh | 5 + 4.5/startpostgres.sh | 4 + 14 files changed, 816 insertions(+) create mode 100644 4.5/Dockerfile create mode 100644 4.5/README.md create mode 100644 4.5/conf.d/apache.conf create mode 100644 4.5/conf.d/env create mode 100644 4.5/conf.d/postgres-import.conf create mode 100644 4.5/conf.d/postgres-tuning.conf create mode 100755 4.5/config.sh create mode 100644 4.5/contrib/docker-compose-planet.yml create mode 100644 4.5/contrib/docker-compose.yml create mode 100644 4.5/example.md create mode 100755 4.5/init.sh create mode 100755 4.5/start.sh create mode 100755 4.5/startapache.sh create mode 100755 4.5/startpostgres.sh diff --git a/4.5/Dockerfile b/4.5/Dockerfile new file mode 100644 index 00000000..fcc23be8 --- /dev/null +++ b/4.5/Dockerfile @@ -0,0 +1,111 @@ +ARG NOMINATIM_VERSION=4.5.0 +ARG USER_AGENT=mediagis/nominatim-docker:${NOMINATIM_VERSION} + +FROM ubuntu:24.04 AS build + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 + +WORKDIR /app + +# Inspired by https://github.com/reproducible-containers/buildkit-cache-dance?tab=readme-ov-file#apt-get-github-actions +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + # Keep downloaded APT packages in the docker build cache + rm -f /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \ + # Do not start daemons after installation. + echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + # Install all required packages. + && apt-get -y update -qq \ + && apt-get -y install \ + locales \ + && locale-gen en_US.UTF-8 \ + && update-locale LANG=en_US.UTF-8 \ + && apt-get -y install \ + -o APT::Install-Recommends="false" \ + -o APT::Install-Suggests="false" \ + # Build tools from sources. \ + build-essential \ + python3-dev \ + osm2pgsql \ + pkg-config \ + libicu-dev python3-pip \ + # PostgreSQL. + postgresql-contrib \ + postgresql-server-dev-16 \ + postgresql-16-postgis-3 \ + postgresql-16-postgis-3-scripts \ + # Misc. + curl \ + sudo \ + sshpass \ + openssh-client + + +# Configure postgres. +RUN true \ + && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/16/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/16/main/postgresql.conf + +ARG NOMINATIM_VERSION +ARG USER_AGENT + +# Osmium install to run continuous updates. +#RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ +RUN pip install --break-system-packages \ + nominatim-db==$NOMINATIM_VERSION \ + osmium \ + psycopg[binary] \ + falcon \ + uvicorn \ + gunicorn \ + nominatim-api + +# Nominatim install. + +RUN true \ + # Remove development and unused packages. + && apt-get -y remove --purge --auto-remove \ + postgresql-server-dev-16 \ + # Clear temporary files and directories. + && rm -rf \ + /tmp/* \ + /var/tmp/* + +# Apache configuration +COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf + +# Postgres config overrides to improve import performance (but reduce crash recovery safety) +COPY conf.d/postgres-import.conf /etc/postgresql/16/main/conf.d/postgres-import.conf.disabled +COPY conf.d/postgres-tuning.conf /etc/postgresql/16/main/conf.d/ + +COPY config.sh /app/config.sh +COPY init.sh /app/init.sh +COPY start.sh /app/start.sh +COPY startapache.sh /app/startapache.sh +COPY startpostgres.sh /app/startpostgres.sh + +# Collapse image to single layer. +FROM scratch + +COPY --from=build / / + +# Please override this +ENV NOMINATIM_PASSWORD=qaIACxO6wMR3 + +ENV PROJECT_DIR=/nominatim + +ARG USER_AGENT +ENV USER_AGENT=${USER_AGENT} + +WORKDIR /app + +EXPOSE 5432 +EXPOSE 8080 + +COPY conf.d/env $PROJECT_DIR/.env + +CMD ["/app/start.sh"] diff --git a/4.5/README.md b/4.5/README.md new file mode 100644 index 00000000..506e1d74 --- /dev/null +++ b/4.5/README.md @@ -0,0 +1,246 @@ +# Nominatim Docker (Nominatim version 4.4) + +## Table of contents + + - [Automatic import](#automatic-import) + - [Configuration](#configuration) + - [General Parameters](#general-parameters) + - [PostgreSQL Tuning](#postgresql-tuning) + - [Import Style](#import-style) + - [Flatnode files](#flatnode-files) + - [Configuration Example](#configuration-example) + - [Persistent container data](#persistent-container-data) + - [OpenStreetMap Data Extracts](#openstreetmap-data-extracts) + - [Updating the database](#updating-the-database) + - [Custom PBF Files](#custom-pbf-files) + - [Importance Dumps, Postcode Data, and Tiger Addresses](#importance-dumps-postcode-data-and-tiger-addresses) + - [Development](#development) + - [Docker Compose](#docker-compose) + - [Assorted use cases documented in issues](#assorted-use-cases-documented-in-issues) + +--- + +## Automatic import + +Download the required data, initialize the database and start nominatim in one go + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. + +If you want to check that your data import was successful, you can use the API with the following URL: http://localhost:8080/search.php?q=avenue%20pasteur + +## Configuration + +### General Parameters + +The following environment variables are available for configuration: + +- `PBF_URL`: Which [OSM extract](#openstreetmap-data-extracts) to download and import. It cannot be used together with `PBF_PATH`. + Check [https://download.geofabrik.de](https://download.geofabrik.de) + Since the download speed is restricted at Geofabrik, there is a recommended list of mirrors for importing the full planet at [OSM Wiki](https://wiki.openstreetmap.org/wiki/Planet.osm#Planet.osm_mirrors). + At the mirror sites you can find the folder /planet which contains the planet-latest.osm.pbf + and often a `/replication` folder for the `REPLICATION_URL`. +- `PBF_PATH`: Which [OSM extract](#openstreetmap-data-extracts) to import from the .pbf file inside the container. It cannot be used together with `PBF_URL`. +- `REPLICATION_URL`: Where to get updates from. For example Geofabrik's update for the Europe extract are available at `https://download.geofabrik.de/europe-updates/` +Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CONTINENT/$COUNTRY-updates/` + +- `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ +- `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) +- `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) +- `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) +- `IMPORT_US_POSTCODES`: Whether to download and import the US postcode dump (`true`) or path to US postcode dump in the container. (default: `false`) +- `IMPORT_GB_POSTCODES`: Whether to download and import the GB postcode dump (`true`) or path to GB postcode dump in the container. (default: `false`) +- `IMPORT_TIGER_ADDRESSES`: Whether to download and import the Tiger address data (`true`) or path to a preprocessed Tiger address set in the container. (default: `false`) +- `THREADS`: How many threads should be used to import (default: number of processing units available to the current process via `nproc`) +- `NOMINATIM_PASSWORD`: The password to connect to the database with (default: `qaIACxO6wMR3`) + +The following run parameters are available for configuration: + +- `shm-size`: Size of the tmpfs in Docker, for bigger imports (e.g. Europe) this needs to be set to at least 1GB or more. Half the size of your available RAM is recommended. (default: `64M`) + +### PostgreSQL Tuning + +The following environment variables are available to tune PostgreSQL: + +- `POSTGRES_SHARED_BUFFERS` (default: `2GB`) +- `POSTGRES_MAINTENANCE_WORK_MEM` (default: `10GB`) +- `POSTGRES_AUTOVACUUM_WORK_MEM` (default: `2GB`) +- `POSTGRES_WORK_MEM` (default: `50MB`) +- `POSTGRES_EFFECTIVE_CACHE_SIZE` (default: `24GB`) +- `POSTGRES_SYNCHRONOUS_COMMIT` (default: `off`) +- `POSTGRES_MAX_WAL_SIZE` (default: `1GB`) +- `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) +- `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) + +See https://nominatim.org/release-docs/4.4.1/admin/Installation/#tuning-the-postgresql-database for more details on those settings. + +### Import Style + +The import style can be modified through an environment variable : + +- `IMPORT_STYLE` (default: `full`) + +Available options are : + +- `admin`: Only import administrative boundaries and places. +- `street`: Like the admin style but also adds streets. +- `address`: Import all data necessary to compute addresses down to house number level. +- `full`: Default style that also includes points of interest. +- `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. + +See https://nominatim.org/release-docs/4.4.1/admin/Import/#filtering-imported-data for more details on those styles. + +### Flatnode files + +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.1/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. + +```sh +docker run -it \ + -v nominatim-flatnode:/nominatim/flatnode \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +### Configuration Example + +Here you can find a [configuration example](example.md) for all flags you can use for the container creation. + + +## Persistent container data + +If you want to keep your imported data across deletion and recreation of your container, make the following folder a volume: + +- `/var/lib/postgresql/16/main` is the storage location of the Postgres database & holds the state about whether the import was successful +- `/nominatim/flatnode` is the storage location of the flatnode file. + +So if you want to be able to kill your container and start it up again with all the data still present use the following command: + +```sh +docker run -it --shm-size=1g \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -e IMPORT_WIKIPEDIA=false \ + -e NOMINATIM_PASSWORD=very_secure_password \ + -v nominatim-data:/var/lib/postgresql/16/main \ + -p 8080:8080 \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +## OpenStreetMap Data Extracts + +Nominatim imports OpenStreetMap (OSM) data extracts. The source of the data can be specified with one of the following environment variables: + +- `PBF_URL` variable specifies the URL. The data is downloaded during initialization, imported and removed from disk afterwards. The data extracts can be freely downloaded, e.g., from [Geofabrik's server](https://download.geofabrik.de). +- `PBF_PATH` variable specifies the path to the mounted OSM extracts data inside the container. No .pbf file is removed after initialization. + +It is not possible to define both `PBF_URL` and `PBF_PATH` sources. + +The replication update can be performed only via HTTP. + +A sample of `PBF_PATH` variable usage is: + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ + +## Updating the database + +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.1/admin/Update/). For a list of other methods see the output of: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --help +``` + +The following command will keep updating the database forever: + +```sh +docker exec -it nominatim sudo -u nominatim nominatim replication --project-dir /nominatim +``` + +If there are no updates available this process will sleep for 15 minutes and try again. + +## Custom PBF Files + +If you want your Nominatim container to host multiple areas from Geofabrik, you can use a tool, such as [Osmium](https://osmcode.org/osmium-tool/manual.html), to merge multiple PBF files into one. + +```sh +docker run -it \ + -e PBF_PATH=/nominatim/data/merged.osm.pbf \ + -p 8080:8080 \ + -v /osm-maps/data:/nominatim/data \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ + +## Importance Dumps, Postcode Data, and Tiger Addresses + +Including the Wikipedia importance dumps, postcode files, and Tiger address data can improve results. These can be automatically downloaded by setting the appropriate options (see above) to `true`. Alternatively, they can be imported from local files by specifying a file path (relative to the container), similar to how `PBF_PATH` is used. For example: + +```sh +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e IMPORT_WIKIPEDIA=/nominatim/extras/wikimedia-importance.sql.gz \ + -p 8080:8080 \ + -v /osm-maps/extras:/nominatim/extras \ + --name nominatim \ + mediagis/nominatim:4.4 +``` + +Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. + +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.1/customize/Tiger/). + +## Development + +If you want to work on the Docker image you can use the following command to build a local +image and run the container with + +```sh +docker build -t nominatim . && \ +docker run -it \ + -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf \ + -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ + -p 8080:8080 \ + --name nominatim \ + nominatim +``` + +## Docker Compose + +In addition, we also provide a basic `contrib/docker-compose.yml` template which you use as a starting point and adapt to your needs. Use this template to set the environment variables, mounts, etc. as needed. + +Besides the basic docker-compose.yml, there are also some advanced YAML configurations available in the `contrib` folder. +These files follow the naming convention of `docker-compose-*.yml` and contain comments about the specific use case. + +## Assorted use cases documented in issues + +- [Using an external Postgres database](https://github.com/mediagis/nominatim-docker/issues/245#issuecomment-1072205751) + - [Using Amazon's RDS](https://github.com/mediagis/nominatim-docker/issues/378#issuecomment-1278653770) +- [Hardware sizing for importing the entire planet](https://github.com/mediagis/nominatim-docker/discussions/265) +- [Upgrading Nominatim](https://github.com/mediagis/nominatim-docker/discussions/317) +- [Using Nominatim UI](https://github.com/mediagis/nominatim-docker/discussions/486#discussioncomment-7239861) + diff --git a/4.5/conf.d/apache.conf b/4.5/conf.d/apache.conf new file mode 100644 index 00000000..922fc296 --- /dev/null +++ b/4.5/conf.d/apache.conf @@ -0,0 +1,13 @@ +Listen 8080 + + DocumentRoot /nominatim/website + CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined + ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" + LogLevel info + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + AddType text/html .php + diff --git a/4.5/conf.d/env b/4.5/conf.d/env new file mode 100644 index 00000000..16cc1226 --- /dev/null +++ b/4.5/conf.d/env @@ -0,0 +1,6 @@ +NOMINATIM_TOKENIZER=icu +NOMINATIM_REPLICATION_URL=__REPLICATION_URL__ +NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 +NOMINATIM_IMPORT_STYLE=__IMPORT_STYLE__ +NOMINATIM_FLATNODE_FILE= diff --git a/4.5/conf.d/postgres-import.conf b/4.5/conf.d/postgres-import.conf new file mode 100644 index 00000000..555067e8 --- /dev/null +++ b/4.5/conf.d/postgres-import.conf @@ -0,0 +1,2 @@ +fsync = off +full_page_writes = off diff --git a/4.5/conf.d/postgres-tuning.conf b/4.5/conf.d/postgres-tuning.conf new file mode 100644 index 00000000..4bf5b4b1 --- /dev/null +++ b/4.5/conf.d/postgres-tuning.conf @@ -0,0 +1,10 @@ +# See https://nominatim.org/release-docs/4.4.1/admin/Installation/#tuning-the-postgresql-database +shared_buffers = 2GB +maintenance_work_mem = 10GB +autovacuum_work_mem = 2GB +work_mem = 50MB +effective_cache_size = 24GB +synchronous_commit = off +max_wal_size = 1GB +checkpoint_timeout = 10min +checkpoint_completion_target = 0.9 diff --git a/4.5/config.sh b/4.5/config.sh new file mode 100755 index 00000000..48b5f219 --- /dev/null +++ b/4.5/config.sh @@ -0,0 +1,62 @@ +CONFIG_FILE=${PROJECT_DIR}/.env + + +if [[ "$PBF_URL" = "" && "$PBF_PATH" = "" ]] || [[ "$PBF_URL" != "" && "$PBF_PATH" != "" ]]; then + echo "You need to specify either the PBF_URL or PBF_PATH environment variable" + echo "docker run -e PBF_URL=https://download.geofabrik.de/europe/monaco-latest.osm.pbf ..." + echo "docker run -e PBF_PATH=/nominatim/data/monaco-latest.osm.pbf ..." + exit 1 +fi + +if [ "$REPLICATION_URL" != "" ]; then + sed -i "s|__REPLICATION_URL__|$REPLICATION_URL|g" ${CONFIG_FILE} +fi + +# Use the specified replication update and recheck interval values if either or both are numbers, or use the default values + +reg_num='^[0-9]+$' +if [[ $REPLICATION_UPDATE_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_UPDATE_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400/NOMINATIM_REPLICATION_UPDATE_INTERVAL=$REPLICATION_UPDATE_INTERVAL/g" ${CONFIG_FILE} +fi +if [[ $REPLICATION_RECHECK_INTERVAL =~ $reg_num ]]; then + if [ "$REPLICATION_URL" = "" ]; then + echo "You need to specify the REPLICATION_URL variable in order to set a REPLICATION_RECHECK_INTERVAL" + exit 1 + fi + sed -i "s/NOMINATIM_REPLICATION_RECHECK_INTERVAL=900/NOMINATIM_REPLICATION_RECHECK_INTERVAL=$REPLICATION_RECHECK_INTERVAL/g" ${CONFIG_FILE} +fi + +# PostgreSQL Tuning + +if [ ! -z "$POSTGRES_SHARED_BUFFERS" ]; then sed -i "s/shared_buffers = 2GB/shared_buffers = $POSTGRES_SHARED_BUFFERS/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAINTENANCE_WORK_MEM" ]; then sed -i "s/maintenance_work_mem = 10GB/maintenance_work_mem = $POSTGRES_MAINTENANCE_WORK_MEM/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_AUTOVACUUM_WORK_MEM" ]; then sed -i "s/autovacuum_work_mem = 2GB/autovacuum_work_mem = $POSTGRES_AUTOVACUUM_WORK_MEM/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_WORK_MEM" ]; then sed -i "s/work_mem = 50MB/work_mem = $POSTGRES_WORK_MEM/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_EFFECTIVE_CACHE_SIZE" ]; then sed -i "s/effective_cache_size = 24GB/effective_cache_size = $POSTGRES_EFFECTIVE_CACHE_SIZE/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_SYNCHRONOUS_COMMIT" ]; then sed -i "s/synchronous_commit = off/synchronous_commit = $POSTGRES_SYNCHRONOUS_COMMIT/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_MAX_WAL_SIZE" ]; then sed -i "s/max_wal_size = 1GB/max_wal_size = $POSTGRES_MAX_WAL_SIZE/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_TIMEOUT" ]; then sed -i "s/checkpoint_timeout = 10min/checkpoint_timeout = $POSTGRES_CHECKPOINT_TIMEOUT/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi +if [ ! -z "$POSTGRES_CHECKPOINT_COMPLETION_TARGET" ]; then sed -i "s/checkpoint_completion_target = 0.9/checkpoint_completion_target = $POSTGRES_CHECKPOINT_COMPLETION_TARGET/g" /etc/postgresql/16/main/conf.d/postgres-tuning.conf; fi + + +# import style tuning + +if [ ! -z "$IMPORT_STYLE" ]; then + sed -i "s|__IMPORT_STYLE__|${IMPORT_STYLE}|g" ${CONFIG_FILE} +else + sed -i "s|__IMPORT_STYLE__|full|g" ${CONFIG_FILE} +fi + +# if flatnode directory was created by volume / mount, use flatnode files + +if [ -d "${PROJECT_DIR}/flatnode" ]; then sed -i 's\^NOMINATIM_FLATNODE_FILE=$\NOMINATIM_FLATNODE_FILE="/nominatim/flatnode/flatnode.file"\g' ${CONFIG_FILE}; fi + +# enable use of optional TIGER address data + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ] || [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + echo NOMINATIM_USE_US_TIGER_DATA=yes >> ${CONFIG_FILE} +fi diff --git a/4.5/contrib/docker-compose-planet.yml b/4.5/contrib/docker-compose-planet.yml new file mode 100644 index 00000000..c2394b7a --- /dev/null +++ b/4.5/contrib/docker-compose-planet.yml @@ -0,0 +1,31 @@ +version: "3" + +# For a full planet instance, we apply some best practices from the documentation (like using a flatnode file). +# This compose file uses bind mounts, so it'll reference /data. You can either also mount the storage (for DB + flat node file) under /data or change it. +# Minimum specification for a machine running the import (if this succeeded on a lower spec machine, please contribute them): +# - 16 core CPU (set THREADS variable to number of cores/threads available) +# - 64GB RAM +# - 1.5TB (NVMe) SSD storage + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.4 + ports: + - "8080:8080" # Do not change the second port, only the first before the colon + environment: + PBF_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf + REPLICATION_URL: https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ + NOMINATIM_PASSWORD: very_secure_password + IMPORT_WIKIPEDIA: "true" + IMPORT_US_POSTCODES: "true" + IMPORT_GB_POSTCODES: "true" + THREADS: 16 + volumes: + - type: bind + source: /data/db + target: /var/lib/postgresql/16/main + - type: bind + source: /data/flatnode + target: /nominatim/flatnode + shm_size: 1gb diff --git a/4.5/contrib/docker-compose.yml b/4.5/contrib/docker-compose.yml new file mode 100644 index 00000000..59e53254 --- /dev/null +++ b/4.5/contrib/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + nominatim: + container_name: nominatim + image: mediagis/nominatim:4.4 + ports: + - "8080:8080" + environment: + # see https://github.com/mediagis/nominatim-docker/tree/master/4.4#configuration for more options + PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf + REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ + NOMINATIM_PASSWORD: very_secure_password + volumes: + - nominatim-data:/var/lib/postgresql/16/main + shm_size: 1gb + +volumes: + nominatim-data: diff --git a/4.5/example.md b/4.5/example.md new file mode 100644 index 00000000..370b64a7 --- /dev/null +++ b/4.5/example.md @@ -0,0 +1,79 @@ +# Configuration Example + +```sh +docker run -it \ + #Sets the flatnode file, which is to reduce the load on the database when you plan to use multiple countrys together bigger than 6GB + #and highly recommended if you want to import the World! + -v nominatim-flatnode:/nominatim/flatnode \ + + #PostgreSQL Tuning, without the need to edit the .conf after the setup (Nominatim default recommended values) + -e POSTGRES_SHARED_BUFFERS=2GB \ + -e POSTGRES_MAINTAINENCE_WORK_MEM=10GB \ + -e POSTGRES_AUTOVACUUM_WORK_MEM=2GB \ + -e POSTGRES_WORK_MEM=50MB \ + -e POSTGRES_EFFECTIVE_CACHE_SIZE=24GB \ + -e POSTGRES_SYNCHRONOUS_COMMIT=off \ + -e POSTGRES_MAX_WAL_SIZE=1GB \ + -e POSTGRES_CHECKPOINT_TIMEOUT=10min \ + -e POSTGRES_CHECKPOINT_COMPLETITION_TARGET=0.9 \ + + #Sets the target for the initial file for the import. If the file is already on the local system you use: + #-e PBF_PATH=/path/to/your/planet-latest.osm.pbf PBF_URL cannot be used together with PBF_PATH! + -e PBF_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/pbf/planet-latest.osm.pbf \ + + #Sets the Path, where Nominatim gets the map updates - the REPLICATION_URL is never a file. + -e REPLICATION_URL=https://ftp5.gwdg.de/pub/misc/openstreetmap/planet.openstreetmap.org/replication/day/ \ + + #How often upstream publishes diffs (in seconds, default: 86400). Requires REPLICATION_URL to be set. + -e REPLICATION_UPDATE_INTERVAL=43200 + + #How long to sleep if no update found yet (in seconds, default: 900). Requires REPLICATION_URL to be set. + -e REPLICATION_RECHECK_INTERVAL=450 + + #Configures the way the map files will be updated (default: none) + -e UPDATE_MODE=continuous/once/catch-up/none + + #Disables the updates to save space for example (default: false) + -e FREEZE=true/false + + #If you only want to use the Nominatim database for reverse lookups. (default: false) + -e REVERSE_ONLY=true/false + + #When enabled additional Wikipedia Data will be loaded (default off) + -e IMPORT_WIKIPEDIA=true/false + + #Whether to download and import the US postcode dump (true) or path to US postcode dump in the container. (default: false) + -e IMPORT_US_POSTCODES=true/false/path + + #Whether to download and import the GB postcode dump (true) or path to GB postcode dump in the container. (default: false) + -e IMPORT_GB_POSTCODES=true/false/path + + #Sets either an importfilter for a reduced data import or the full set and the full set with additional data (default: full): + #admin: Only import administrative boundaries and places. + #street: Like the admin style but also adds streets. + #address: Import all data necessary to compute addresses down to house number level. + #full: Default style that also includes points of interest. + #extratags: Like the full style but also adds most of the OSM tags into the extratags column. + -e IMPORT_STYLE=admin/street/address/full/extratags + + #Whether to download and import the Tiger address data (true) or path to a preprocessed Tiger address set in the container. (default: false) + -e IMPORT_TIGER_ADDRESSES=true/false/path + + #Sets the used threads at the import (default 16) + -e THREADS=10 \ + + #Sets the Docker tmpfs. Highly recommended for bigger imports like Europe. At least 1GB - ideally half of the available RAM. + --shm-size=60g \ + + #The password to connect to the database with (default: qaIACxO6wMR3) + -e NOMINATIM_PASSWORD=supersafepassword + + #Sets the ports of the container guest:host + -p 8080:8080 \ + + #Sets the name of the container + --name nominatim \ + + #Here you choose the Docker image and version + mediagis/nominatim:4.4 +``` diff --git a/4.5/init.sh b/4.5/init.sh new file mode 100755 index 00000000..51d19963 --- /dev/null +++ b/4.5/init.sh @@ -0,0 +1,145 @@ +#!/bin/bash -ex + +OSMFILE=${PROJECT_DIR}/data.osm.pbf + +CURL=("curl" "-L" "-A" "${USER_AGENT}" "--fail-with-body") + +SCP='sshpass -p DMg5bmLPY7npHL2Q scp -o StrictHostKeyChecking=no u355874-sub1@u355874-sub1.your-storagebox.de' + +# Check if THREADS is not set or is empty +if [ -z "$THREADS" ]; then + THREADS=$(nproc) +fi + +# we re-host the files on a Hetzner storage box because inconsiderate users eat up all of +# nominatim.org's bandwidth +# https://github.com/mediagis/nominatim-docker/issues/416 + +if [ "$IMPORT_WIKIPEDIA" = "true" ]; then + echo "Downloading Wikipedia importance dump" + ${SCP}:wikimedia-importance.sql.gz ${PROJECT_DIR}/wikimedia-importance.sql.gz +elif [ -f "$IMPORT_WIKIPEDIA" ]; then + # use local file if asked + ln -s "$IMPORT_WIKIPEDIA" ${PROJECT_DIR}/wikimedia-importance.sql.gz +else + echo "Skipping optional Wikipedia importance import" +fi; + +if [ "$IMPORT_GB_POSTCODES" = "true" ]; then + ${SCP}:gb_postcodes.csv.gz ${PROJECT_DIR}/gb_postcodes.csv.gz +elif [ -f "$IMPORT_GB_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_GB_POSTCODES" ${PROJECT_DIR}/gb_postcodes.csv.gz +else \ + echo "Skipping optional GB postcode import" +fi; + +if [ "$IMPORT_US_POSTCODES" = "true" ]; then + ${SCP}:us_postcodes.csv.gz ${PROJECT_DIR}/us_postcodes.csv.gz +elif [ -f "$IMPORT_US_POSTCODES" ]; then + # use local file if asked + ln -s "$IMPORT_US_POSTCODES" ${PROJECT_DIR}/us_postcodes.csv.gz +else + echo "Skipping optional US postcode import" +fi; + +if [ "$IMPORT_TIGER_ADDRESSES" = "true" ]; then + ${SCP}:tiger2023-nominatim-preprocessed.csv.tar.gz ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +elif [ -f "$IMPORT_TIGER_ADDRESSES" ]; then + # use local file if asked + ln -s "$IMPORT_TIGER_ADDRESSES" ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz +else + echo "Skipping optional Tiger addresses import" +fi + +if [ "$PBF_URL" != "" ]; then + echo Downloading OSM extract from "$PBF_URL" + "${CURL[@]}" "$PBF_URL" -C - --create-dirs -o $OSMFILE +fi + +if [ "$PBF_PATH" != "" ]; then + echo Reading OSM extract from "$PBF_PATH" + OSMFILE=$PBF_PATH +fi + + +# if we use a bind mount then the PG directory is empty and we have to create it +if [ ! -f /var/lib/postgresql/16/main/PG_VERSION ]; then + chown postgres:postgres /var/lib/postgresql/16/main + sudo -u postgres /usr/lib/postgresql/16/bin/initdb -D /var/lib/postgresql/16/main +fi + +# temporarily enable unsafe import optimization config +cp /etc/postgresql/16/main/conf.d/postgres-import.conf.disabled /etc/postgresql/16/main/conf.d/postgres-import.conf + +sudo service postgresql start && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -E -u postgres createuser -s nominatim && \ +sudo -E -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -E -u postgres createuser -SDR www-data && \ + +sudo -E -u postgres psql postgres -tAc "ALTER USER nominatim WITH ENCRYPTED PASSWORD '$NOMINATIM_PASSWORD'" && \ +sudo -E -u postgres psql postgres -tAc "ALTER USER \"www-data\" WITH ENCRYPTED PASSWORD '${NOMINATIM_PASSWORD}'" && \ + +sudo -E -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" + +chown -R nominatim:nominatim ${PROJECT_DIR} + +cd ${PROJECT_DIR} + +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS --reverse-only +else + sudo -E -u nominatim nominatim import --osm-file $OSMFILE --threads $THREADS +fi + +if [ -f tiger-nominatim-preprocessed.csv.tar.gz ]; then + echo "Importing Tiger address data" + sudo -E -u nominatim nominatim add-data --tiger-data tiger-nominatim-preprocessed.csv.tar.gz +fi + +# Sometimes Nominatim marks parent places to be indexed during the initial +# import which leads to '123 entries are not yet indexed' errors in --check-database +# Thus another quick additional index here for the remaining places +sudo -E -u nominatim nominatim index --threads $THREADS + +sudo -E -u nominatim nominatim admin --check-database + +if [ "$REPLICATION_URL" != "" ]; then + sudo -E -u nominatim nominatim replication --init + if [ "$FREEZE" = "true" ]; then + echo "Skipping freeze because REPLICATION_URL is not empty" + fi +else + if [ "$FREEZE" = "true" ]; then + echo "Freezing database" + sudo -E -u nominatim nominatim freeze + fi +fi + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + sudo -H -E -u nominatim nominatim admin --warm --reverse +else + sudo -H -E -u nominatim nominatim admin --warm +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 + +# gather statistics for query planner to potentially improve query performance +# see, https://github.com/osm-search/Nominatim/issues/1023 +# and https://github.com/osm-search/Nominatim/issues/1139 +sudo -E -u nominatim psql -d nominatim -c "ANALYZE VERBOSE" + +sudo service postgresql stop + +# Remove slightly unsafe postgres config overrides that made the import faster +rm /etc/postgresql/16/main/conf.d/postgres-import.conf + +echo "Deleting downloaded dumps in ${PROJECT_DIR}" +rm -f ${PROJECT_DIR}/*sql.gz +rm -f ${PROJECT_DIR}/*csv.gz +rm -f ${PROJECT_DIR}/tiger-nominatim-preprocessed.csv.tar.gz + +if [ "$PBF_URL" != "" ]; then + rm -f ${OSMFILE} +fi diff --git a/4.5/start.sh b/4.5/start.sh new file mode 100755 index 00000000..d3127717 --- /dev/null +++ b/4.5/start.sh @@ -0,0 +1,83 @@ +#!/bin/bash -ex + +tailpid=0 +replicationpid=0 + +stopServices() { + service apache2 stop + service postgresql stop + # Check if the replication process is active + if [ $replicationpid -ne 0 ]; then + echo "Shutting down replication process" + kill $replicationpid + fi + kill $tailpid + # Force exit code 0 to signal a succesfull shutdown to Docker + exit 0 +} +trap stopServices SIGTERM TERM INT + +/app/config.sh + +if id nominatim >/dev/null 2>&1; then + echo "user nominatim already exists" +else + useradd -m -p ${NOMINATIM_PASSWORD} nominatim +fi + +IMPORT_FINISHED=/var/lib/postgresql/16/main/import-finished + +if [ ! -f ${IMPORT_FINISHED} ]; then + /app/init.sh + touch ${IMPORT_FINISHED} +else + chown -R nominatim:nominatim ${PROJECT_DIR} +fi + +service postgresql start + +cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions + +service apache2 start + +# start continous replication process +if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then + # run init in case replication settings changed + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --init + if [ "$UPDATE_MODE" == "continuous" ]; then + echo "starting continuous replication" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "once" ]; then + echo "starting replication once" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --once &> /var/log/replication.log & + replicationpid=${!} + elif [ "$UPDATE_MODE" == "catch-up" ]; then + echo "starting replication once in catch-up mode" + sudo -E -u nominatim nominatim replication --project-dir ${PROJECT_DIR} --catch-up &> /var/log/replication.log & + replicationpid=${!} + else + echo "skipping replication" + fi +fi + +# fork a process and wait for it +tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tailpid=${!} + +export NOMINATIM_QUERY_TIMEOUT=600 +export NOMINATIM_REQUEST_TIMEOUT=3600 +if [ "$REVERSE_ONLY" = "true" ]; then + echo "Warm database caches for reverse queries" + sudo -H -E -u nominatim nominatim admin --warm --reverse > /dev/null +else + echo "Warm database caches for search and reverse queries" + sudo -H -E -u nominatim nominatim admin --warm > /dev/null +fi +export NOMINATIM_QUERY_TIMEOUT=10 +export NOMINATIM_REQUEST_TIMEOUT=60 +echo "Warming finished" + +echo "--> Nominatim is ready to accept requests" + +wait diff --git a/4.5/startapache.sh b/4.5/startapache.sh new file mode 100755 index 00000000..0d2f82ec --- /dev/null +++ b/4.5/startapache.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cp /data/local.php /app/src/build/settings/local.php + +/usr/sbin/apache2ctl -D FOREGROUND +tail -f /var/log/apache2/error.log diff --git a/4.5/startpostgres.sh b/4.5/startpostgres.sh new file mode 100755 index 00000000..e3afb39d --- /dev/null +++ b/4.5/startpostgres.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +service postgresql start +tail -f /var/log/postgresql/postgresql-14-main.log From 33259019a3ed084a17ef9fbd67bc7b3d52469987 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 15:29:42 +0100 Subject: [PATCH 02/26] Replace apache with gunicorn --- 4.5/Dockerfile | 1 - 4.5/start.sh | 6 ++---- 4.5/startapache.sh | 5 ----- 3 files changed, 2 insertions(+), 10 deletions(-) delete mode 100755 4.5/startapache.sh diff --git a/4.5/Dockerfile b/4.5/Dockerfile index fcc23be8..aa482744 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -85,7 +85,6 @@ COPY conf.d/postgres-tuning.conf /etc/postgresql/16/main/conf.d/ COPY config.sh /app/config.sh COPY init.sh /app/init.sh COPY start.sh /app/start.sh -COPY startapache.sh /app/startapache.sh COPY startpostgres.sh /app/startpostgres.sh # Collapse image to single layer. diff --git a/4.5/start.sh b/4.5/start.sh index d3127717..d4d8de38 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -38,8 +38,6 @@ service postgresql start cd ${PROJECT_DIR} && sudo -E -u nominatim nominatim refresh --website --functions -service apache2 start - # start continous replication process if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then # run init in case replication settings changed @@ -62,7 +60,7 @@ if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then fi # fork a process and wait for it -tail -Fv /var/log/postgresql/postgresql-14-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tail -Fv /var/log/postgresql/postgresql-16-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & tailpid=${!} export NOMINATIM_QUERY_TIMEOUT=600 @@ -80,4 +78,4 @@ echo "Warming finished" echo "--> Nominatim is ready to accept requests" -wait +gunicorn --bind :8000 -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi diff --git a/4.5/startapache.sh b/4.5/startapache.sh deleted file mode 100755 index 0d2f82ec..00000000 --- a/4.5/startapache.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -cp /data/local.php /app/src/build/settings/local.php - -/usr/sbin/apache2ctl -D FOREGROUND -tail -f /var/log/apache2/error.log From c805c1cd311451480ebc08368839e5676d2515c7 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 15:32:43 +0100 Subject: [PATCH 03/26] Remove more apache stuff --- 4.5/start.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/4.5/start.sh b/4.5/start.sh index d4d8de38..28df068a 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -4,7 +4,6 @@ tailpid=0 replicationpid=0 stopServices() { - service apache2 stop service postgresql stop # Check if the replication process is active if [ $replicationpid -ne 0 ]; then @@ -12,7 +11,7 @@ stopServices() { kill $replicationpid fi kill $tailpid - # Force exit code 0 to signal a succesfull shutdown to Docker + # Force exit code 0 to signal a succesful shutdown to Docker exit 0 } trap stopServices SIGTERM TERM INT @@ -60,7 +59,7 @@ if [ "$REPLICATION_URL" != "" ] && [ "$FREEZE" != "true" ]; then fi # fork a process and wait for it -tail -Fv /var/log/postgresql/postgresql-16-main.log /var/log/apache2/access.log /var/log/apache2/error.log /var/log/replication.log & +tail -Fv /var/log/postgresql/postgresql-16-main.log & tailpid=${!} export NOMINATIM_QUERY_TIMEOUT=600 @@ -78,4 +77,4 @@ echo "Warming finished" echo "--> Nominatim is ready to accept requests" -gunicorn --bind :8000 -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi +gunicorn --bind :8080 -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi From 582ea9ead91fe5fa7944af70638d61b4ae6e5ec0 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 15:44:22 +0100 Subject: [PATCH 04/26] Run gunicorn under correct user --- 4.5/start.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/4.5/start.sh b/4.5/start.sh index 28df068a..4a29c339 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -77,4 +77,5 @@ echo "Warming finished" echo "--> Nominatim is ready to accept requests" -gunicorn --bind :8080 -b unix:/run/nominatim.sock -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi +cd "$PROJECT_DIR" +sudo -u nominatim gunicorn --bind :8080 -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi From bf2b3e7d09c710a9dc8d0fc9108f1261ab9104f4 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 17:40:28 +0100 Subject: [PATCH 05/26] Update URL for Rutland --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be602e2a..1c50429d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,7 +196,7 @@ jobs: - name: GB postcode import commands: |- docker run -i --rm \ - -e PBF_URL=https://download.geofabrik.de/europe/great-britain/england/rutland-latest.osm.pbf \ + -e PBF_URL=https://download.geofabrik.de/europe/united-kingdom/england/rutland-latest.osm.pbf \ -e IMPORT_GB_POSTCODES="true" \ -p 8010:8080 \ nominatim & From 9fcfb15ee85e639fb25192aa2e4316f61cb0100e Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 17:50:55 +0100 Subject: [PATCH 06/26] Remove apache.conf --- 4.5/conf.d/apache.conf | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 4.5/conf.d/apache.conf diff --git a/4.5/conf.d/apache.conf b/4.5/conf.d/apache.conf deleted file mode 100644 index 922fc296..00000000 --- a/4.5/conf.d/apache.conf +++ /dev/null @@ -1,13 +0,0 @@ -Listen 8080 - - DocumentRoot /nominatim/website - CustomLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/access.log 86400" combined - ErrorLog "|$/usr/bin/rotatelogs -n 7 /var/log/apache2/error.log 86400" - LogLevel info - - Options FollowSymLinks MultiViews - DirectoryIndex search.php - Require all granted - - AddType text/html .php - From 217effd57540654a5c9fb368fdca585e9f6b9de9 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 18:03:29 +0100 Subject: [PATCH 07/26] Update docs, remove build-essential --- .github/workflows/ci.yml | 4 ++-- 4.5/Dockerfile | 6 ++++-- 4.5/README.md | 26 +++++++++++++------------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c50429d..29f8a0ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: strategy: matrix: nominatim: - - version: "4.3" - version: "4.4" - + - version: "4.5" + runs-on: ubuntu-latest steps: - name: Checkout diff --git a/4.5/Dockerfile b/4.5/Dockerfile index aa482744..ce382fb1 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -29,10 +29,11 @@ RUN \ -o APT::Install-Suggests="false" \ # Build tools from sources. \ build-essential \ - python3-dev \ osm2pgsql \ pkg-config \ - libicu-dev python3-pip \ + libicu-dev \ + python3-dev \ + python3-pip \ # PostgreSQL. postgresql-contrib \ postgresql-server-dev-16 \ @@ -69,6 +70,7 @@ RUN pip install --break-system-packages \ RUN true \ # Remove development and unused packages. && apt-get -y remove --purge --auto-remove \ + build-essential \ postgresql-server-dev-16 \ # Clear temporary files and directories. && rm -rf \ diff --git a/4.5/README.md b/4.5/README.md index 506e1d74..12d436e0 100644 --- a/4.5/README.md +++ b/4.5/README.md @@ -1,4 +1,4 @@ -# Nominatim Docker (Nominatim version 4.4) +# Nominatim Docker (Nominatim version 4.5) ## Table of contents @@ -30,7 +30,7 @@ docker run -it \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` Port 8080 is the nominatim HTTP API port and 5432 is the Postgres port, which you may or may not want to expose. @@ -54,7 +54,7 @@ Other places at Geofabrik follow the pattern `https://download.geofabrik.de/$CON - `REPLICATION_UPDATE_INTERVAL`: How often upstream publishes diffs (in seconds, default: `86400`). _Requires `REPLICATION_URL` to be set._ - `REPLICATION_RECHECK_INTERVAL`: How long to sleep if no update found yet (in seconds, default: `900`). _Requires `REPLICATION_URL` to be set._ -- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.4.1/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) +- `UPDATE_MODE`: How to run replication to [update nominatim data](https://nominatim.org/release-docs/4.5.0/admin/Update/#updating-nominatim). Options: `continuous`/`once`/`catch-up`/`none` (default: `none`) - `FREEZE`: Freeze database and disable dynamic updates to save space. (default: `false`) - `REVERSE_ONLY`: If you only want to use the Nominatim database for reverse lookups. (default: `false`) - `IMPORT_WIKIPEDIA`: Whether to download and import the Wikipedia importance dumps (`true`) or path to importance dump in the container. Importance dumps improve the scoring of results. On a beefy 10 core server, this takes around 5 minutes. (default: `false`) @@ -82,7 +82,7 @@ The following environment variables are available to tune PostgreSQL: - `POSTGRES_CHECKPOINT_TIMEOUT` (default: `10min`) - `POSTGRES_CHECKPOINT_COMPLETION_TARGET` (default: `0.9`) -See https://nominatim.org/release-docs/4.4.1/admin/Installation/#tuning-the-postgresql-database for more details on those settings. +See https://nominatim.org/release-docs/4.5.0/admin/Installation/#tuning-the-postgresql-database for more details on those settings. ### Import Style @@ -98,11 +98,11 @@ Available options are : - `full`: Default style that also includes points of interest. - `extratags`: Like the full style but also adds most of the OSM tags into the extratags column. -See https://nominatim.org/release-docs/4.4.1/admin/Import/#filtering-imported-data for more details on those styles. +See https://nominatim.org/release-docs/4.5.0/admin/Import/#filtering-imported-data for more details on those styles. ### Flatnode files -In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.4.1/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. +In addition you can also mount a volume / bind-mount on `/nominatim/flatnode` (see: Persistent container data) to use flatnode storage. This is advised for bigger imports (Europe, North America etc.), see: https://nominatim.org/release-docs/4.5.0/admin/Import/#flatnode-files. If the mount is available for the container, the flatnode configuration is automatically set and used. ```sh docker run -it \ @@ -111,7 +111,7 @@ docker run -it \ -e REPLICATION_URL=https://download.geofabrik.de/europe/monaco-updates/ \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` ### Configuration Example @@ -137,7 +137,7 @@ docker run -it --shm-size=1g \ -v nominatim-data:/var/lib/postgresql/16/main \ -p 8080:8080 \ --name nominatim \ - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` ## OpenStreetMap Data Extracts @@ -160,14 +160,14 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ --name nominatim \ - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` where the _/osm-maps/data/_ directory contains _monaco-latest.osm.pbf_ file that is mounted and available in container: _/nominatim/data/monaco-latest.osm.pbf_ ## Updating the database -Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.4.1/admin/Update/). For a list of other methods see the output of: +Full documentation for Nominatim update available [here](https://nominatim.org/release-docs/4.5.0/admin/Update/). For a list of other methods see the output of: ```sh docker exec -it nominatim sudo -u nominatim nominatim replication --help @@ -191,7 +191,7 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/data:/nominatim/data \ --name nominatim \ - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` where the _/osm-maps/data/_ directory contains _merged.osm.pbf_ file that is mounted and available in container: _/nominatim/data/merged.osm.pbf_ @@ -207,12 +207,12 @@ docker run -it \ -p 8080:8080 \ -v /osm-maps/extras:/nominatim/extras \ --name nominatim \ - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` Where the path to the importance dump is given relative to the container. (The file does not need to be named `wikimedia-importance.sql.gz`.) The same works for `IMPORT_US_POSTCODES` and `IMPORT_GB_POSTCODES`. -For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.4.1/customize/Tiger/). +For more information about the Tiger address file, see [Installing TIGER housenumber data for the US](https://nominatim.org/release-docs/4.5.0/customize/Tiger/). ## Development From 209b8b02ab58b84e70c66d4e8a6385ff508e09ae Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 18:12:53 +0100 Subject: [PATCH 08/26] Remove apache config from Dockerfile --- 4.5/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/4.5/Dockerfile b/4.5/Dockerfile index ce382fb1..07ac1e8f 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -77,9 +77,6 @@ RUN true \ /tmp/* \ /var/tmp/* -# Apache configuration -COPY conf.d/apache.conf /etc/apache2/sites-enabled/000-default.conf - # Postgres config overrides to improve import performance (but reduce crash recovery safety) COPY conf.d/postgres-import.conf /etc/postgresql/16/main/conf.d/postgres-import.conf.disabled COPY conf.d/postgres-tuning.conf /etc/postgresql/16/main/conf.d/ From e84cd340d970ee21bbe55dc4ef0f6a8cf8b53517 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 18:16:14 +0100 Subject: [PATCH 09/26] Use cache again --- 4.5/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/4.5/Dockerfile b/4.5/Dockerfile index 07ac1e8f..933468db 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -55,8 +55,7 @@ ARG NOMINATIM_VERSION ARG USER_AGENT # Osmium install to run continuous updates. -#RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ -RUN pip install --break-system-packages \ +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ nominatim-db==$NOMINATIM_VERSION \ osmium \ psycopg[binary] \ From c4de1f0a9263c35dddef0c7f679db35e0c8d63ee Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 18:21:16 +0100 Subject: [PATCH 10/26] Update docs --- 4.5/Dockerfile | 3 ++- README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/4.5/Dockerfile b/4.5/Dockerfile index 933468db..fba75d70 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -74,7 +74,8 @@ RUN true \ # Clear temporary files and directories. && rm -rf \ /tmp/* \ - /var/tmp/* + /var/tmp/* \ + && pip cache purge # Postgres config overrides to improve import performance (but reduce crash recovery safety) COPY conf.d/postgres-import.conf /etc/postgresql/16/main/conf.d/postgres-import.conf.disabled diff --git a/README.md b/README.md index 5485d5fb..5f6b9fc3 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ See relevant installation and usage instructions for each version in the ``` Date: Mon, 11 Nov 2024 18:23:21 +0100 Subject: [PATCH 11/26] Use correct versions --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29f8a0ff..95f6692e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,9 +43,9 @@ jobs: strategy: matrix: nominatim: - - version: "4.3" - version: "4.4" - + - version: "4.5" + test_scenario: - name: Import with PBF_URL and update commands: |- @@ -257,9 +257,9 @@ jobs: strategy: matrix: nominatim: - - version: "4.3" - version: "4.4" - + - version: "4.5" + steps: - name: Checkout uses: actions/checkout@v4 From 1caf48575df9dc6354c6a6433b54e058d03242e4 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Mon, 11 Nov 2024 18:24:48 +0100 Subject: [PATCH 12/26] Update docs --- 4.5/example.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/4.5/example.md b/4.5/example.md index 370b64a7..0a70a252 100644 --- a/4.5/example.md +++ b/4.5/example.md @@ -75,5 +75,5 @@ docker run -it \ --name nominatim \ #Here you choose the Docker image and version - mediagis/nominatim:4.4 + mediagis/nominatim:4.5 ``` From 3fd19fbea8c0843c0e385ff15d32bd5939f7a828 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 14:11:46 +0100 Subject: [PATCH 13/26] Shutdown cleanly --- 4.5/Dockerfile | 3 ++- 4.5/start.sh | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/4.5/Dockerfile b/4.5/Dockerfile index fba75d70..098e5699 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -55,7 +55,8 @@ ARG NOMINATIM_VERSION ARG USER_AGENT # Osmium install to run continuous updates. -RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ +#RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ +RUN pip install --break-system-packages \ nominatim-db==$NOMINATIM_VERSION \ osmium \ psycopg[binary] \ diff --git a/4.5/start.sh b/4.5/start.sh index 4a29c339..f1ffde30 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -2,6 +2,7 @@ tailpid=0 replicationpid=0 +GUNICORN_PID_FILE=/tmp/gunicorn.pid stopServices() { service postgresql stop @@ -11,7 +12,8 @@ stopServices() { kill $replicationpid fi kill $tailpid - # Force exit code 0 to signal a succesful shutdown to Docker + cat $GUNICORN_PID_FILE | sudo xargs kill + # Force exit code 0 to signal a successful shutdown to Docker exit 0 } trap stopServices SIGTERM TERM INT @@ -78,4 +80,8 @@ echo "Warming finished" echo "--> Nominatim is ready to accept requests" cd "$PROJECT_DIR" -sudo -u nominatim gunicorn --bind :8080 -w 4 -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi +sudo -u nominatim gunicorn \ + --bind :8080 \ + --pid $GUNICORN_PID_FILE \ + -w 4 \ + -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi From b024960f6756b0620ffc42da35946880fd58a143 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 14:12:01 +0100 Subject: [PATCH 14/26] Update version in docs, mention n7m --- 4.5/conf.d/postgres-tuning.conf | 2 +- 4.5/contrib/docker-compose-planet.yml | 2 +- 4.5/contrib/docker-compose.yml | 4 ++-- README.md | 9 ++++++++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/4.5/conf.d/postgres-tuning.conf b/4.5/conf.d/postgres-tuning.conf index 4bf5b4b1..39952b79 100644 --- a/4.5/conf.d/postgres-tuning.conf +++ b/4.5/conf.d/postgres-tuning.conf @@ -1,4 +1,4 @@ -# See https://nominatim.org/release-docs/4.4.1/admin/Installation/#tuning-the-postgresql-database +# See https://nominatim.org/release-docs/4.5.0/admin/Installation/#tuning-the-postgresql-database shared_buffers = 2GB maintenance_work_mem = 10GB autovacuum_work_mem = 2GB diff --git a/4.5/contrib/docker-compose-planet.yml b/4.5/contrib/docker-compose-planet.yml index c2394b7a..1b258b80 100644 --- a/4.5/contrib/docker-compose-planet.yml +++ b/4.5/contrib/docker-compose-planet.yml @@ -10,7 +10,7 @@ version: "3" services: nominatim: container_name: nominatim - image: mediagis/nominatim:4.4 + image: mediagis/nominatim:4.5 ports: - "8080:8080" # Do not change the second port, only the first before the colon environment: diff --git a/4.5/contrib/docker-compose.yml b/4.5/contrib/docker-compose.yml index 59e53254..aa36b0e0 100644 --- a/4.5/contrib/docker-compose.yml +++ b/4.5/contrib/docker-compose.yml @@ -3,11 +3,11 @@ version: "3" services: nominatim: container_name: nominatim - image: mediagis/nominatim:4.4 + image: mediagis/nominatim:4.5 ports: - "8080:8080" environment: - # see https://github.com/mediagis/nominatim-docker/tree/master/4.4#configuration for more options + # see https://github.com/mediagis/nominatim-docker/tree/master/4.5#configuration for more options PBF_URL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf REPLICATION_URL: https://download.geofabrik.de/europe/monaco-updates/ NOMINATIM_PASSWORD: very_secure_password diff --git a/README.md b/README.md index 5f6b9fc3..dc206c64 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,16 @@ See relevant installation and usage instructions for each version in the ``` Date: Tue, 12 Nov 2024 14:24:20 +0100 Subject: [PATCH 15/26] Change gunicorn killing --- 4.5/start.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/4.5/start.sh b/4.5/start.sh index f1ffde30..ff6032d8 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -12,7 +12,8 @@ stopServices() { kill $replicationpid fi kill $tailpid - cat $GUNICORN_PID_FILE | sudo xargs kill + pkill gunicorn + # Force exit code 0 to signal a successful shutdown to Docker exit 0 } From 17676ea89ae62348d43eda073738caad19fa7fc6 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 14:35:43 +0100 Subject: [PATCH 16/26] Update execution --- 4.5/start.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/4.5/start.sh b/4.5/start.sh index ff6032d8..9d14357c 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -84,5 +84,6 @@ cd "$PROJECT_DIR" sudo -u nominatim gunicorn \ --bind :8080 \ --pid $GUNICORN_PID_FILE \ - -w 4 \ - -k uvicorn.workers.UvicornWorker nominatim_api.server.falcon.server:run_wsgi + --workers 4 \ + --worker-class uvicorn.workers.UvicornWorker \ + nominatim_api.server.falcon.server:run_wsgi From c7e46ed808b53d8eaef52032ca0d4d4d1d7820de Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 14:42:08 +0100 Subject: [PATCH 17/26] Try running in daemon mode --- 4.5/start.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/4.5/start.sh b/4.5/start.sh index 9d14357c..32cd1f64 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -12,8 +12,10 @@ stopServices() { kill $replicationpid fi kill $tailpid - pkill gunicorn - + echo `cat $GUNICORN_PID_FILE` + echo + cat $GUNICORN_PID_FILE | sudo xargs kill + # Force exit code 0 to signal a successful shutdown to Docker exit 0 } @@ -85,5 +87,8 @@ sudo -u nominatim gunicorn \ --bind :8080 \ --pid $GUNICORN_PID_FILE \ --workers 4 \ + --daemon \ --worker-class uvicorn.workers.UvicornWorker \ nominatim_api.server.falcon.server:run_wsgi + +wait \ No newline at end of file From deb09cf3b838e0fa3dd5e52b9560c49e153f983b Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 15:08:43 +0100 Subject: [PATCH 18/26] Don't buffer output --- 4.5/start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/4.5/start.sh b/4.5/start.sh index 32cd1f64..e27518f3 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -3,6 +3,8 @@ tailpid=0 replicationpid=0 GUNICORN_PID_FILE=/tmp/gunicorn.pid +# send gunicorn logs straight to the console without buffering: https://stackoverflow.com/questions/59812009 +export PYTHONUNBUFFERED=1 stopServices() { service postgresql stop From 81261b51f0ba71f89f19d9a0fcc6d421ab5c5456 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 15:20:18 +0100 Subject: [PATCH 19/26] Enable stdio inheritance --- 4.5/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/4.5/start.sh b/4.5/start.sh index e27518f3..50d8cb50 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -90,6 +90,7 @@ sudo -u nominatim gunicorn \ --pid $GUNICORN_PID_FILE \ --workers 4 \ --daemon \ + --enable-stdio-inheritance \ --worker-class uvicorn.workers.UvicornWorker \ nominatim_api.server.falcon.server:run_wsgi From c0fbd54d3057ce7d85d52731734525eaf43979f4 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 15:35:44 +0100 Subject: [PATCH 20/26] Don't assert that a reverse-only returns a 404 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95f6692e..77b87065 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -213,7 +213,7 @@ jobs: -p 8011:8080 \ nominatim & - ./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" + #./assert-reverse-only "http://localhost:8011/search.php?q=avenue%20pasteur" ./assert-non-empty-json "http://localhost:8011/reverse.php?lat=43.734&lon=7.42&format=jsonv2" - name: Check for clean shutdown From 6a247d480c4e9be4fa4bdc5e560c253da4af38f7 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 21:17:59 +0100 Subject: [PATCH 21/26] Remove nominatim container during CI --- .github/workflows/ci.yml | 1 + 4.5/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77b87065..7ef4d990 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: # Stop container docker stop nominatim + docker ps --all # import to bind mount is done previously docker run -i --rm \ diff --git a/4.5/Dockerfile b/4.5/Dockerfile index 098e5699..fba75d70 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -55,8 +55,7 @@ ARG NOMINATIM_VERSION ARG USER_AGENT # Osmium install to run continuous updates. -#RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ -RUN pip install --break-system-packages \ +RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ nominatim-db==$NOMINATIM_VERSION \ osmium \ psycopg[binary] \ From 17ed3da64e5b5857aeb9067ee0094834f5063222 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 21:44:01 +0100 Subject: [PATCH 22/26] Properly shutdown background processes --- 4.5/start.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/4.5/start.sh b/4.5/start.sh index 50d8cb50..93328c0a 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -14,9 +14,8 @@ stopServices() { kill $replicationpid fi kill $tailpid - echo `cat $GUNICORN_PID_FILE` - echo cat $GUNICORN_PID_FILE | sudo xargs kill + gkill nominatim # Force exit code 0 to signal a successful shutdown to Docker exit 0 From c04f181e84d88cd9dfa46bcca48fb5560285c31a Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Tue, 12 Nov 2024 22:02:01 +0100 Subject: [PATCH 23/26] Sleep after stopping container --- .github/workflows/ci.yml | 2 +- 4.5/start.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ef4d990..4dfdc0c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: # Stop container docker stop nominatim - docker ps --all + sleep 60 # import to bind mount is done previously docker run -i --rm \ diff --git a/4.5/start.sh b/4.5/start.sh index 93328c0a..06349391 100755 --- a/4.5/start.sh +++ b/4.5/start.sh @@ -15,7 +15,6 @@ stopServices() { fi kill $tailpid cat $GUNICORN_PID_FILE | sudo xargs kill - gkill nominatim # Force exit code 0 to signal a successful shutdown to Docker exit 0 From 37a9cd9177976aaee92b96fb6c22c9bb30fb2cc9 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 13 Nov 2024 08:18:53 +0100 Subject: [PATCH 24/26] Lower sleep --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dfdc0c3..e91c2d85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,8 @@ jobs: # Stop container docker stop nominatim - sleep 60 + # the container needs a few seconds to shutdown cleanly apparently + sleep 15 # import to bind mount is done previously docker run -i --rm \ From 70b16f0677178eef586c62c0f751dd4b13758333 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 13 Nov 2024 10:20:44 +0100 Subject: [PATCH 25/26] Remove startpostgres.sh --- 4.5/Dockerfile | 5 ++--- 4.5/startpostgres.sh | 4 ---- 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100755 4.5/startpostgres.sh diff --git a/4.5/Dockerfile b/4.5/Dockerfile index fba75d70..d233eea3 100644 --- a/4.5/Dockerfile +++ b/4.5/Dockerfile @@ -54,7 +54,7 @@ RUN true \ ARG NOMINATIM_VERSION ARG USER_AGENT -# Osmium install to run continuous updates. +# Nominatim install. RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --break-system-packages \ nominatim-db==$NOMINATIM_VERSION \ osmium \ @@ -64,8 +64,8 @@ RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked pip install --brea gunicorn \ nominatim-api -# Nominatim install. +# remove build-only packages RUN true \ # Remove development and unused packages. && apt-get -y remove --purge --auto-remove \ @@ -84,7 +84,6 @@ COPY conf.d/postgres-tuning.conf /etc/postgresql/16/main/conf.d/ COPY config.sh /app/config.sh COPY init.sh /app/init.sh COPY start.sh /app/start.sh -COPY startpostgres.sh /app/startpostgres.sh # Collapse image to single layer. FROM scratch diff --git a/4.5/startpostgres.sh b/4.5/startpostgres.sh deleted file mode 100755 index e3afb39d..00000000 --- a/4.5/startpostgres.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -service postgresql start -tail -f /var/log/postgresql/postgresql-14-main.log From d8abeb78a09d3cac7737df7c4baebeb6fb5dcc55 Mon Sep 17 00:00:00 2001 From: Leonard Ehrenfried Date: Wed, 13 Nov 2024 10:50:56 +0100 Subject: [PATCH 26/26] Rephrase README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dc206c64..c978e6fe 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ See relevant installation and usage instructions for each version in the ```