Merge pull request #384 from camptocamp/poetry-gsgmf-1726

Use Poetry instead of Pipenv

.dockerignore

@@ -4,7 +4,8 @@
 !runtime-desktop
 !requirements.txt
 !requirements-desktop.txt
-!Pipfile*
 !cache
 !package.json
 !package-lock.json
+!pyproject.toml
+!poetry.lock

.hadolint.yaml

@@ -1,5 +1,7 @@
 ignored:
- - DL3008 # pin version: https://github.com/hadolint/hadolint/wiki/DL3008
- - DL3003 # use workdir: https://github.com/hadolint/hadolint/wiki/DL3003
+ - DL3003 # Use WORKDIR to switch to a directory
+ - DL3008 # Pin versions in apt get install. Instead of `apt-get install <package>` use `apt-get install <package>=<version>`
+ - DL3042 # Avoid use of cache directory with pip. Use `pip install --no-cache-dir <package>`
  - DL3059 # Multiple consecutive `RUN` instructions. Consider consolidation.
+ - SC1091 # Not following: File not included in mock.
  - SC2164 # Use cd ... || exit in case cd fails.

Dockerfile

@@ -1,17 +1,38 @@
-FROM osgeo/gdal:ubuntu-small-3.5.0 as base
+FROM osgeo/gdal:ubuntu-small-3.5.0 as base-all
+LABEL maintainer Camptocamp "info@camptocamp.com"
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
 
-RUN --mount=type=cache,target=/var/lib/apt/lists,id=apt-list \
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+ --mount=type=cache,target=/var/cache,sharing=locked \
  apt-get update \
- && apt-get upgrade --yes
+ && apt-get upgrade --assume-yes \
+ && DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends python3-pip
 
-SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+# Used to convert the locked packages by poetry to pip requirements format
+# We don't directly use `poetry install` because it force to use a virtual environment.
+FROM base-all as poetry
+
+# Install Poetry
+WORKDIR /tmp
+COPY requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache \
+ python3 -m pip install --disable-pip-version-check --requirement=requirements.txt
 
-FROM base as builder
+# Do the conversion
+COPY poetry.lock pyproject.toml ./
+RUN poetry export --output=requirements.txt \
+ && poetry export --extras=desktop --output=requirements-desktop.txt
+
+# Base, the biggest thing is to install the Python packages
+FROM base-all as builder
 LABEL maintainer="info@camptocamp.com"
 
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+
 RUN --mount=type=cache,target=/var/lib/apt/lists,id=apt-list \
  --mount=type=cache,target=/var/cache,id=var-cache,sharing=locked \
- . /etc/os-release \
+ apt-get update \
+ && . /etc/os-release \
  && apt-get install --assume-yes --no-install-recommends apt-utils software-properties-common \
  && apt-get autoremove --assume-yes software-properties-common \
  && LC_ALL=C DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends cmake gcc \
@@ -20,7 +41,7 @@ RUN --mount=type=cache,target=/var/lib/apt/lists,id=apt-list \
  libqt5scintilla2-dev libqt5opengl5-dev libqt5sql5-sqlite libqt5webkit5-dev qtpositioning5-dev \
  qtxmlpatterns5-dev-tools libqt5xmlpatterns5-dev libqt5svg5-dev libqwt-qt5-dev libspatialindex-dev \
  libspatialite-dev libsqlite3-dev libqt5designer5 qttools5-dev qt5keychain-dev lighttpd locales \
- pkg-config poppler-utils python3 python3-dev python3-pip python3-setuptools \
+ pkg-config poppler-utils python3 python3-dev python3-pip \
  pyqt5-dev pyqt5-dev-tools pyqt5.qsci-dev python3-pyqt5.qtsql python3-pyqt5.qsci python3-pyqt5.qtpositioning \
  python3-sip python3-sip-dev qtscript5-dev spawn-fcgi xauth xfonts-100dpi \
  xfonts-75dpi xfonts-base xfonts-scalable xvfb git ninja-build ccache clang libpython3-dev \
@@ -38,16 +59,9 @@ RUN npm install
 
 WORKDIR /tmp
 
-COPY requirements.txt ./
-# hadolint ignore=DL3042
-RUN --mount=type=cache,target=/root/.cache,id=root-cache \
- python3 -m pip install --disable-pip-version-check --requirement=requirements.txt \
- && rm --recursive --force /tmp/*
-
-COPY Pipfile Pipfile.lock ./
-RUN --mount=type=cache,target=/root/.cache,id=root-cache \
- pipenv sync --system --dev \
- && rm --recursive --force /usr/local/lib/python3.*/dist-packages/tests/ /tmp/* /root/.cache/* \
+RUN --mount=type=cache,target=/root/.cache \
+ --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
+ python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \
  && (strip /usr/local/lib/python3.*/dist-packages/*/*.so || true)
 
 RUN ln -s /usr/local/lib/libproj.so.* /usr/local/lib/libproj.so
@@ -115,36 +129,29 @@ RUN --mount=type=cache,target=/root/.ccache,id=ccache \
 
 RUN ninja install
 
-FROM base as runner
+FROM base-all as runner
 LABEL maintainer="info@camptocamp.com"
 
 RUN --mount=type=cache,target=/var/lib/apt/lists,id=apt-list \
  --mount=type=cache,target=/var/cache,id=var-cache,sharing=locked \
- DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends \
- libfcgi libgslcblas0 libqca-qt5-2 libqca-qt5-2-plugins libzip5 \
- libqt5opengl5 libqt5sql5-sqlite libqt5concurrent5 libqt5positioning5 libqt5script5 \
- libqt5webkit5 libqwt-qt5-6 libspatialindex6 libspatialite7 libsqlite3-0 libqt5keychain1 \
- python3 python3-pip python3-setuptools \
- python3-pyqt5 python3-pyqt5.qtsql python3-pyqt5.qsci python3-pyqt5.qtpositioning \
- spawn-fcgi xauth xfonts-100dpi xfonts-75dpi xfonts-base xfonts-scalable xvfb \
- apache2 libapache2-mod-fcgid python3 \
- libqt5serialport5 libqt5quickwidgets5 libexiv2-27 libprotobuf17 libprotobuf-lite17 \
- libgsl23 libzstd1 binutils \
+ apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends \
+ libfcgi libgslcblas0 libqca-qt5-2 libqca-qt5-2-plugins libzip5 \
+ libqt5opengl5 libqt5sql5-sqlite libqt5concurrent5 libqt5positioning5 libqt5script5 \
+ libqt5webkit5 libqwt-qt5-6 libspatialindex6 libspatialite7 libsqlite3-0 libqt5keychain1 \
+ python3 python3-pip \
+ python3-pyqt5 python3-pyqt5.qtsql python3-pyqt5.qsci python3-pyqt5.qtpositioning \
+ spawn-fcgi xauth xfonts-100dpi xfonts-75dpi xfonts-base xfonts-scalable xvfb \
+ apache2 libapache2-mod-fcgid python3 \
+ libqt5serialport5 libqt5quickwidgets5 libexiv2-27 libprotobuf17 libprotobuf-lite17 \
+ libgsl23 libzstd1 binutils \
  && strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5
 
 WORKDIR /tmp
 
-COPY requirements.txt ./
-# hadolint ignore=DL3042
-RUN --mount=type=cache,target=/root/.cache,id=root-cache \
- python3 -m pip install --disable-pip-version-check --requirement=requirements.txt \
- && rm --recursive --force /tmp/*
-
-COPY Pipfile Pipfile.lock ./
-RUN --mount=type=cache,target=/root/.cache,id=root-cache \
- pipenv sync --system \
- && rm --recursive --force /usr/local/lib/python3.*/dist-packages/tests/ /tmp/* /root/.cache/* \
- && (strip /usr/local/lib/python3.*/dist-packages/*/*.so || true)
+RUN --mount=type=cache,target=/root/.cache \
+ --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
+ python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt
 
 FROM runner as runner-server
 
@@ -227,6 +234,10 @@ RUN --mount=type=cache,target=/root/.cache,id=root-cache \
  python3 -m pip install --disable-pip-version-check --requirement=requirements-desktop.txt \
  && rm --recursive --force /tmp/*
 
+RUN --mount=type=cache,target=/root/.cache \
+ --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
+ python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements-desktop.txt
+
 COPY --from=builder-desktop /usr/local/bin /usr/local/bin/
 COPY --from=builder-desktop /usr/local/lib /usr/local/lib/
 COPY --from=builder-desktop /usr/local/share /usr/local/share/

Makefile

@@ -2,26 +2,35 @@ QGIS_BRANCH = master
 DOCKER_TAG ?= latest
 DOCKER_BASE = camptocamp/qgis-server
 ROOT = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+export DOCKER_BUILDKIT = 1
 
 DOCKER_COMPOSE_TTY := $(shell [ ! -t 0 ] && echo -T)
 BUILD_OPTIONS = build
 
+.PHONY: help
+help: ## Display this help message
+ @echo "Usage: make <target>"
+ @echo
+ @echo "Available targets:"
+ @grep --extended-regexp --no-filename '^[a-zA-Z_-]+:.*## ' $(MAKEFILE_LIST) | sort | \
+ awk 'BEGIN {FS = ":.*?## "}; {printf " %-20s%s\n", $$1, $$2}'
+
 .PHONY: all
-all: build acceptance
+all: build acceptance ## Build and run acceptance tests
 
 .PHONY: build-server
-build-server:
+build-server: ## Build the server Docker image
  DOCKER_BUILDKIT=1 docker $(BUILD_OPTIONS) --target=runner-server --tag=$(DOCKER_BASE):$(DOCKER_TAG) --build-arg=QGIS_BRANCH=$(QGIS_BRANCH) .
 
 .PHONY: build-desktop
-build-desktop:
+build-desktop: ## Build the desktop Docker image
  DOCKER_BUILDKIT=1 docker $(BUILD_OPTIONS) --target=runner-desktop --tag=$(DOCKER_BASE):$(DOCKER_TAG)-desktop --build-arg=QGIS_BRANCH=$(QGIS_BRANCH) .
 
 .PHONY: build-cache
 build-cache:
  DOCKER_BUILDKIT=1 docker $(BUILD_OPTIONS) --target=cache --tag=qgis-cache --build-arg=QGIS_BRANCH=$(QGIS_BRANCH) .
 
-.PHONY: build
+.PHONY: build ## Build all the Docker images
 build: build-server build-desktop
 
 .PHONY: build-acceptance-config
@@ -38,16 +47,12 @@ run: build-acceptance
  cd acceptance_tests; docker-compose up -d
 
 .PHONY: acceptance
-acceptance: run
+acceptance: run ## Run the acceptance tests
  cd acceptance_tests; docker-compose exec $(DOCKER_COMPOSE_TTY) run pytest -vv --color=yes --junitxml=/tmp/junitxml/results.xml
  cd acceptance_tests; docker-compose exec $(DOCKER_COMPOSE_TTY) qgis python3 -c 'import qgis'
 
-.PHONY: pull
-pull:
- for image in `find -name Dockerfile | xargs grep --no-filename ^FROM |grep -v 'FROM runner'|grep -v 'FROM builder'| awk '{print $$2}'`; do docker pull $$image; done
-
 .PHONY: run-client
-run-client:
+run-client: ## Run the desktop application
  docker run --rm -ti -e DISPLAY=unix${DISPLAY} --volume=/tmp/.X11-unix:/tmp/.X11-unix --volume=${HOME}:${HOME} $(DOCKER_BASE):$(DOCKER_TAG)-desktop
 
 clean:

acceptance_tests/Dockerfile

@@ -1,19 +1,41 @@
-FROM python:3.10.5
+FROM python:3.10.5 as base-all
+LABEL maintainer "info@camptocamp.org"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+ --mount=type=cache,target=/var/cache,sharing=locked \
+ apt-get update \
+ && apt-get upgrade --assume-yes \
+ && DEBIAN_FRONTEND=noninteractive apt-get install --assume-yes --no-install-recommends python3-pip
+
+# Used to convert the locked packages by poetry to pip requirements format
+# We don't directly use `poetry install` because it force to use a virtual environment.
+FROM base-all as poetry
+
+# Install Poetry
+WORKDIR /tmp
+COPY requirements.txt ./
+RUN --mount=type=cache,target=/root/.cache \
+ python3 -m pip install --disable-pip-version-check --requirement=requirements.txt
+
+# Do the conversion
+COPY poetry.lock pyproject.toml ./
+RUN poetry export --dev --output=requirements-dev.txt
+
+# Base, the biggest thing is to install the Python packages
+FROM base-all
 
 RUN mkdir -p /tmp/junitxml
 
-COPY requirements.txt /tmp/
-RUN python3 -m pip install --disable-pip-version-check --no-cache-dir --requirement=/tmp/requirements.txt \
- && rm --recursive --force /tmp/*
+RUN --mount=type=cache,target=/root/.cache \
+ --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
+ python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements-dev.txt
 
 WORKDIR /acceptance_tests
 
-COPY Pipfile Pipfile.lock ./
-RUN pipenv sync --system --clear --dev \
- && rm --recursive --force /usr/local/lib/python3.*/dist-packages/tests/ /tmp/* /root/.cache/*
-
 COPY . ./
 
 ENV DOCKER_RUN 1
 
-CMD ["sleep", "infinity"]
+CMD ["tail", "--follow", "/dev/null"]