Refactor/SK-1217 | Update image and remove support for py 3.8 (#755)

.github/workflows/code-checks.yaml

@@ -4,7 +4,7 @@ on: push
 
 jobs:
   code-checks:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-tests.yaml

@@ -23,9 +23,9 @@ jobs:
         to_test:
           - "mnist-keras numpyhelper"
           - "mnist-pytorch numpyhelper"
-        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python_version: ["3.9", "3.10", "3.11", "3.12"]
         os:
-          - ubuntu-22.04
+          - ubuntu-24.04
     runs-on: ${{ matrix.os }}
     steps:
       - name: checkout

Dockerfile

@@ -1,54 +1,61 @@
-# Base image
-ARG BASE_IMG=python:3.10-slim
-FROM $BASE_IMG
+# Stage 1: Builder
+ARG BASE_IMG=python:3.12-slim
+FROM $BASE_IMG as builder
 
 ARG GRPC_HEALTH_PROBE_VERSION=""
-
-# Requirements (use MNIST Keras as default)
 ARG REQUIREMENTS=""
 
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends python3-dev gcc wget \
+  && rm -rf /var/lib/apt/lists/*
+
 # Add FEDn and default configs
-COPY . /app
-COPY config/settings-client.yaml.template /app/config/settings-client.yaml
-COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
-COPY config/settings-hooks.yaml.template /app/config/settings-hooks.yaml
-COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
-COPY $REQUIREMENTS /app/config/requirements.txt
+COPY . /build
+COPY $REQUIREMENTS /build/requirements.txt
 
-# Install developer tools (needed for psutil)
-RUN apt-get update && apt-get install -y python3-dev gcc
+# Install dependencies
+RUN python -m venv /venv \
+  && /venv/bin/pip install --upgrade pip \
+  && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \
+  && /venv/bin/pip install --no-cache-dir . \
+  && if [[ ! -z "$REQUIREMENTS" ]]; then \
+  /venv/bin/pip install --no-cache-dir -r /build/requirements.txt; \
+  fi \
+  && rm -rf /build/requirements.txt
 
-# Install grpc health probe checker
+
+# Install grpc health probe
 RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \
-  apt-get install -y wget && \
-  wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
-  chmod +x /bin/grpc_health_probe && \
-  apt-get remove -y wget && apt autoremove -y; \
-  else \
-  echo "No grpc_health_probe version specified, skipping installation"; \
+  wget -qO /build/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+  chmod +x /build/grpc_health_probe; \
   fi
 
-# Setup working directory
+# Stage 2: Runtime
+FROM $BASE_IMG
+
 WORKDIR /app
 
-# Create FEDn app directory
-SHELL ["/bin/bash", "-c"]
-RUN mkdir -p /app \
-  && mkdir -p /app/client \
-  && mkdir -p /app/certs \
-  && mkdir -p /app/client/package \
-  && mkdir -p /app/certs \
-  #
-  # Install FEDn and requirements
-  && python -m venv /venv \
-  && /venv/bin/pip install --upgrade pip \
-  && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \
-  && /venv/bin/pip install --no-cache-dir -e . \
-  && if [[ ! -z "$REQUIREMENTS" ]]; then \
-  /venv/bin/pip install --no-cache-dir -r /app/config/requirements.txt; \
-  fi \
-  #
-  # Clean up
-  && rm -r /app/config/requirements.txt
+# Copy application and venv from the builder stage
+COPY --from=builder /venv /venv
+COPY --from=builder /build /app
+
+# Use a non-root user
+RUN set -ex \
+  # Create a non-root user
+  && addgroup --system --gid 1001 appgroup \
+  && adduser --system --uid 1001 --gid 1001 --no-create-home appuser \
+  # Creare application specific tmp directory, set ENV TMPDIR to /app/tmp
+  && mkdir -p /app/tmp \
+  && chown -R appuser:appgroup /venv /app \
+  # Upgrade the package index and install security upgrades
+  && apt-get update \
+  && apt-get upgrade -y \
+  && apt-get autoremove -y \
+  && apt-get clean -y \
+  && rm -rf /var/lib/apt/lists/*
+USER appuser
+
+ENTRYPOINT [ "/venv/bin/fedn" ]
 
-ENTRYPOINT [ "/venv/bin/fedn" ]

docker-compose.yaml

@@ -58,21 +58,23 @@ services:
       - USER=test
       - PROJECT=project
       - FLASK_DEBUG=1
-      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml
-      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml
+      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml.template
+      - FEDN_COMPUTE_PACKAGE_DIR=/app
+      - TMPDIR=/app/tmp
     build:
       context: .
       args:
-        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     depends_on:
       - minio
       - mongo
-    entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start"
+      - controller
+      - start
     ports:
       - 8092:8092
 
@@ -81,24 +83,27 @@ services:
     environment:
       - PYTHONUNBUFFERED=0
       - GET_HOSTS_FROM=dns
-      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml
-      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml
+      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml.template
       - HOOK_SERVICE_HOST=hook:12081
+      - TMPDIR=/app/tmp
     build:
       context: .
       args:
-        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
-        GRPC_HEALTH_PROBE_VERSION: v0.4.24
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+        GRPC_HEALTH_PROBE_VERSION: v0.4.35
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
-    entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn combiner start --init config/settings-combiner.yaml"
+      - combiner
+      - start
+      - --init
+      - config/settings-combiner.yaml.template
     ports:
       - 12080:12080
     healthcheck:
-      test: [ "CMD", "/bin/grpc_health_probe", "-addr=localhost:12080" ]
+      test: [ "CMD", "/app/grpc_health_probe", "-addr=localhost:12080" ]
       interval: 20s
       timeout: 10s
       retries: 5
@@ -110,11 +115,12 @@ services:
     container_name: hook
     environment:
       - GET_HOSTS_FROM=dns
+      - TMPDIR=/app/tmp
     build:
       context: .
       args:
-        BASE_IMG: ${BASE_IMG:-python:3.10-slim}
-        GRPC_HEALTH_PROBE_VERSION: v0.4.24
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+        GRPC_HEALTH_PROBE_VERSION: v0.4.35
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
@@ -141,9 +147,11 @@ services:
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
-    entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn client start --api-url http://api-server:8092"
+      - client
+      - start
+      - --api-url
+      - http://api-server:8092
     deploy:
       replicas: 0
     depends_on:

docs/conf.py

@@ -11,7 +11,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.19.0"
+release = "0.20.0"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

docs/quickstart.rst

@@ -9,7 +9,7 @@ Getting started with FEDn
 
 **Prerequisites**
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__ 
 
 

examples/FedSimSiam/README.rst

@@ -16,7 +16,7 @@ To run the example, follow the steps below. For a more detailed explanation, fol
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
 Creating the compute package and seed model

examples/FedSimSiam/client/python_env.yaml

@@ -9,7 +9,6 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - fedn

examples/flower-client/client/python_env.yaml

@@ -10,8 +10,7 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - fire==0.3.1
   - flwr-datasets[vision]==0.1.0

examples/huggingface/README.rst

@@ -27,7 +27,7 @@ To run the example, follow the steps below. For a more detailed explanation, fol
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
 Creating the compute package and seed model

examples/huggingface/client/python_env.yaml

@@ -9,9 +9,8 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - transformers
   - datasets
   - fedn

examples/mnist-keras/README.rst

@@ -8,7 +8,7 @@ This is a TF/Keras version of the PyTorch Quickstart Tutorial. For a step-by-ste
 Prerequisites
 -------------------------------------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 
 Creating the compute package and seed model
 -------------------------------------------

examples/mnist-pytorch-DPSGD/README.rst

@@ -9,7 +9,7 @@ We have expanded our baseline MNIST-PyTorch example by incorporating the Opacus
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
 Edit Differential Privacy budget

examples/mnist-pytorch-DPSGD/client/python_env.yaml

@@ -10,7 +10,6 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")
   - opacus

examples/mnist-pytorch/README.rst

@@ -9,7 +9,7 @@ The example is intented as a minimalistic quickstart to learn how to use FEDn.
 Prerequisites
 -------------
 
--  `Python >=3.8, <=3.12 <https://www.python.org/downloads>`__
+-  `Python >=3.9, <=3.12 <https://www.python.org/downloads>`__
 -  `A project in FEDn Studio  <https://fedn.scaleoutsystems.com/signup>`__   
 
 Creating the compute package and seed model

examples/mnist-pytorch/client/python_env.yaml

@@ -10,6 +10,5 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
-  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
+  - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64")

examples/monai-2D-mednist/README.rst

@@ -16,7 +16,7 @@ Prerequisites
 
 Using FEDn Studio:
 
--  `Python 3.8, 3.9, 3.10 or 3.11 <https://www.python.org/downloads>`__
+-  `Python 3.9, 3.10 or 3.11 <https://www.python.org/downloads>`__
 -  `A FEDn Studio account <https://fedn.scaleoutsystems.com/signup>`__
 
 

examples/monai-2D-mednist/client/python_env.yaml

@@ -10,7 +10,6 @@ dependencies:
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==1.26.4; python_version >= "3.9"
-  - numpy==1.24.4; python_version == "3.8"
+  - numpy==1.26.4
   - monai-weekly[pillow, tqdm]
   - scikit-learn  

fedn/network/controller/controlbase.py

@@ -261,8 +261,8 @@ def commit(self, model_id, model=None, session_id=None):
         """
         helper = self.get_helper()
         if model is not None:
-            logger.info("Saving model file temporarily to disk...")
             outfile_name = helper.save(model)
+            logger.info("Saving model file temporarily to {}".format(outfile_name))
             logger.info("CONTROL: Uploading model to Minio...")
             model_id = self.model_repository.set_model(outfile_name, is_file=True)
 

fedn/utils/process.py

@@ -101,9 +101,6 @@ def _exec_cmd(
 
     env = env if extra_env is None else {**os.environ, **extra_env}
 
-    # In Python < 3.8, `subprocess.Popen` doesn't accept a command containing path-like
-    # objects (e.g. `["ls", pathlib.Path("abc")]`) on Windows. To avoid this issue,
-    # stringify all elements in `cmd`. Note `str(pathlib.Path("abc"))` returns 'abc'.
     if isinstance(cmd, list):
         cmd = list(map(str, cmd))