diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index c379eb07c..d7a097558 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -4,7 +4,7 @@ on: push jobs: code-checks: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: checkout uses: actions/checkout@v4 diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 22c38dd1b..867860c83 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -23,9 +23,9 @@ jobs: to_test: - "mnist-keras numpyhelper" - "mnist-pytorch numpyhelper" - python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python_version: ["3.9", "3.10", "3.11", "3.12"] os: - - ubuntu-22.04 + - ubuntu-24.04 runs-on: ${{ matrix.os }} steps: - name: checkout diff --git a/Dockerfile b/Dockerfile index b651dbea4..169fc5097 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,54 +1,61 @@ -# Base image -ARG BASE_IMG=python:3.10-slim -FROM $BASE_IMG +# Stage 1: Builder +ARG BASE_IMG=python:3.12-slim +FROM $BASE_IMG as builder ARG GRPC_HEALTH_PROBE_VERSION="" - -# Requirements (use MNIST Keras as default) ARG REQUIREMENTS="" +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends python3-dev gcc wget \ + && rm -rf /var/lib/apt/lists/* + # Add FEDn and default configs -COPY . /app -COPY config/settings-client.yaml.template /app/config/settings-client.yaml -COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml -COPY config/settings-hooks.yaml.template /app/config/settings-hooks.yaml -COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml -COPY $REQUIREMENTS /app/config/requirements.txt +COPY . /build +COPY $REQUIREMENTS /build/requirements.txt -# Install developer tools (needed for psutil) -RUN apt-get update && apt-get install -y python3-dev gcc +# Install dependencies +RUN python -m venv /venv \ + && /venv/bin/pip install --upgrade pip \ + && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \ + && /venv/bin/pip install --no-cache-dir . \ + && if [[ ! -z "$REQUIREMENTS" ]]; then \ + /venv/bin/pip install --no-cache-dir -r /build/requirements.txt; \ + fi \ + && rm -rf /build/requirements.txt -# Install grpc health probe checker + +# Install grpc health probe RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \ - apt-get install -y wget && \ - wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ - chmod +x /bin/grpc_health_probe && \ - apt-get remove -y wget && apt autoremove -y; \ - else \ - echo "No grpc_health_probe version specified, skipping installation"; \ + wget -qO /build/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ + chmod +x /build/grpc_health_probe; \ fi -# Setup working directory +# Stage 2: Runtime +FROM $BASE_IMG + WORKDIR /app -# Create FEDn app directory -SHELL ["/bin/bash", "-c"] -RUN mkdir -p /app \ - && mkdir -p /app/client \ - && mkdir -p /app/certs \ - && mkdir -p /app/client/package \ - && mkdir -p /app/certs \ - # - # Install FEDn and requirements - && python -m venv /venv \ - && /venv/bin/pip install --upgrade pip \ - && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \ - && /venv/bin/pip install --no-cache-dir -e . \ - && if [[ ! -z "$REQUIREMENTS" ]]; then \ - /venv/bin/pip install --no-cache-dir -r /app/config/requirements.txt; \ - fi \ - # - # Clean up - && rm -r /app/config/requirements.txt +# Copy application and venv from the builder stage +COPY --from=builder /venv /venv +COPY --from=builder /build /app + +# Use a non-root user +RUN set -ex \ + # Create a non-root user + && addgroup --system --gid 1001 appgroup \ + && adduser --system --uid 1001 --gid 1001 --no-create-home appuser \ + # Creare application specific tmp directory, set ENV TMPDIR to /app/tmp + && mkdir -p /app/tmp \ + && chown -R appuser:appgroup /venv /app \ + # Upgrade the package index and install security upgrades + && apt-get update \ + && apt-get upgrade -y \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* +USER appuser + +ENTRYPOINT [ "/venv/bin/fedn" ] -ENTRYPOINT [ "/venv/bin/fedn" ] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index f020d9a0c..598aad0cb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,21 +58,23 @@ services: - USER=test - PROJECT=project - FLASK_DEBUG=1 - - STATESTORE_CONFIG=/app/config/settings-reducer.yaml - - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml + - STATESTORE_CONFIG=/app/config/settings-reducer.yaml.template + - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml.template + - FEDN_COMPUTE_PACKAGE_DIR=/app + - TMPDIR=/app/tmp build: context: . args: - BASE_IMG: ${BASE_IMG:-python:3.10-slim} + BASE_IMG: ${BASE_IMG:-python:3.12-slim} working_dir: /app volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn depends_on: - minio - mongo - entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start" + - controller + - start ports: - 8092:8092 @@ -81,24 +83,27 @@ services: environment: - PYTHONUNBUFFERED=0 - GET_HOSTS_FROM=dns - - STATESTORE_CONFIG=/app/config/settings-combiner.yaml - - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml + - STATESTORE_CONFIG=/app/config/settings-combiner.yaml.template + - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml.template - HOOK_SERVICE_HOST=hook:12081 + - TMPDIR=/app/tmp build: context: . args: - BASE_IMG: ${BASE_IMG:-python:3.10-slim} - GRPC_HEALTH_PROBE_VERSION: v0.4.24 + BASE_IMG: ${BASE_IMG:-python:3.12-slim} + GRPC_HEALTH_PROBE_VERSION: v0.4.35 working_dir: /app volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn - entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn combiner start --init config/settings-combiner.yaml" + - combiner + - start + - --init + - config/settings-combiner.yaml.template ports: - 12080:12080 healthcheck: - test: [ "CMD", "/bin/grpc_health_probe", "-addr=localhost:12080" ] + test: [ "CMD", "/app/grpc_health_probe", "-addr=localhost:12080" ] interval: 20s timeout: 10s retries: 5 @@ -110,11 +115,12 @@ services: container_name: hook environment: - GET_HOSTS_FROM=dns + - TMPDIR=/app/tmp build: context: . args: - BASE_IMG: ${BASE_IMG:-python:3.10-slim} - GRPC_HEALTH_PROBE_VERSION: v0.4.24 + BASE_IMG: ${BASE_IMG:-python:3.12-slim} + GRPC_HEALTH_PROBE_VERSION: v0.4.35 working_dir: /app volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn @@ -141,9 +147,11 @@ services: working_dir: /app volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn - entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn client start --api-url http://api-server:8092" + - client + - start + - --api-url + - http://api-server:8092 deploy: replicas: 0 depends_on: diff --git a/docs/conf.py b/docs/conf.py index c48e378f4..f451f8b74 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,7 +11,7 @@ author = "Scaleout Systems AB" # The full version, including alpha/beta/rc tags -release = "0.19.0" +release = "0.20.0" # Add any Sphinx extension module names here, as strings extensions = [ diff --git a/docs/quickstart.rst b/docs/quickstart.rst index e7641f0d4..0a309bb32 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -9,7 +9,7 @@ Getting started with FEDn **Prerequisites** -- `Python >=3.8, <=3.12 `__ +- `Python >=3.9, <=3.12 `__ - `A FEDn Studio account `__ diff --git a/examples/FedSimSiam/README.rst b/examples/FedSimSiam/README.rst index 5831fd3ea..47fa93c6b 100644 --- a/examples/FedSimSiam/README.rst +++ b/examples/FedSimSiam/README.rst @@ -16,7 +16,7 @@ To run the example, follow the steps below. For a more detailed explanation, fol Prerequisites ------------- -- `Python >=3.8, <=3.12 `__ +- `Python >=3.9, <=3.12 `__ - `A project in FEDn Studio `__ Creating the compute package and seed model diff --git a/examples/FedSimSiam/client/python_env.yaml b/examples/FedSimSiam/client/python_env.yaml index d728b82be..45f23ad30 100644 --- a/examples/FedSimSiam/client/python_env.yaml +++ b/examples/FedSimSiam/client/python_env.yaml @@ -9,7 +9,6 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9") - - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") - - numpy==1.24.4; python_version == "3.8" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64") - fedn diff --git a/examples/flower-client/client/python_env.yaml b/examples/flower-client/client/python_env.yaml index a82e7e50d..06b00186c 100644 --- a/examples/flower-client/client/python_env.yaml +++ b/examples/flower-client/client/python_env.yaml @@ -10,8 +10,7 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9") - - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") - - numpy==1.24.4; python_version == "3.8" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64") - fire==0.3.1 - flwr-datasets[vision]==0.1.0 \ No newline at end of file diff --git a/examples/huggingface/README.rst b/examples/huggingface/README.rst index eaaad3254..68bceb685 100644 --- a/examples/huggingface/README.rst +++ b/examples/huggingface/README.rst @@ -27,7 +27,7 @@ To run the example, follow the steps below. For a more detailed explanation, fol Prerequisites ------------- -- `Python >=3.8, <=3.12 `__ +- `Python >=3.9, <=3.12 `__ - `A project in FEDn Studio `__ Creating the compute package and seed model diff --git a/examples/huggingface/client/python_env.yaml b/examples/huggingface/client/python_env.yaml index 87ee6f32d..6cc2925b4 100644 --- a/examples/huggingface/client/python_env.yaml +++ b/examples/huggingface/client/python_env.yaml @@ -9,9 +9,8 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9") - - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") - - numpy==1.24.4; python_version == "3.8" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64") - transformers - datasets - fedn diff --git a/examples/mnist-keras/README.rst b/examples/mnist-keras/README.rst index aaf13c21d..741813362 100644 --- a/examples/mnist-keras/README.rst +++ b/examples/mnist-keras/README.rst @@ -8,7 +8,7 @@ This is a TF/Keras version of the PyTorch Quickstart Tutorial. For a step-by-ste Prerequisites ------------------------------------------- -- `Python >=3.8, <=3.12 `__ +- `Python >=3.9, <=3.12 `__ Creating the compute package and seed model ------------------------------------------- diff --git a/examples/mnist-pytorch-DPSGD/README.rst b/examples/mnist-pytorch-DPSGD/README.rst index 88220584a..cfe03f081 100644 --- a/examples/mnist-pytorch-DPSGD/README.rst +++ b/examples/mnist-pytorch-DPSGD/README.rst @@ -9,7 +9,7 @@ We have expanded our baseline MNIST-PyTorch example by incorporating the Opacus Prerequisites ------------- -- `Python >=3.8, <=3.12 `__ +- `Python >=3.9, <=3.12 `__ - `A project in FEDn Studio `__ Edit Differential Privacy budget diff --git a/examples/mnist-pytorch-DPSGD/client/python_env.yaml b/examples/mnist-pytorch-DPSGD/client/python_env.yaml index 13d586102..526022145 100644 --- a/examples/mnist-pytorch-DPSGD/client/python_env.yaml +++ b/examples/mnist-pytorch-DPSGD/client/python_env.yaml @@ -10,7 +10,6 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9") - - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") - - numpy==1.24.4; python_version == "3.8" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64") - opacus diff --git a/examples/mnist-pytorch/README.rst b/examples/mnist-pytorch/README.rst index 990b902b2..1c0afc5d0 100644 --- a/examples/mnist-pytorch/README.rst +++ b/examples/mnist-pytorch/README.rst @@ -9,7 +9,7 @@ The example is intented as a minimalistic quickstart to learn how to use FEDn. Prerequisites ------------- -- `Python >=3.8, <=3.12 `__ +- `Python >=3.9, <=3.12 `__ - `A project in FEDn Studio `__ Creating the compute package and seed model diff --git a/examples/mnist-pytorch/client/python_env.yaml b/examples/mnist-pytorch/client/python_env.yaml index 272b196ea..7a35ff7a2 100644 --- a/examples/mnist-pytorch/client/python_env.yaml +++ b/examples/mnist-pytorch/client/python_env.yaml @@ -10,6 +10,5 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9") - - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9") - - numpy==1.24.4; python_version == "3.8" + - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") + - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64") diff --git a/examples/monai-2D-mednist/README.rst b/examples/monai-2D-mednist/README.rst index f61820682..00eca5321 100644 --- a/examples/monai-2D-mednist/README.rst +++ b/examples/monai-2D-mednist/README.rst @@ -16,7 +16,7 @@ Prerequisites Using FEDn Studio: -- `Python 3.8, 3.9, 3.10 or 3.11 `__ +- `Python 3.9, 3.10 or 3.11 `__ - `A FEDn Studio account `__ diff --git a/examples/monai-2D-mednist/client/python_env.yaml b/examples/monai-2D-mednist/client/python_env.yaml index 389b3a42a..546f1ffbe 100644 --- a/examples/monai-2D-mednist/client/python_env.yaml +++ b/examples/monai-2D-mednist/client/python_env.yaml @@ -10,7 +10,6 @@ dependencies: - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64" - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux") - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64" - - numpy==1.26.4; python_version >= "3.9" - - numpy==1.24.4; python_version == "3.8" + - numpy==1.26.4 - monai-weekly[pillow, tqdm] - scikit-learn diff --git a/fedn/network/controller/controlbase.py b/fedn/network/controller/controlbase.py index 297efd426..397a117bb 100644 --- a/fedn/network/controller/controlbase.py +++ b/fedn/network/controller/controlbase.py @@ -261,8 +261,8 @@ def commit(self, model_id, model=None, session_id=None): """ helper = self.get_helper() if model is not None: - logger.info("Saving model file temporarily to disk...") outfile_name = helper.save(model) + logger.info("Saving model file temporarily to {}".format(outfile_name)) logger.info("CONTROL: Uploading model to Minio...") model_id = self.model_repository.set_model(outfile_name, is_file=True) diff --git a/fedn/utils/process.py b/fedn/utils/process.py index c2574a760..b08af99b6 100644 --- a/fedn/utils/process.py +++ b/fedn/utils/process.py @@ -101,9 +101,6 @@ def _exec_cmd( env = env if extra_env is None else {**os.environ, **extra_env} - # In Python < 3.8, `subprocess.Popen` doesn't accept a command containing path-like - # objects (e.g. `["ls", pathlib.Path("abc")]`) on Windows. To avoid this issue, - # stringify all elements in `cmd`. Note `str(pathlib.Path("abc"))` returns 'abc'. if isinstance(cmd, list): cmd = list(map(str, cmd)) diff --git a/pyproject.toml b/pyproject.toml index beb511290..94ad2ffdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "fedn" -version = "0.19.0" +version = "0.20.0" description = "Scaleout Federated Learning" authors = [{ name = "Scaleout Systems AB", email = "contact@scaleoutsystems.com" }] readme = "README.rst" @@ -20,30 +20,29 @@ keywords = [ ] classifiers = [ "Natural Language :: English", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] -requires-python = '>=3.8,<3.13' +requires-python = '>=3.9,<3.13' dependencies = [ "requests", "urllib3>=1.26.4", "gunicorn>=20.0.4", "minio", - "grpcio>=1.60,<1.67", - "grpcio-tools>=1.60,<1.67", + "grpcio>=1.60,<1.69", + "grpcio-tools>=1.60,<1.69", "numpy>=1.21.6", "protobuf>=5.0.0,<5.29.0", "pymongo", - "Flask==3.0.3", + "Flask==3.1.0", "pyjwt", "pyopenssl", "psutil", "click==8.1.7", - "grpcio-health-checking>=1.60,<1.67", + "grpcio-health-checking>=1.60,<1.69", "pyyaml", "plotly", "virtualenv",