Add client auth E2E test (#2979)

Co-authored-by: jafermarq <javier@flower.ai>
Co-authored-by: Daniel J. Beutel <daniel@flower.ai>

.github/workflows/e2e.yml

@@ -64,6 +64,8 @@ jobs:
 
           - directory: bare-https
 
+          - directory: bare-client-auth
+
           - directory: jax
 
           - directory: pytorch
@@ -130,17 +132,23 @@ jobs:
         if: ${{ matrix.dataset }}
         run: python -c "${{ matrix.dataset }}"
       - name: Run edge client test
+        if: ${{ matrix.directory != 'bare-client-auth' }}
         run: ./../test.sh "${{ matrix.directory }}"
       - name: Run virtual client test
+        if: ${{ matrix.directory != 'bare-client-auth' }}
         run: python simulation.py
       - name: Run driver test
+        if: ${{ matrix.directory != 'bare-client-auth' }}
         run: ./../test_driver.sh "${{ matrix.directory }}"
       - name: Run driver test with REST
         if: ${{ matrix.directory == 'bare' }}
         run: ./../test_driver.sh bare rest
       - name: Run driver test with SQLite database
         if: ${{ matrix.directory == 'bare' }}
         run: ./../test_driver.sh bare sqlite
+      - name: Run driver test with client authentication
+        if: ${{ matrix.directory == 'bare-client-auth' }}
+        run: ./../test_driver.sh bare client-auth
 
   strategies:
     runs-on: ubuntu-22.04

e2e/bare-client-auth/README.md

@@ -0,0 +1,3 @@
+# Bare Flower testing
+
+This directory is used for testing Flower in a bare minimum scenario, that is, with a dummy model and dummy operations. This is mainly to test the core functionality of Flower independently from any framework. It can easily be extended to test more complex communication set-ups.

e2e/bare-client-auth/certificate.conf

@@ -0,0 +1,20 @@
+[req]
+default_bits = 4096
+prompt = no
+default_md = sha256
+req_extensions = req_ext
+distinguished_name = dn
+
+[dn]
+C = DE
+ST = HH
+O = Flower
+CN = localhost
+
+[req_ext]
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+IP.1 = ::1
+IP.2 = 127.0.0.1

e2e/bare-client-auth/client.py

@@ -0,0 +1,30 @@
+import flwr as fl
+import numpy as np
+from pathlib import Path
+
+
+model_params = np.array([1])
+objective = 5
+
+# Define Flower client
+class FlowerClient(fl.client.NumPyClient):
+    def get_parameters(self, config):
+        return model_params
+
+    def fit(self, parameters, config):
+        model_params = parameters
+        model_params = [param * (objective/np.mean(param)) for param in model_params]
+        return model_params, 1, {}
+
+    def evaluate(self, parameters, config):
+        model_params = parameters
+        loss = min(np.abs(1 - np.mean(model_params)/objective), 1)
+        accuracy = 1 - loss
+        return loss, 1, {"accuracy": accuracy}
+
+def client_fn(cid):
+    return FlowerClient().to_client()
+
+app = fl.client.ClientApp(
+    client_fn=client_fn,
+)

e2e/bare-client-auth/driver.py

@@ -0,0 +1,12 @@
+import flwr as fl
+from pathlib import Path
+
+
+# Start Flower server
+hist = fl.server.start_driver(
+    server_address="127.0.0.1:9091",
+    config=fl.server.ServerConfig(num_rounds=3),
+    root_certificates=Path("certificates/ca.crt").read_bytes(),
+)
+
+assert hist.losses_distributed[-1][1] == 0

e2e/bare-client-auth/generate.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# This script will generate all certificates if ca.crt does not exist
+
+set -e
+# Change directory to the script's directory
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+CERT_DIR=certificates
+
+# Generate directories if not exists
+mkdir -p $CERT_DIR
+
+# Clearing any existing files in the certificates directory
+rm -f $CERT_DIR/*
+
+# Generate the root certificate authority key and certificate based on key
+openssl genrsa -out $CERT_DIR/ca.key 4096
+openssl req \
+    -new \
+    -x509 \
+    -key $CERT_DIR/ca.key \
+    -sha256 \
+    -subj "/C=DE/ST=HH/O=CA, Inc." \
+    -days 365 -out $CERT_DIR/ca.crt
+
+# Generate a new private key for the server
+openssl genrsa -out $CERT_DIR/server.key 4096
+
+# Create a signing CSR
+openssl req \
+    -new \
+    -key $CERT_DIR/server.key \
+    -out $CERT_DIR/server.csr \
+    -config certificate.conf
+
+# Generate a certificate for the server
+openssl x509 \
+    -req \
+    -in $CERT_DIR/server.csr \
+    -CA $CERT_DIR/ca.crt \
+    -CAkey $CERT_DIR/ca.key \
+    -CAcreateserial \
+    -out $CERT_DIR/server.pem \
+    -days 365 \
+    -sha256 \
+    -extfile certificate.conf \
+    -extensions req_ext
+
+KEY_DIR=keys
+
+mkdir -p $KEY_DIR
+
+rm -f $KEY_DIR/*
+
+ssh-keygen -t ecdsa -b 384 -N "" -f "${KEY_DIR}/server_credentials" -C ""
+
+generate_client_credentials() {
+    local num_clients=${1:-2} 
+    for ((i=1; i<=num_clients; i++))
+    do
+        ssh-keygen -t ecdsa -b 384 -N "" -f "${KEY_DIR}/client_credentials_$i" -C ""
+    done
+}
+
+generate_client_credentials "$1"
+
+printf "%s" "$(cat "${KEY_DIR}/client_credentials_1.pub" | sed 's/.$//')" > $KEY_DIR/client_public_keys.csv
+for ((i=2; i<=${1:-2}; i++))
+do
+    printf ",%s" "$(sed 's/.$//' < "${KEY_DIR}/client_credentials_$i.pub")" >> $KEY_DIR/client_public_keys.csv
+done
+printf "\n" >> $KEY_DIR/client_public_keys.csv

e2e/bare-client-auth/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["poetry-core>=1.4.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "bare_client_auth_test"
+version = "0.1.0"
+description = "Client-auth-enabled bare Federated Learning test with Flower"
+authors = ["The Flower Authors <hello@flower.ai>"]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+flwr = { path = "../../", develop = true }

e2e/bare-client-auth/server.py

@@ -0,0 +1,15 @@
+import flwr as fl
+from pathlib import Path
+
+
+hist = fl.server.start_server(
+    server_address="127.0.0.1:8080",
+    config=fl.server.ServerConfig(num_rounds=3),
+    certificates=(
+        Path("certificates/ca.crt").read_bytes(),
+        Path("certificates/server.pem").read_bytes(),
+        Path("certificates/server.key").read_bytes(),
+    )
+)
+
+assert hist.losses_distributed[-1][1] == 0 or (hist.losses_distributed[0][1] / hist.losses_distributed[-1][1]) >= 0.98

e2e/test_driver.sh

@@ -18,28 +18,48 @@ case "$2" in
     rest_arg="--rest"
     server_address="http://localhost:9093"
     db_arg="--database :flwr-in-memory-state:"
+    server_auth=""
+    client_auth_1=""
+    client_auth_2=""
     ;;
   sqlite)
     rest_arg=""
     server_address="127.0.0.1:9092"
     db_arg="--database $(date +%s).db"
+    server_auth=""
+    client_auth_1=""
+    client_auth_2=""
+    ;;
+  client-auth)
+    ./generate.sh
+    rest_arg=""
+    server_address="127.0.0.1:9092"
+    db_arg="--database :flwr-in-memory-state:"
+    server_arg="--certificates certificates/ca.crt certificates/server.pem certificates/server.key"
+    client_arg="--root-certificates certificates/ca.crt"
+    server_auth="--require-client-authentication keys/client_public_keys.csv keys/server_credentials keys/server_credentials.pub"
+    client_auth_1="--authentication-keys keys/client_credentials_1 keys/client_credentials_1.pub"
+    client_auth_2="--authentication-keys keys/client_credentials_2 keys/client_credentials_2.pub"
     ;;
   *)
     rest_arg=""
     server_address="127.0.0.1:9092"
     db_arg="--database :flwr-in-memory-state:"
+    server_auth=""
+    client_auth_1=""
+    client_auth_2=""
     ;;
 esac
 
-timeout 2m flower-superlink $server_arg $db_arg $rest_arg &
+timeout 2m flower-superlink $server_arg $db_arg $rest_arg $server_auth &
 sl_pid=$!
 sleep 3
 
-timeout 2m flower-client-app client:app $client_arg $rest_arg --server $server_address &
+timeout 2m flower-client-app client:app $client_arg $rest_arg --server $server_address $client_auth_1 &
 cl1_pid=$!
 sleep 3
 
-timeout 2m flower-client-app client:app $client_arg $rest_arg --server $server_address &
+timeout 2m flower-client-app client:app $client_arg $rest_arg --server $server_address $client_auth_2 &
 cl2_pid=$!
 sleep 3