Milestone2-3ORG-UC1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rhgpM92PMCd-"
   },
   "source": [
    "# Milestone 2: FL Inter-company\n",
    "\n",
    "Train a single model in a federated way with the companies. This way, it will keep the privacy of each company, but not within each company. It measures the peformance compared to the baselines of the companies and the centralized approach.\n",
    "\n",
    "The notebook is similar to Milestone1-IntraCompany but with the modifications performed to allow that each client (two in total) have the complete data from one of the companies."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rhgpM92PMCd-"
   },
   "source": [
    "# Scenario config with Flower\n",
    "\n",
    "Use of the 'simulation' module (version 0.18.0) which removes the scalability restrictions presented in previous versions\n",
    "\n",
    "It is necessary to install Flower with the 'simulation' extra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BXG21C3dLj6i",
    "outputId": "f60586a0-3fc5-4801-919d-01977224a39b",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !pip install -U flwr[\"simulation\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "oKvjox6uMkhj"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "\n",
    "# Make TensorFlow logs less verbose\n",
    "os.environ[\"TF_CPP_MIN_LOG_LEVEL\"] = \"3\"\n",
    "\n",
    "import flwr as fl\n",
    "import tensorflow as tf\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, BatchNormalization\n",
    "from tensorflow.keras.optimizers import Adadelta\n",
    "\n",
    "from typing import Dict\n",
    "\n",
    "from flwr.common.logger import log\n",
    "from logging import INFO\n",
    "from csv import writer\n",
    "\n",
    "\n",
    "from numpy.random import seed\n",
    "from tensorflow.keras.utils import set_random_seed\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "vE3mqBs0NHZi"
   },
   "outputs": [],
   "source": [
    "class FlowerClient(fl.client.NumPyClient):\n",
    "    def __init__(self, cid, model, x_train, y_train, x_val, y_val) -> None:\n",
    "        self.cid = cid\n",
    "        self.model = model\n",
    "        self.x_train, self.y_train = x_train, y_train\n",
    "        self.x_val, self.y_val = x_val, y_val\n",
    "\n",
    "    def get_parameters(self):\n",
    "        return self.model.get_weights()\n",
    "\n",
    "    def fit(self, parameters, config):\n",
    "        self.model.set_weights(parameters)\n",
    "        \n",
    "        seed(1)\n",
    "        set_random_seed(2)\n",
    "        \n",
    "        self.model.fit(self.x_train, self.y_train,\n",
    "                       epochs=int(config[\"epochs\"]),\n",
    "                       batch_size=32,\n",
    "                       verbose=0)\n",
    "        \n",
    "        return self.model.get_weights(), len(self.x_train), {}\n",
    "\n",
    "    def evaluate(self, parameters, config):\n",
    "        self.model.set_weights(parameters)\n",
    "        \n",
    "        loss, acc = self.model.evaluate(self.x_val, self.y_val, verbose=0)\n",
    "        \n",
    "        return loss, len(self.x_val), {\"accuracy\": acc, \"client\": self.cid}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Load data from a particular client '''\n",
    "def prepare_model_data(client_file):\n",
    "    df = pd.read_csv(client_file)\n",
    "    \n",
    "    train, test = train_test_split(df, test_size=0.30, random_state=42)\n",
    "    \n",
    "    X_train = train[['psd_delta', 'psd_theta', 'psd_alpha', 'psd_beta', 'psd_gamma','eog_blinks', 'eog_var']]\n",
    "    X_test = test[['psd_delta', 'psd_theta', 'psd_alpha', 'psd_beta', 'psd_gamma','eog_blinks', 'eog_var']]\n",
    "    y_train = train['y_class']\n",
    "    y_test = test['y_class']\n",
    "    \n",
    "    scaler = StandardScaler()\n",
    "\n",
    "    X_train = scaler.fit_transform(X_train)\n",
    "    X_test = scaler.transform(X_test)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "''' FUNCTION TO LOAD DATA FROM A COMPANY '''\n",
    "def get_data_company(company):\n",
    "    base_path = \"./data/horizontal_v3\"\n",
    "    # Load and process data from all its clients\n",
    "    clients = os.listdir(f\"{base_path}/{company}/\")\n",
    "\n",
    "    for client_id in UNSEEN_CLIENTS:\n",
    "        try:\n",
    "            clients.remove(f'client_{client_id}.csv')\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    X_train, X_test, y_train, y_test = prepare_model_data(f'{base_path}/{company}/{clients[0]}')\n",
    "\n",
    "    # Must work even when only exists clients[0]\n",
    "    for file in clients[1:]:\n",
    "        path = f'{base_path}/{company}/{file}'\n",
    "        X_train_act, X_test_act, y_train_act, y_test_act = prepare_model_data(path)\n",
    "\n",
    "        X_train = np.vstack((X_train, X_train_act))\n",
    "        X_test = np.vstack((X_test, X_test_act))\n",
    "        y_train = np.concatenate((y_train, y_train_act))\n",
    "        y_test = np.concatenate((y_test, y_test_act))\n",
    "        \n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "OtlBA2OFO0tf"
   },
   "outputs": [],
   "source": [
    "def client_fn(cid: str) -> fl.client.Client:\n",
    "    # Model best hyperparameters (See notebook Milestone0-Optimization-Baseline)\n",
    "    neurons = 36\n",
    "    activation = \"relu\"\n",
    "    learning_rate = 0.180165\n",
    "    optimizer = Adadelta(learning_rate=learning_rate)\n",
    "    \n",
    "    input_shape = (7,)\n",
    "    \n",
    "    # Create model\n",
    "    model = Sequential()\n",
    "    \n",
    "    model.add(Dense(neurons, input_shape=input_shape, activation=activation))\n",
    "    \n",
    "    model.add(BatchNormalization())\n",
    "        \n",
    "    model.add(Dense(neurons, activation=activation))\n",
    "    model.add(Dense(neurons, activation=activation))\n",
    "    model.add(Dense(neurons, activation=activation))\n",
    "    \n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "\n",
    "    # Load data partition    \n",
    "    x_train_cid, x_val_cid, y_train_cid, y_val_cid = get_data_company(cid)\n",
    "\n",
    "    # Create and return client\n",
    "    return FlowerClient(cid, model, x_train_cid, y_train_cid, x_val_cid, y_val_cid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveModelStrategy(fl.server.strategy.FedAvg):\n",
    "    def aggregate_fit(self, rnd, results, failures):\n",
    "        aggregated_weights = super().aggregate_fit(rnd, results, failures)\n",
    "            \n",
    "        return aggregated_weights\n",
    "\n",
    "    def aggregate_evaluate(self, rnd, results, failures):\n",
    "        super_result = super().aggregate_evaluate(rnd, results, failures)\n",
    "        \n",
    "        log(\n",
    "            INFO,\n",
    "            f\"round-{rnd}-EVALUATION\"\n",
    "        )\n",
    "        \n",
    "        accuracy = []\n",
    "        data = {}\n",
    "        for r in results:\n",
    "            acc = r[1].metrics[\"accuracy\"]\n",
    "            client = r[1].metrics[\"client\"]\n",
    "            data[client] = acc\n",
    "            accuracy.append(acc)\n",
    "        \n",
    "        df = pd.DataFrame(data, index=[0], columns=sorted(data.keys()))\n",
    "        df.to_csv(f\"./results/milestone2.csv\", mode='a', index=False, header=False)\n",
    "        \n",
    "        log(\n",
    "            INFO,\n",
    "            sorted(data.items())\n",
    "        )\n",
    "        \n",
    "        np.array(accuracy)\n",
    "        log(\n",
    "            INFO,\n",
    "            f\"Aggregated accuracy: {np.mean(accuracy)} +- {np.std(accuracy)}\"\n",
    "        )\n",
    "        \n",
    "        return super_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed(1)\n",
    "set_random_seed(2)\n",
    "\n",
    "neurons = 36\n",
    "activation = \"relu\"\n",
    "learning_rate = 0.180165\n",
    "optimizer = Adadelta(learning_rate=learning_rate)\n",
    "\n",
    "input_shape = (7,)\n",
    "\n",
    "# Create model\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Dense(neurons, input_shape=input_shape, activation=activation))\n",
    "\n",
    "model.add(BatchNormalization())\n",
    "\n",
    "model.add(Dense(neurons, activation=activation))\n",
    "model.add(Dense(neurons, activation=activation))\n",
    "model.add(Dense(neurons, activation=activation))\n",
    "\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "\n",
    "# Get model weights as a list of NumPy ndarray's\n",
    "weights = model.get_weights()\n",
    "# Serialize ndarrays to `Parameters`\n",
    "parameters = fl.common.weights_to_parameters(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter config for training from the server\n",
    "def fit_config(rnd: int) -> Dict[str, str]:\n",
    "    config = {\n",
    "        \"round\": str(rnd),\n",
    "        \"epochs\": str(1) \n",
    "    }\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "collapsed": true,
    "id": "0Yxjysu5PM-A",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "cf5f1bab-0d94-4876-bd6a-cdb0652826ff",
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO flower 2022-04-14 23:02:05,950 | app.py:144 | Ray initialized with resources: {'GPU': 1.0, 'object_store_memory': 5872553164.0, 'memory': 11745106331.0, 'CPU': 8.0, 'node:127.0.0.1': 1.0}\n",
      "INFO flower 2022-04-14 23:02:05,951 | app.py:153 | Starting Flower simulation running: {'num_rounds': 50}\n",
      "INFO flower 2022-04-14 23:02:05,952 | server.py:128 | Initializing global parameters\n",
      "INFO flower 2022-04-14 23:02:05,952 | server.py:323 | Using initial parameters provided by strategy\n",
      "INFO flower 2022-04-14 23:02:05,952 | server.py:130 | Evaluating initial parameters\n",
      "INFO flower 2022-04-14 23:02:05,953 | server.py:143 | FL starting\n",
      "DEBUG flower 2022-04-14 23:02:05,953 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:14,815 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:14,821 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:16,048 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:16,048 | 2494052621.py:15 | round-1-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:16,051 | 2494052621.py:31 | [('empresa_1', 0.6392874717712402), ('empresa_2', 0.7642413973808289), ('empresa_3', 0.5496312975883484)]\n",
      "INFO flower 2022-04-14 23:02:16,052 | 2494052621.py:37 | Aggregated accuracy: 0.6510533889134725 +- 0.08800833800605869\n",
      "DEBUG flower 2022-04-14 23:02:16,052 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:19,963 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:19,970 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:21,196 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:21,197 | 2494052621.py:15 | round-2-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:21,199 | 2494052621.py:31 | [('empresa_1', 0.7046017050743103), ('empresa_2', 0.7213761806488037), ('empresa_3', 0.6954055428504944)]\n",
      "INFO flower 2022-04-14 23:02:21,199 | 2494052621.py:37 | Aggregated accuracy: 0.7071278095245361 +- 0.010751880755874956\n",
      "DEBUG flower 2022-04-14 23:02:21,200 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:25,284 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:25,291 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:26,504 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:26,504 | 2494052621.py:15 | round-3-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:26,506 | 2494052621.py:31 | [('empresa_1', 0.7021276354789734), ('empresa_2', 0.7021996378898621), ('empresa_3', 0.7254679799079895)]\n",
      "INFO flower 2022-04-14 23:02:26,507 | 2494052621.py:37 | Aggregated accuracy: 0.709931751092275 +- 0.010985812075808567\n",
      "DEBUG flower 2022-04-14 23:02:26,507 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:30,470 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:30,476 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:31,677 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:31,678 | 2494052621.py:15 | round-4-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:31,680 | 2494052621.py:31 | [('empresa_1', 0.7065808773040771), ('empresa_2', 0.7010716199874878), ('empresa_3', 0.7368122339248657)]\n",
      "INFO flower 2022-04-14 23:02:31,681 | 2494052621.py:37 | Aggregated accuracy: 0.7148215770721436 +- 0.01571156093785148\n",
      "DEBUG flower 2022-04-14 23:02:31,681 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:35,671 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:35,678 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:36,779 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:36,780 | 2494052621.py:15 | round-5-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:36,782 | 2494052621.py:31 | [('empresa_1', 0.7090549468994141), ('empresa_2', 0.7038916945457458), ('empresa_3', 0.7430515885353088)]\n",
      "INFO flower 2022-04-14 23:02:36,782 | 2494052621.py:37 | Aggregated accuracy: 0.7186660766601562 +- 0.017371522400695935\n",
      "DEBUG flower 2022-04-14 23:02:36,782 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:40,447 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:40,454 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:41,649 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:41,649 | 2494052621.py:15 | round-6-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:41,651 | 2494052621.py:31 | [('empresa_1', 0.7144977450370789), ('empresa_2', 0.7055837512016296), ('empresa_3', 0.7441860437393188)]\n",
      "INFO flower 2022-04-14 23:02:41,652 | 2494052621.py:37 | Aggregated accuracy: 0.7214225133260092 +- 0.016502495956326618\n",
      "DEBUG flower 2022-04-14 23:02:41,652 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:45,587 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:45,594 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:46,793 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:46,793 | 2494052621.py:15 | round-7-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:46,795 | 2494052621.py:31 | [('empresa_1', 0.7179614305496216), ('empresa_2', 0.7055837512016296), ('empresa_3', 0.7441860437393188)]\n",
      "INFO flower 2022-04-14 23:02:46,796 | 2494052621.py:37 | Aggregated accuracy: 0.7225770751635233 +- 0.016093733328901953\n",
      "DEBUG flower 2022-04-14 23:02:46,797 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:50,594 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:50,601 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:51,914 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:51,914 | 2494052621.py:15 | round-8-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:51,917 | 2494052621.py:31 | [('empresa_1', 0.7184562087059021), ('empresa_2', 0.7072758078575134), ('empresa_3', 0.740215539932251)]\n",
      "INFO flower 2022-04-14 23:02:51,917 | 2494052621.py:37 | Aggregated accuracy: 0.7219825188318888 +- 0.013676808463413551\n",
      "DEBUG flower 2022-04-14 23:02:51,918 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:55,986 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:02:55,993 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:02:57,254 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:02:57,255 | 2494052621.py:15 | round-9-EVALUATION\n",
      "INFO flower 2022-04-14 23:02:57,257 | 2494052621.py:31 | [('empresa_1', 0.7179614305496216), ('empresa_2', 0.7089678645133972), ('empresa_3', 0.7339761853218079)]\n",
      "INFO flower 2022-04-14 23:02:57,257 | 2494052621.py:37 | Aggregated accuracy: 0.7203018267949423 +- 0.010342859641734467\n",
      "DEBUG flower 2022-04-14 23:02:57,258 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:01,346 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:01,353 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:02,664 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:02,664 | 2494052621.py:15 | round-10-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:02,666 | 2494052621.py:31 | [('empresa_1', 0.7204354405403137), ('empresa_2', 0.7140439748764038), ('empresa_3', 0.7317073345184326)]\n",
      "INFO flower 2022-04-14 23:03:02,667 | 2494052621.py:37 | Aggregated accuracy: 0.7220622499783834 +- 0.00730221200147633\n",
      "DEBUG flower 2022-04-14 23:03:02,667 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:06,673 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:06,680 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:07,936 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:07,937 | 2494052621.py:15 | round-11-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:07,939 | 2494052621.py:31 | [('empresa_1', 0.7214250564575195), ('empresa_2', 0.7140439748764038), ('empresa_3', 0.7345433831214905)]\n",
      "INFO flower 2022-04-14 23:03:07,939 | 2494052621.py:37 | Aggregated accuracy: 0.7233374714851379 +- 0.008477398698031427\n",
      "DEBUG flower 2022-04-14 23:03:07,940 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:11,776 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:11,783 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:13,013 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:13,013 | 2494052621.py:15 | round-12-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:13,015 | 2494052621.py:31 | [('empresa_1', 0.7229094505310059), ('empresa_2', 0.71235191822052), ('empresa_3', 0.7356778383255005)]\n",
      "INFO flower 2022-04-14 23:03:13,016 | 2494052621.py:37 | Aggregated accuracy: 0.7236464023590088 +- 0.00953701423170378\n",
      "DEBUG flower 2022-04-14 23:03:13,016 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:16,720 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:16,727 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:18,097 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:18,097 | 2494052621.py:15 | round-13-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:18,099 | 2494052621.py:31 | [('empresa_1', 0.7204354405403137), ('empresa_2', 0.7134799957275391), ('empresa_3', 0.7339761853218079)]\n",
      "INFO flower 2022-04-14 23:03:18,100 | 2494052621.py:37 | Aggregated accuracy: 0.7226305405298868 +- 0.008510279860036772\n",
      "DEBUG flower 2022-04-14 23:03:18,100 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:21,613 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:21,621 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:22,863 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:22,864 | 2494052621.py:15 | round-14-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:22,866 | 2494052621.py:31 | [('empresa_1', 0.7214250564575195), ('empresa_2', 0.7134799957275391), ('empresa_3', 0.7345433831214905)]\n",
      "INFO flower 2022-04-14 23:03:22,867 | 2494052621.py:37 | Aggregated accuracy: 0.7231494784355164 +- 0.008685113528160957\n",
      "DEBUG flower 2022-04-14 23:03:22,867 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:26,767 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:26,774 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:27,952 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:27,952 | 2494052621.py:15 | round-15-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:27,955 | 2494052621.py:31 | [('empresa_1', 0.7214250564575195), ('empresa_2', 0.7129159569740295), ('empresa_3', 0.7317073345184326)]\n",
      "INFO flower 2022-04-14 23:03:27,955 | 2494052621.py:37 | Aggregated accuracy: 0.7220161159833273 +- 0.007682923967100029\n",
      "DEBUG flower 2022-04-14 23:03:27,956 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:31,729 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:31,736 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:32,907 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:32,908 | 2494052621.py:15 | round-16-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:32,910 | 2494052621.py:31 | [('empresa_1', 0.7219198346138), ('empresa_2', 0.7146080136299133), ('empresa_3', 0.7317073345184326)]\n",
      "INFO flower 2022-04-14 23:03:32,910 | 2494052621.py:37 | Aggregated accuracy: 0.7227450609207153 +- 0.0070051144437644885\n",
      "DEBUG flower 2022-04-14 23:03:32,911 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:36,836 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:36,843 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:38,002 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:38,003 | 2494052621.py:15 | round-17-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:38,004 | 2494052621.py:31 | [('empresa_1', 0.7234042286872864), ('empresa_2', 0.7151720523834229), ('empresa_3', 0.73000568151474)]\n",
      "INFO flower 2022-04-14 23:03:38,005 | 2494052621.py:37 | Aggregated accuracy: 0.7228606541951498 +- 0.006067989410161123\n",
      "DEBUG flower 2022-04-14 23:03:38,006 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:42,091 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:42,103 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:43,430 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:43,430 | 2494052621.py:15 | round-18-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:43,433 | 2494052621.py:31 | [('empresa_1', 0.7209302186965942), ('empresa_2', 0.7157360315322876), ('empresa_3', 0.7283040285110474)]\n",
      "INFO flower 2022-04-14 23:03:43,434 | 2494052621.py:37 | Aggregated accuracy: 0.7216567595799764 +- 0.005156519062051321\n",
      "DEBUG flower 2022-04-14 23:03:43,434 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:47,208 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:47,215 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:48,377 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:48,377 | 2494052621.py:15 | round-19-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:48,380 | 2494052621.py:31 | [('empresa_1', 0.7234042286872864), ('empresa_2', 0.7146080136299133), ('empresa_3', 0.7294384837150574)]\n",
      "INFO flower 2022-04-14 23:03:48,380 | 2494052621.py:37 | Aggregated accuracy: 0.7224835753440857 +- 0.006089412268867335\n",
      "DEBUG flower 2022-04-14 23:03:48,381 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:51,999 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:52,006 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:53,220 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:53,220 | 2494052621.py:15 | round-20-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:53,222 | 2494052621.py:31 | [('empresa_1', 0.7243938446044922), ('empresa_2', 0.7146080136299133), ('empresa_3', 0.7271695733070374)]\n",
      "INFO flower 2022-04-14 23:03:53,223 | 2494052621.py:37 | Aggregated accuracy: 0.7220571438471476 +- 0.005387845778695974\n",
      "DEBUG flower 2022-04-14 23:03:53,224 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:56,869 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:03:56,877 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:03:58,109 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:03:58,109 | 2494052621.py:15 | round-21-EVALUATION\n",
      "INFO flower 2022-04-14 23:03:58,112 | 2494052621.py:31 | [('empresa_1', 0.7238990664482117), ('empresa_2', 0.7129159569740295), ('empresa_3', 0.7260351777076721)]\n",
      "INFO flower 2022-04-14 23:03:58,112 | 2494052621.py:37 | Aggregated accuracy: 0.7209500670433044 +- 0.005747517509447983\n",
      "DEBUG flower 2022-04-14 23:03:58,113 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:01,768 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:01,775 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:02,964 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:02,965 | 2494052621.py:15 | round-22-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:02,967 | 2494052621.py:31 | [('empresa_1', 0.7238990664482117), ('empresa_2', 0.7112239003181458), ('empresa_3', 0.7271695733070374)]\n",
      "INFO flower 2022-04-14 23:04:02,968 | 2494052621.py:37 | Aggregated accuracy: 0.720764180024465 +- 0.0068768576367655284\n",
      "DEBUG flower 2022-04-14 23:04:02,968 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:06,748 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:06,755 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:07,862 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:07,863 | 2494052621.py:15 | round-23-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:07,865 | 2494052621.py:31 | [('empresa_1', 0.7238990664482117), ('empresa_2', 0.7112239003181458), ('empresa_3', 0.7277368307113647)]\n",
      "INFO flower 2022-04-14 23:04:07,865 | 2494052621.py:37 | Aggregated accuracy: 0.7209532658259074 +- 0.00705584974909\n",
      "DEBUG flower 2022-04-14 23:04:07,865 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:11,458 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:11,465 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:12,792 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:12,792 | 2494052621.py:15 | round-24-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:12,794 | 2494052621.py:31 | [('empresa_1', 0.7229094505310059), ('empresa_2', 0.7112239003181458), ('empresa_3', 0.72887122631073)]\n",
      "INFO flower 2022-04-14 23:04:12,795 | 2494052621.py:37 | Aggregated accuracy: 0.7210015257199606 +- 0.007329718570567654\n",
      "DEBUG flower 2022-04-14 23:04:12,795 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:16,660 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:16,667 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:17,927 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:17,927 | 2494052621.py:15 | round-25-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:17,929 | 2494052621.py:31 | [('empresa_1', 0.725383460521698), ('empresa_2', 0.71235191822052), ('empresa_3', 0.7294384837150574)]\n",
      "INFO flower 2022-04-14 23:04:17,930 | 2494052621.py:37 | Aggregated accuracy: 0.7223912874857584 +- 0.007289376045887208\n",
      "DEBUG flower 2022-04-14 23:04:17,930 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:21,515 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:21,522 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:22,681 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:22,682 | 2494052621.py:15 | round-26-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:22,684 | 2494052621.py:31 | [('empresa_1', 0.7278574705123901), ('empresa_2', 0.7146080136299133), ('empresa_3', 0.7283040285110474)]\n",
      "INFO flower 2022-04-14 23:04:22,685 | 2494052621.py:37 | Aggregated accuracy: 0.7235898375511169 +- 0.006353724588783615\n",
      "DEBUG flower 2022-04-14 23:04:22,685 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:26,268 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:26,275 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:27,429 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:27,430 | 2494052621.py:15 | round-27-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:27,432 | 2494052621.py:31 | [('empresa_1', 0.7283523082733154), ('empresa_2', 0.7157360315322876), ('empresa_3', 0.7266023755073547)]\n",
      "INFO flower 2022-04-14 23:04:27,432 | 2494052621.py:37 | Aggregated accuracy: 0.723563571770986 +- 0.005580821672637866\n",
      "DEBUG flower 2022-04-14 23:04:27,432 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:31,385 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:31,393 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:32,731 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:32,732 | 2494052621.py:15 | round-28-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:32,734 | 2494052621.py:31 | [('empresa_1', 0.7298367023468018), ('empresa_2', 0.7191201448440552), ('empresa_3', 0.7249007225036621)]\n",
      "INFO flower 2022-04-14 23:04:32,734 | 2494052621.py:37 | Aggregated accuracy: 0.724619189898173 +- 0.004379543098953255\n",
      "DEBUG flower 2022-04-14 23:04:32,734 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:36,242 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:36,249 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:37,686 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:37,686 | 2494052621.py:15 | round-29-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:37,688 | 2494052621.py:31 | [('empresa_1', 0.7308263182640076), ('empresa_2', 0.7202481627464294), ('empresa_3', 0.7237663269042969)]\n",
      "INFO flower 2022-04-14 23:04:37,688 | 2494052621.py:37 | Aggregated accuracy: 0.724946935971578 +- 0.004398463509441061\n",
      "DEBUG flower 2022-04-14 23:04:37,689 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:41,466 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:41,473 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:42,727 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:42,728 | 2494052621.py:15 | round-30-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:42,730 | 2494052621.py:31 | [('empresa_1', 0.7318159341812134), ('empresa_2', 0.7179921269416809), ('empresa_3', 0.7243335247039795)]\n",
      "INFO flower 2022-04-14 23:04:42,730 | 2494052621.py:37 | Aggregated accuracy: 0.7247138619422913 +- 0.0056499500859194945\n",
      "DEBUG flower 2022-04-14 23:04:42,731 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:46,550 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:46,557 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:47,702 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:47,702 | 2494052621.py:15 | round-31-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:47,704 | 2494052621.py:31 | [('empresa_1', 0.7347847819328308), ('empresa_2', 0.7174280881881714), ('empresa_3', 0.7237663269042969)]\n",
      "INFO flower 2022-04-14 23:04:47,705 | 2494052621.py:37 | Aggregated accuracy: 0.725326399008433 +- 0.00717119577030464\n",
      "DEBUG flower 2022-04-14 23:04:47,705 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:51,220 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:51,227 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:52,404 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:52,405 | 2494052621.py:15 | round-32-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:52,407 | 2494052621.py:31 | [('empresa_1', 0.7333003282546997), ('empresa_2', 0.7163000702857971), ('empresa_3', 0.7226318717002869)]\n",
      "INFO flower 2022-04-14 23:04:52,408 | 2494052621.py:37 | Aggregated accuracy: 0.7240774234135946 +- 0.007015193395727785\n",
      "DEBUG flower 2022-04-14 23:04:52,408 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:56,135 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:04:56,142 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:04:57,282 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:04:57,282 | 2494052621.py:15 | round-33-EVALUATION\n",
      "INFO flower 2022-04-14 23:04:57,284 | 2494052621.py:31 | [('empresa_1', 0.7342899441719055), ('empresa_2', 0.7146080136299133), ('empresa_3', 0.7231990694999695)]\n",
      "INFO flower 2022-04-14 23:04:57,285 | 2494052621.py:37 | Aggregated accuracy: 0.7240323424339294 +- 0.00805668895191252\n",
      "DEBUG flower 2022-04-14 23:04:57,285 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:01,266 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:01,273 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:02,477 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:02,477 | 2494052621.py:15 | round-34-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:02,479 | 2494052621.py:31 | [('empresa_1', 0.7342899441719055), ('empresa_2', 0.7157360315322876), ('empresa_3', 0.7231990694999695)]\n",
      "INFO flower 2022-04-14 23:05:02,480 | 2494052621.py:37 | Aggregated accuracy: 0.7244083484013876 +- 0.007622715402256799\n",
      "DEBUG flower 2022-04-14 23:05:02,480 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:06,371 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:06,378 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:07,546 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:07,546 | 2494052621.py:15 | round-35-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:07,548 | 2494052621.py:31 | [('empresa_1', 0.733795166015625), ('empresa_2', 0.7191201448440552), ('empresa_3', 0.7249007225036621)]\n",
      "INFO flower 2022-04-14 23:05:07,549 | 2494052621.py:37 | Aggregated accuracy: 0.7259386777877808 +- 0.006035841558301434\n",
      "DEBUG flower 2022-04-14 23:05:07,549 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:11,397 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:11,405 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:12,579 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:12,580 | 2494052621.py:15 | round-36-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:12,582 | 2494052621.py:31 | [('empresa_1', 0.7347847819328308), ('empresa_2', 0.7202481627464294), ('empresa_3', 0.7249007225036621)]\n",
      "INFO flower 2022-04-14 23:05:12,582 | 2494052621.py:37 | Aggregated accuracy: 0.7266445557276408 +- 0.006061300194809801\n",
      "DEBUG flower 2022-04-14 23:05:12,583 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:16,532 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:16,539 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:17,660 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:17,660 | 2494052621.py:15 | round-37-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:17,662 | 2494052621.py:31 | [('empresa_1', 0.7342899441719055), ('empresa_2', 0.720812201499939), ('empresa_3', 0.7237663269042969)]\n",
      "INFO flower 2022-04-14 23:05:17,663 | 2494052621.py:37 | Aggregated accuracy: 0.7262894908587137 +- 0.005784297948676146\n",
      "DEBUG flower 2022-04-14 23:05:17,663 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:21,647 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:21,654 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:22,922 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:22,923 | 2494052621.py:15 | round-38-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:22,925 | 2494052621.py:31 | [('empresa_1', 0.7357743978500366), ('empresa_2', 0.7253243327140808), ('empresa_3', 0.7237663269042969)]\n",
      "INFO flower 2022-04-14 23:05:22,925 | 2494052621.py:37 | Aggregated accuracy: 0.7282883524894714 +- 0.0053315102226392215\n",
      "DEBUG flower 2022-04-14 23:05:22,926 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:27,027 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:27,034 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:28,297 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:28,298 | 2494052621.py:15 | round-39-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:28,300 | 2494052621.py:31 | [('empresa_1', 0.7367639541625977), ('empresa_2', 0.7253243327140808), ('empresa_3', 0.7226318717002869)]\n",
      "INFO flower 2022-04-14 23:05:28,301 | 2494052621.py:37 | Aggregated accuracy: 0.7282400528589884 +- 0.006126717808253356\n",
      "DEBUG flower 2022-04-14 23:05:28,301 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:31,968 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:31,975 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:33,251 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:33,252 | 2494052621.py:15 | round-40-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:33,254 | 2494052621.py:31 | [('empresa_1', 0.7357743978500366), ('empresa_2', 0.723632276058197), ('empresa_3', 0.7209302186965942)]\n",
      "INFO flower 2022-04-14 23:05:33,255 | 2494052621.py:37 | Aggregated accuracy: 0.7267789642016093 +- 0.006455677004972285\n",
      "DEBUG flower 2022-04-14 23:05:33,255 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:36,862 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:36,869 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:38,064 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:38,065 | 2494052621.py:15 | round-41-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:38,067 | 2494052621.py:31 | [('empresa_1', 0.7352795600891113), ('empresa_2', 0.7253243327140808), ('empresa_3', 0.7180941700935364)]\n",
      "INFO flower 2022-04-14 23:05:38,067 | 2494052621.py:37 | Aggregated accuracy: 0.7262326876322428 +- 0.0070452460954421245\n",
      "DEBUG flower 2022-04-14 23:05:38,068 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:42,035 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:42,042 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:43,195 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:43,195 | 2494052621.py:15 | round-42-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:43,197 | 2494052621.py:31 | [('empresa_1', 0.7362691760063171), ('empresa_2', 0.7241962552070618), ('empresa_3', 0.7180941700935364)]\n",
      "INFO flower 2022-04-14 23:05:43,198 | 2494052621.py:37 | Aggregated accuracy: 0.7261865337689718 +- 0.007552201292687354\n",
      "DEBUG flower 2022-04-14 23:05:43,198 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:46,886 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:46,893 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:48,084 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:48,084 | 2494052621.py:15 | round-43-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:48,086 | 2494052621.py:31 | [('empresa_1', 0.7377535700798035), ('empresa_2', 0.7247602939605713), ('empresa_3', 0.7169597148895264)]\n",
      "INFO flower 2022-04-14 23:05:48,087 | 2494052621.py:37 | Aggregated accuracy: 0.7264911929766337 +- 0.008576833602653443\n",
      "DEBUG flower 2022-04-14 23:05:48,087 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:52,009 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:52,016 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:53,235 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:53,236 | 2494052621.py:15 | round-44-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:53,238 | 2494052621.py:31 | [('empresa_1', 0.7377535700798035), ('empresa_2', 0.7253243327140808), ('empresa_3', 0.7169597148895264)]\n",
      "INFO flower 2022-04-14 23:05:53,238 | 2494052621.py:37 | Aggregated accuracy: 0.7266792058944702 +- 0.008542945019908038\n",
      "DEBUG flower 2022-04-14 23:05:53,239 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:57,177 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:05:57,184 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:05:58,389 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:05:58,389 | 2494052621.py:15 | round-45-EVALUATION\n",
      "INFO flower 2022-04-14 23:05:58,391 | 2494052621.py:31 | [('empresa_1', 0.7412171959877014), ('empresa_2', 0.7247602939605713), ('empresa_3', 0.718661367893219)]\n",
      "INFO flower 2022-04-14 23:05:58,392 | 2494052621.py:37 | Aggregated accuracy: 0.7282129526138306 +- 0.009526523818228251\n",
      "DEBUG flower 2022-04-14 23:05:58,392 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:02,230 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:06:02,237 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:03,395 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:06:03,396 | 2494052621.py:15 | round-46-EVALUATION\n",
      "INFO flower 2022-04-14 23:06:03,398 | 2494052621.py:31 | [('empresa_1', 0.7422068119049072), ('empresa_2', 0.7247602939605713), ('empresa_3', 0.718661367893219)]\n",
      "INFO flower 2022-04-14 23:06:03,398 | 2494052621.py:37 | Aggregated accuracy: 0.7285428245862325 +- 0.009977562724347567\n",
      "DEBUG flower 2022-04-14 23:06:03,399 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:07,039 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:06:07,046 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:08,527 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:06:08,527 | 2494052621.py:15 | round-47-EVALUATION\n",
      "INFO flower 2022-04-14 23:06:08,529 | 2494052621.py:31 | [('empresa_1', 0.7427016496658325), ('empresa_2', 0.7241962552070618), ('empresa_3', 0.718661367893219)]\n",
      "INFO flower 2022-04-14 23:06:08,530 | 2494052621.py:37 | Aggregated accuracy: 0.7285197575887045 +- 0.010279536032322206\n",
      "DEBUG flower 2022-04-14 23:06:08,530 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:11,981 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:06:11,988 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:13,287 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:06:13,287 | 2494052621.py:15 | round-48-EVALUATION\n",
      "INFO flower 2022-04-14 23:06:13,290 | 2494052621.py:31 | [('empresa_1', 0.7422068119049072), ('empresa_2', 0.7258883118629456), ('empresa_3', 0.7175269722938538)]\n",
      "INFO flower 2022-04-14 23:06:13,290 | 2494052621.py:37 | Aggregated accuracy: 0.7285406986872355 +- 0.0102485767371566\n",
      "DEBUG flower 2022-04-14 23:06:13,291 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:16,988 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:06:16,995 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:18,141 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:06:18,142 | 2494052621.py:15 | round-49-EVALUATION\n",
      "INFO flower 2022-04-14 23:06:18,144 | 2494052621.py:31 | [('empresa_1', 0.7427016496658325), ('empresa_2', 0.7247602939605713), ('empresa_3', 0.7180941700935364)]\n",
      "INFO flower 2022-04-14 23:06:18,144 | 2494052621.py:37 | Aggregated accuracy: 0.7285187045733134 +- 0.010391543051145721\n",
      "DEBUG flower 2022-04-14 23:06:18,145 | server.py:265 | fit_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:21,854 | server.py:277 | fit_round received 3 results and 0 failures\n",
      "DEBUG flower 2022-04-14 23:06:21,861 | server.py:211 | evaluate_round: strategy sampled 3 clients (out of 3)\n",
      "DEBUG flower 2022-04-14 23:06:23,021 | server.py:223 | evaluate_round received 3 results and 0 failures\n",
      "INFO flower 2022-04-14 23:06:23,021 | 2494052621.py:15 | round-50-EVALUATION\n",
      "INFO flower 2022-04-14 23:06:23,023 | 2494052621.py:31 | [('empresa_1', 0.7422068119049072), ('empresa_2', 0.723632276058197), ('empresa_3', 0.7163925170898438)]\n",
      "INFO flower 2022-04-14 23:06:23,024 | 2494052621.py:37 | Aggregated accuracy: 0.7274105350176493 +- 0.010872009466922065\n",
      "INFO flower 2022-04-14 23:06:23,024 | server.py:182 | FL finished in 257.0715694000028\n",
      "INFO flower 2022-04-14 23:06:23,025 | app.py:149 | app_fit: losses_distributed [(1, 0.5967591227291513), (2, 0.5606816712634821), (3, 0.5476938600760689), (4, 0.5424982326395846), (5, 0.5394070722204287), (6, 0.5374017741711465), (7, 0.5360171608687176), (8, 0.534942697734311), (9, 0.5340324036397586), (10, 0.5332358294475501), (11, 0.5325677073085525), (12, 0.5319181492544451), (13, 0.5312388371773411), (14, 0.5306426708767349), (15, 0.5301155087242929), (16, 0.5295728818426807), (17, 0.529067204842491), (18, 0.5285882555539603), (19, 0.5281484581400041), (20, 0.5277406354055186), (21, 0.5274027718128179), (22, 0.5270458952179696), (23, 0.5267199382349508), (24, 0.526440251092377), (25, 0.5262336562129047), (26, 0.5260101187372885), (27, 0.525737449014618), (28, 0.5254386769463867), (29, 0.5251376486862867), (30, 0.5248408708476712), (31, 0.524571110264998), (32, 0.5243204962559391), (33, 0.5241368931468623), (34, 0.5239786022061685), (35, 0.5238899581076667), (36, 0.5238003619169403), (37, 0.5236549401950424), (38, 0.523661802763001), (39, 0.5235536819592383), (40, 0.5234379266875241), (41, 0.5233310359039601), (42, 0.5232498590060655), (43, 0.523107164081361), (44, 0.5230494187861569), (45, 0.5229863395670753), (46, 0.5229381228417871), (47, 0.5229264288169994), (48, 0.5229760490402121), (49, 0.5229224831707855), (50, 0.5228685841020629)]\n",
      "INFO flower 2022-04-14 23:06:23,026 | app.py:150 | app_fit: metrics_distributed {}\n",
      "INFO flower 2022-04-14 23:06:23,026 | app.py:151 | app_fit: losses_centralized []\n",
      "INFO flower 2022-04-14 23:06:23,026 | app.py:152 | app_fit: metrics_centralized {}\n"
     ]
    }
   ],
   "source": [
    "# Now the clients are the companies\n",
    "CLIENTS_IDS = [\"company_1\", \"company_2\", \"company_3\"]\n",
    "\n",
    " # Initialize the results file\n",
    "header = CLIENTS_IDS\n",
    "\n",
    "# Start Flower simulation\n",
    "    # min_fit_clients and min_eval_clients are set to the max clients available as we are in a controled scenario\n",
    "    \n",
    "results_fed = pd.DataFrame(columns=[\"UC\", \"best_rnd\", \"acc\"])\n",
    "\n",
    "seed(1)\n",
    "set_random_seed(2)\n",
    "for i in range(1,22):\n",
    "    global UNSEEN_CLIENT \n",
    "    UNSEEN_CLIENT = f'client_{i}.csv'\n",
    "    \n",
    "    with open(f\"./results/milestone2.csv\", 'w', ) as f:\n",
    "        csv_writer = writer(f)\n",
    "        csv_writer.writerow(header)\n",
    "    \n",
    "    fl.simulation.start_simulation(\n",
    "        client_fn=client_fn,\n",
    "        clients_ids=CLIENTS_IDS,\n",
    "        client_resources={\"num_cpus\": 6},\n",
    "        num_rounds=50,\n",
    "        strategy=SaveModelStrategy(\n",
    "            min_available_clients = len(CLIENTS_IDS),\n",
    "            min_fit_clients = len(CLIENTS_IDS),\n",
    "            min_eval_clients = len(CLIENTS_IDS),\n",
    "            on_fit_config_fn = fit_config,\n",
    "            on_evaluate_config_fn = fit_config,\n",
    "            accept_failures=False,\n",
    "            initial_parameters=parameters\n",
    "        ),\n",
    "    )\n",
    "    \n",
    "    df = pd.read_csv('./results/milestone2.csv')\n",
    "    df[\"mean\"] = df.mean(numeric_only=True, axis=1)\n",
    "    \n",
    "    fed_res = {\n",
    "        \"UC\": UNSEEN_CLIENT,\n",
    "        \"best_rnd\": df[\"mean\"].idxmax()+1,\n",
    "        \"acc\": df[\"mean\"].max()\n",
    "    }\n",
    "    \n",
    "    results_fed = results_fed.append(fed_res, ignore_index=True)\n",
    "    \n",
    "results_fed.to_csv(\"./results/milestone2_UCs.csv\", index=False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_14816/2694975666.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      7\u001B[0m \u001B[0mmatplotlib\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mrc\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m'font'\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mfont\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      8\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 9\u001B[1;33m \u001B[0mdf\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread_csv\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m'./results/hito2.csv'\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     10\u001B[0m \u001B[1;31m# df[\"mean\"] = df.mean(numeric_only=True, axis=1)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     11\u001B[0m \u001B[0mdf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mcolumns\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;33m[\u001B[0m\u001B[1;34m\"ORG 1\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m\"ORG 2\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m\"ORG 3\"\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mNameError\u001B[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "import matplotlib\n",
    "\n",
    "font = {'family' : 'serif',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 16}\n",
    "\n",
    "matplotlib.rc('font', **font)\n",
    "\n",
    "df = pd.read_csv('./results/milestone2.csv')\n",
    "df.columns = [\"ORG 1\", \"ORG 2\", \"ORG 3\"]\n",
    "\n",
    "print(f'Acc max: {df.mean(numeric_only=True, axis=1).max()}')\n",
    "\n",
    "f = plt.figure(figsize=(10,6))\n",
    "ax = f.add_subplot()\n",
    "\n",
    "df.plot(ax=ax, ylim=(0.5,1), linewidth=2.5)\n",
    "ax.set_ylabel(\"Accuracy\")\n",
    "ax.set_xlabel(\"Federated round\")\n",
    "\n",
    "plt.savefig('./training-federated.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UC</th>\n",
       "      <th>best_rnd</th>\n",
       "      <th>acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cliente_1.csv</td>\n",
       "      <td>16</td>\n",
       "      <td>0.721702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cliente_2.csv</td>\n",
       "      <td>46</td>\n",
       "      <td>0.728610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cliente_3.csv</td>\n",
       "      <td>11</td>\n",
       "      <td>0.721907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cliente_4.csv</td>\n",
       "      <td>15</td>\n",
       "      <td>0.712347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cliente_5.csv</td>\n",
       "      <td>12</td>\n",
       "      <td>0.717796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cliente_6.csv</td>\n",
       "      <td>49</td>\n",
       "      <td>0.730177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cliente_7.csv</td>\n",
       "      <td>18</td>\n",
       "      <td>0.723837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cliente_8.csv</td>\n",
       "      <td>12</td>\n",
       "      <td>0.724856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>cliente_9.csv</td>\n",
       "      <td>9</td>\n",
       "      <td>0.723093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>cliente_10.csv</td>\n",
       "      <td>45</td>\n",
       "      <td>0.719289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>cliente_11.csv</td>\n",
       "      <td>19</td>\n",
       "      <td>0.712470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>cliente_12.csv</td>\n",
       "      <td>46</td>\n",
       "      <td>0.711797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>cliente_13.csv</td>\n",
       "      <td>46</td>\n",
       "      <td>0.715570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>cliente_14.csv</td>\n",
       "      <td>43</td>\n",
       "      <td>0.721329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>cliente_15.csv</td>\n",
       "      <td>46</td>\n",
       "      <td>0.719602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>cliente_16.csv</td>\n",
       "      <td>50</td>\n",
       "      <td>0.712111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>cliente_17.csv</td>\n",
       "      <td>31</td>\n",
       "      <td>0.726344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>cliente_18.csv</td>\n",
       "      <td>48</td>\n",
       "      <td>0.715683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>cliente_19.csv</td>\n",
       "      <td>50</td>\n",
       "      <td>0.724705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>cliente_20.csv</td>\n",
       "      <td>14</td>\n",
       "      <td>0.729947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>cliente_21.csv</td>\n",
       "      <td>50</td>\n",
       "      <td>0.721129</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                UC  best_rnd       acc\n",
       "0    cliente_1.csv        16  0.721702\n",
       "1    cliente_2.csv        46  0.728610\n",
       "2    cliente_3.csv        11  0.721907\n",
       "3    cliente_4.csv        15  0.712347\n",
       "4    cliente_5.csv        12  0.717796\n",
       "5    cliente_6.csv        49  0.730177\n",
       "6    cliente_7.csv        18  0.723837\n",
       "7    cliente_8.csv        12  0.724856\n",
       "8    cliente_9.csv         9  0.723093\n",
       "9   cliente_10.csv        45  0.719289\n",
       "10  cliente_11.csv        19  0.712470\n",
       "11  cliente_12.csv        46  0.711797\n",
       "12  cliente_13.csv        46  0.715570\n",
       "13  cliente_14.csv        43  0.721329\n",
       "14  cliente_15.csv        46  0.719602\n",
       "15  cliente_16.csv        50  0.712111\n",
       "16  cliente_17.csv        31  0.726344\n",
       "17  cliente_18.csv        48  0.715683\n",
       "18  cliente_19.csv        50  0.724705\n",
       "19  cliente_20.csv        14  0.729947\n",
       "20  cliente_21.csv        50  0.721129"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./results/milestone2_UCs.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "flower.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}