CSE8803_MPP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "XWQGBQwshUq2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c023644f-cb51-4501-8787-07aa709466cc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install arrow\n",
        "!pip install geopandas\n",
        "# !pip install osmnx==1.1.1\n",
        "# !pip install pandana\n",
        "!pip install haversine\n",
        "!pip install --upgrade folium\n",
        "!pip install --upgrade statsmodels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3UdiXQ6wtt94",
        "outputId": "d63d786a-5e56-49c9-e57f-2c2d802b863a"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting arrow\n",
            "  Downloading arrow-1.2.3-py3-none-any.whl (66 kB)\n",
            "\u001b[K     |████████████████████████████████| 66 kB 2.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.7.0 in /usr/local/lib/python3.7/dist-packages (from arrow) (2.8.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from arrow) (4.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.0->arrow) (1.15.0)\n",
            "Installing collected packages: arrow\n",
            "Successfully installed arrow-1.2.3\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting geopandas\n",
            "  Downloading geopandas-0.10.2-py2.py3-none-any.whl (1.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0 MB 6.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas>=0.25.0 in /usr/local/lib/python3.7/dist-packages (from geopandas) (1.3.5)\n",
            "Collecting pyproj>=2.2.0\n",
            "  Downloading pyproj-3.2.1-cp37-cp37m-manylinux2010_x86_64.whl (6.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.3 MB 33.5 MB/s \n",
            "\u001b[?25hCollecting fiona>=1.8\n",
            "  Downloading Fiona-1.8.22-cp37-cp37m-manylinux2014_x86_64.whl (16.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 16.7 MB 23.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: shapely>=1.6 in /usr/local/lib/python3.7/dist-packages (from geopandas) (1.8.5.post1)\n",
            "Requirement already satisfied: click>=4.0 in /usr/local/lib/python3.7/dist-packages (from fiona>=1.8->geopandas) (7.1.2)\n",
            "Requirement already satisfied: attrs>=17 in /usr/local/lib/python3.7/dist-packages (from fiona>=1.8->geopandas) (22.1.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from fiona>=1.8->geopandas) (57.4.0)\n",
            "Collecting click-plugins>=1.0\n",
            "  Downloading click_plugins-1.1.1-py2.py3-none-any.whl (7.5 kB)\n",
            "Collecting munch\n",
            "  Downloading munch-2.5.0-py2.py3-none-any.whl (10 kB)\n",
            "Collecting cligj>=0.5\n",
            "  Downloading cligj-0.7.2-py3-none-any.whl (7.1 kB)\n",
            "Requirement already satisfied: six>=1.7 in /usr/local/lib/python3.7/dist-packages (from fiona>=1.8->geopandas) (1.15.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from fiona>=1.8->geopandas) (2022.9.24)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25.0->geopandas) (1.21.6)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25.0->geopandas) (2022.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25.0->geopandas) (2.8.2)\n",
            "Installing collected packages: munch, cligj, click-plugins, pyproj, fiona, geopandas\n",
            "Successfully installed click-plugins-1.1.1 cligj-0.7.2 fiona-1.8.22 geopandas-0.10.2 munch-2.5.0 pyproj-3.2.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting haversine\n",
            "  Downloading haversine-2.7.0-py2.py3-none-any.whl (6.9 kB)\n",
            "Installing collected packages: haversine\n",
            "Successfully installed haversine-2.7.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: folium in /usr/local/lib/python3.7/dist-packages (0.12.1.post1)\n",
            "Collecting folium\n",
            "  Downloading folium-0.13.0-py2.py3-none-any.whl (96 kB)\n",
            "\u001b[K     |████████████████████████████████| 96 kB 3.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from folium) (1.21.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from folium) (2.23.0)\n",
            "Requirement already satisfied: branca>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from folium) (0.6.0)\n",
            "Requirement already satisfied: jinja2>=2.9 in /usr/local/lib/python3.7/dist-packages (from folium) (2.11.3)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2>=2.9->folium) (2.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->folium) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->folium) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->folium) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->folium) (2022.9.24)\n",
            "Installing collected packages: folium\n",
            "  Attempting uninstall: folium\n",
            "    Found existing installation: folium 0.12.1.post1\n",
            "    Uninstalling folium-0.12.1.post1:\n",
            "      Successfully uninstalled folium-0.12.1.post1\n",
            "Successfully installed folium-0.13.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: statsmodels in /usr/local/lib/python3.7/dist-packages (0.12.2)\n",
            "Collecting statsmodels\n",
            "  Downloading statsmodels-0.13.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 9.9 MB 7.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy<1.8,>=1.3 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (1.7.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (1.21.6)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (21.3)\n",
            "Requirement already satisfied: pandas>=0.25 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (1.3.5)\n",
            "Requirement already satisfied: patsy>=0.5.2 in /usr/local/lib/python3.7/dist-packages (from statsmodels) (0.5.3)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=21.3->statsmodels) (3.0.9)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25->statsmodels) (2022.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.25->statsmodels) (2.8.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from patsy>=0.5.2->statsmodels) (1.15.0)\n",
            "Installing collected packages: statsmodels\n",
            "  Attempting uninstall: statsmodels\n",
            "    Found existing installation: statsmodels 0.12.2\n",
            "    Uninstalling statsmodels-0.12.2:\n",
            "      Successfully uninstalled statsmodels-0.12.2\n",
            "Successfully installed statsmodels-0.13.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import random\n",
        "import branca\n",
        "import folium\n",
        "import geopandas as gpd\n",
        "import seaborn as sns\n",
        "import cProfile\n",
        "import sys\n",
        "import pickle\n",
        "import arrow\n",
        "import datetime\n",
        "from argparse import ArgumentParser\n",
        "\n",
        "from shapely.geometry import Polygon, Point, MultiPoint, MultiPolygon\n",
        "from shapely.affinity import affine_transform\n",
        "from descartes.patch import PolygonPatch\n",
        "from shapely.ops import cascaded_union\n",
        "from matplotlib.path import Path\n",
        "\n",
        "from scipy.stats import multivariate_normal\n",
        "from scipy.spatial.distance import pdist, squareform\n",
        "from scipy.stats import gaussian_kde\n",
        "\n",
        "from sklearn import preprocessing\n",
        "from sklearn.cluster import KMeans\n",
        "\n",
        "import scipy.sparse as sp"
      ],
      "metadata": {
        "id": "BtpDyAuquJsp"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "from abc import ABC, abstractmethod\n",
        "import networkx as nx\n",
        "from community import community_louvain\n",
        "from haversine import haversine, haversine_vector"
      ],
      "metadata": {
        "id": "hTwC8YUVuK-f"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import os\n",
        "import itertools"
      ],
      "metadata": {
        "id": "NMiTfYfyuNrJ"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Kernel"
      ],
      "metadata": {
        "id": "dnWCQeXvuiYf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Multivariate_Exponential_Kernel(torch.nn.Module):\n",
        "    \"\"\"\n",
        "    Exponential Decaying Kernel\n",
        "    \"\"\"\n",
        "    def __init__(self, alphas, beta, device):\n",
        "        \"\"\"\n",
        "        Arg:\n",
        "        - alphas: influence coefficient matrix, numpy array                     [ n_class, n_class ]\n",
        "        - beta: temporal decaying parameter for historical influence, numpy array        [ n_class ]\n",
        "        - device: cpu or cuda\n",
        "        \"\"\"\n",
        "        super(Multivariate_Exponential_Kernel, self).__init__()\n",
        "\n",
        "        # configuration\n",
        "        self._alphas     = torch.nn.Parameter(torch.tensor(alphas))\n",
        "        self._beta       = torch.nn.Parameter(torch.tensor(beta))\n",
        "        self.n_class     = alphas.shape[0]\n",
        "\n",
        "    def init_paras(self):\n",
        "        \"\"\"\n",
        "        initialize model parameters\n",
        "        \"\"\"\n",
        "\n",
        "        self._alphas.data = torch.empty(torch.empty(self.n_class, self.n_class).uniform_(0,to=self.init_std)).to(self._alphas.device)\n",
        "        self._beta.data = torch.empty(torch.empty(self.n_class).uniform_(0,to=self.init_std)).to(self._beta.device)\n",
        "    \n",
        "    def forward(self, x, y):\n",
        "        \"\"\"\n",
        "        customized forward function returning kernel evaluation at index x and y \n",
        "        - x: the current input (t, type) [ batch_size, data_dim = 2 ]\n",
        "        - y: the history input (t, type) [ batch_size, data_dim = 2 ]\n",
        "        \"\"\"\n",
        "\n",
        "        batch_size   = x.shape[0]\n",
        "        alphas_hist  = self._alphas[x[:, 1].long(), y[:, 1].long()]\n",
        "        beta_hist    = self._beta[y[:, 1].long()]\n",
        "\n",
        "        mask         = x[:, 0] > 0\n",
        "        tds          = (x[:, 0] - y[:, 0]) * mask\n",
        "\n",
        "        return alphas_hist * beta_hist * torch.exp(- beta_hist * tds) * mask    # [ batch_size ]"
      ],
      "metadata": {
        "id": "b0Af2HxLuT3Z"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "zla5IzsJzTAn"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# PP"
      ],
      "metadata": {
        "id": "LTEcUKlNzT68"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class BasePointProcess(torch.nn.Module):\n",
        "    \"\"\"\n",
        "    PyTorch Module for Multivariate Point Process with diffusion kernel\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, T, mu, data_dim, device):\n",
        "        \"\"\"\n",
        "        data dim = 8\n",
        "\n",
        "        Args:\n",
        "        \"\"\"\n",
        "        super(BasePointProcess, self).__init__()\n",
        "        self.data_dim      = data_dim\n",
        "        self.T             = T # time horizon. e.g. (0, 1)\n",
        "        self.device        = device\n",
        "        self.n_class       = len(mu)\n",
        "        \n",
        "        # parameters\n",
        "        self._mu = torch.nn.Parameter(torch.tensor(mu), requires_grad=False)\n",
        "        # # pre-compute\n",
        "        # self.data_transformation()\n",
        "\n",
        "    def cond_lambda(self, xi, hti):\n",
        "        \"\"\"\n",
        "        return conditional intensity given x\n",
        "        Args:\n",
        "        - xi:   current i-th point       [ batch_size, data_dim ]\n",
        "        - hti:  history points before ti [ batch_size, seq_len, data_dim ]\n",
        "        Return:\n",
        "        - lami: i-th lambda              [ batch_size ]\n",
        "        \"\"\"\n",
        "        batch_size, seq_len, _ = hti.shape\n",
        "        mask_all = (hti[:, :, 0] > 0) * (xi[:, 0] > 0)[:, None]\n",
        "        # if length of the history is zero\n",
        "        if seq_len == 0:\n",
        "            return self._mu[xi[:, 1].long()]                                    # [ batch_size ]\n",
        "\n",
        "        xi2  = xi.unsqueeze(-2).repeat(1, seq_len, 1)                           # [ batch_size, seq_len, data_dim ]\n",
        "        K    = self.kernel(xi2.reshape(-1, self.data_dim),\n",
        "                           hti.reshape(-1, self.data_dim)).reshape(batch_size, seq_len)                                           \n",
        "                                                                                # [ batch_size, seq_len ]\n",
        "        lami = (K * mask_all).sum(-1) + self._mu[xi[:, 1].long()]                                 \n",
        "                                                                                # [ batch_size ]\n",
        "        return lami             # [ batch_size ]\n",
        "\n",
        "    def log_likelihood(self, X):\n",
        "        \"\"\"\n",
        "        return log-likelihood given sequence X\n",
        "        Args:\n",
        "        - X:      input points sequence [ batch_size, seq_len, data_dim ]\n",
        "        Return:\n",
        "        - lams:   sequence of lambda    [ batch_size, seq_len ]\n",
        "        - loglik: log-likelihood        scalar\n",
        "        \"\"\"\n",
        "        \n",
        "        raise NotImplementedError()\n",
        "\n",
        "    def forward(self, X):\n",
        "        \"\"\"\n",
        "        custom forward function returning conditional intensities and corresponding log-likelihood\n",
        "        \"\"\"\n",
        "        # return conditional intensities and corresponding log-likelihood\n",
        "        return self.log_likelihood(X)\n",
        "\n",
        "    # def mu(self):\n",
        "    #     \"\"\"\n",
        "    #     return base intensity\n",
        "    #     \"\"\"\n",
        "    #     raise NotImplementedError()\n",
        "\n",
        "\n",
        "\n",
        "class MultivariateExponentialHawkes(BasePointProcess):\n",
        "    \"\"\"\n",
        "    PyTorch Module for Multivariate Temporal Point Process\n",
        "    \"\"\"\n",
        "    def __init__(self, T, mu, alphas, beta, data_dim, device):\n",
        "        \"\"\"\n",
        "        data dim = 5(time, x, y, type)\n",
        "\n",
        "        Args:\n",
        "        \"\"\"\n",
        "        super(MultivariateExponentialHawkes, self).__init__(T, mu, data_dim, device)\n",
        "        \n",
        "        # kernel\n",
        "        self.kernel = Multivariate_Exponential_Kernel(alphas, beta, device)\n",
        "        # # pre-compute\n",
        "        # self.data_transformation()\n",
        "\n",
        "    def log_likelihood(self, X):\n",
        "        \"\"\"\n",
        "        return log-likelihood given sequence X\n",
        "        Args:\n",
        "        - X:      input points sequence [ batch_size, seq_len, data_dim ]\n",
        "        Return:\n",
        "        - lams:   sequence of lambda    [ batch_size, seq_len ]\n",
        "        - loglik: log-likelihood        scalar\n",
        "        \"\"\"\n",
        "\n",
        "        batch_size, seq_len, _ = X.shape\n",
        "        ts       = X[:, :, 0].clone()\n",
        "        ms       = X[:, :, 1].clone().long()\n",
        "        mask     = ts > 0\n",
        "\n",
        "        lams     = [\n",
        "            self.cond_lambda(X[:, i, :].clone(), X[:, :i, :].clone())\n",
        "            for i in range(seq_len) ]\n",
        "        lams     = torch.stack(lams, dim=0).T                                   # [ batch_size, seq_len ]\n",
        "        ## log-likelihood\n",
        "        mask     = ts > 0                                                       # [ batch_size, seq_len ]\n",
        "        sumlog   = (torch.log(lams + 1e-8) * mask).sum()                        # [ seq_len ]\n",
        "\n",
        "        baserate = torch.sum(self._mu * (self.T[1] - self.T[0]))\n",
        "\n",
        "        temp_int = 1 - torch.exp(-self.kernel._beta[ms] * (self.T[1] - ts))            # [ batch_size, seq_len ]\n",
        "        alpha_int = self.kernel._alphas[:, ms].sum(0)\n",
        "        integ = alpha_int * temp_int * mask\n",
        "        integ = integ.sum()\n",
        "\n",
        "        loglik = sumlog - baserate - integ\n",
        "        \n",
        "        return loglik"
      ],
      "metadata": {
        "id": "1tT8YBBSzUqE"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training Function"
      ],
      "metadata": {
        "id": "xRLGu2Vy7r-3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train(model,\n",
        "          train_data,\n",
        "          device,\n",
        "          modelname,\n",
        "          rootpath,\n",
        "          num_epochs=20, \n",
        "          lr=1e-4,\n",
        "          batch_size=5,\n",
        "          print_iter=2,\n",
        "          tol=1e-2,\n",
        "          testing=False,\n",
        "          test_data=None,\n",
        "          save_model=False):\n",
        "    \"\"\"training procedure for one epoch\"\"\"\n",
        "\n",
        "    clipper1  = NonNegativeClipper()\n",
        "    ##clipper2  = ProximityClipper(coords, k=k)\n",
        "    # NOTE: gradient for loss is expected to be None, \n",
        "    #       since it is not leaf node. (it's root node)\n",
        "\n",
        "    path = rootpath + \"/Results/saved_models/%s\" % modelname\n",
        "    \n",
        "    if save_model:\n",
        "        if os.path.exists(path): \n",
        "            print(\"Duplicated folder!\")\n",
        "            return None\n",
        "        else:\n",
        "            os.makedirs(path)\n",
        "\n",
        "    model.to(device)\n",
        "    optimizer = optim.Adadelta(model.parameters(), lr=lr)\n",
        "\n",
        "    best_lglk = -np.inf\n",
        "    prev_lglk = -np.inf\n",
        "    no_incre = 0\n",
        "    converge = 0\n",
        "    _lr = lr\n",
        "    n_batches = int(train_data.shape[0] / batch_size) # number of batches\n",
        "\n",
        "    train_llk = []\n",
        "    test_llk = []\n",
        "\n",
        "    for i in range(num_epochs):\n",
        "        try:\n",
        "            epoch_llk_loss = 0\n",
        "            epoch_l1_loss  = 0\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            for b in range(n_batches):\n",
        "                idx  = np.arange(batch_size * b, batch_size * (b + 1))\n",
        "                data = train_data[idx]\n",
        "                loglik = model(data.to(device))\n",
        "                loss      = - loglik\n",
        "\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "                model.apply(clipper1)\n",
        "            \n",
        "                epoch_llk_loss += loglik.item()\n",
        "            \n",
        "            event_num = (train_data[..., 0] > 0).sum()\n",
        "            event_llk = epoch_llk_loss / event_num\n",
        "\n",
        "            if (i+1) % print_iter == 0:\n",
        "\n",
        "                train_llk.append(event_llk)\n",
        "                print(\"[%s] Epoch: %d\\tTrain Loglik: %.3e\\t stag: %d converge: %d\" % (arrow.now(), \n",
        "                    i, \n",
        "                    event_llk,\n",
        "                    no_incre,\n",
        "                    converge))\n",
        "                \n",
        "                if testing:\n",
        "                    with torch.no_grad():\n",
        "                        test_loglik = model(test_data.to(device))\n",
        "                        test_event_num = (test_data[..., 0] > 0).sum()\n",
        "                        test_event_llk = test_loglik / test_event_num\n",
        "\n",
        "                        test_llk.append(test_event_llk)\n",
        "                \n",
        "                        print(\"[%s] Epoch: %d\\tTest Loglik: %.3e\" % (arrow.now(), \n",
        "                            i, \n",
        "                            test_event_llk))\n",
        "                \n",
        "            if event_llk > best_lglk:\n",
        "                best_lglk = event_llk\n",
        "                no_incre = 0\n",
        "            else:\n",
        "                no_incre += 1\n",
        "            if no_incre == 10:\n",
        "                print(\"Learning rate decrease!\")\n",
        "                _lr = _lr / 10\n",
        "                optimizer = optim.Adadelta(model.parameters(), lr=_lr)\n",
        "                no_incre = 0\n",
        "                # best_lglk = -np.inf\n",
        "            if np.abs(event_llk - prev_lglk) > tol:\n",
        "                converge = 0\n",
        "            else:\n",
        "                converge += 1\n",
        "\n",
        "            prev_lglk = event_llk\n",
        "            \n",
        "            if converge == 50:\n",
        "                return train_llk, test_llk\n",
        "\n",
        "            if save_model:\n",
        "                model.cpu()\n",
        "                torch.save(model.state_dict(), \"%s/%s-%d.pth\" % (path, modelname, i))\n",
        "                model.to(device)\n",
        "\n",
        "        except KeyboardInterrupt:\n",
        "            break\n",
        "    \n",
        "    return train_llk, test_llk\n",
        "\n",
        "\n",
        "class NonNegativeClipper(object):\n",
        "    \"\"\"\n",
        "    References:\n",
        "    https://discuss.pytorch.org/t/restrict-range-of-variable-during-gradient-descent/1933\n",
        "    https://discuss.pytorch.org/t/set-constraints-on-parameters-or-layers/23620/3\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self):\n",
        "        pass\n",
        "\n",
        "    def __call__(self, module):\n",
        "        \"\"\"enforce non-negative constraints\"\"\"\n",
        "        if hasattr(module, '_mu'):\n",
        "            _mu = module._mu.data\n",
        "            module._mu.data = torch.clamp(_mu, min=0.)\n",
        "        # Multivariate Hawkes\n",
        "        if hasattr(module, '_alphas'):\n",
        "            _alphas = module._alphas.data\n",
        "            module._alphas.data = torch.clamp(_alphas, min=1e-5)\n",
        "        if hasattr(module, '_beta'):\n",
        "            _beta  = module._beta.data\n",
        "            module._beta.data  = torch.clamp(_beta, min=1e-5)"
      ],
      "metadata": {
        "id": "atDQ4YEz6yt_"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "C4Qh4G_eIY-c"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Loading"
      ],
      "metadata": {
        "id": "So5GQPfxLFdA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "rootpath = \"/content/drive/MyDrive/CSE8803_DSE\"\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "2RN1CBuALGrL"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_raw_data = np.load(rootpath + \"/data/train.npy\", allow_pickle=True)\n",
        "test_raw_data = np.load(rootpath + \"/data/test.npy\", allow_pickle=True)"
      ],
      "metadata": {
        "id": "G0vgKrJ9LPi0"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lens_test = [test_raw_data[i][0].shape[0] for i in range(test_raw_data.shape[0])]\n",
        "max_len_test = max(lens_test)\n",
        "\n",
        "test_data = np.zeros((test_raw_data.shape[0], max_len_test, 2))\n",
        "for i in range(test_raw_data.shape[0]):\n",
        "    data = test_raw_data[i][0]\n",
        "    data[:, 0] = data[:, 0] - 200\n",
        "    test_data[i, :lens_test[i], :] = data"
      ],
      "metadata": {
        "id": "wAgy9zWeO0VQ"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data_all = []\n",
        "\n",
        "for i in range(train_raw_data.shape[0]):\n",
        "\n",
        "    train_data_list = []\n",
        "    for k in range(4):\n",
        "        dat = train_raw_data[i][k]\n",
        "        mask = dat[:, 0] > 0\n",
        "        dat[:, 0] = (dat[:, 0] - k * 50) * mask\n",
        "        train_data_list.append(dat)\n",
        "    train_data_list = np.stack(train_data_list, axis=0)\n",
        "\n",
        "    train_data_all.append(train_data_list)\n",
        "\n",
        "lens_train = [data.shape[1] for data in train_data_all]\n",
        "max_len_train = max(lens_train)\n",
        "\n",
        "train_data = np.zeros((train_raw_data.shape[0], train_raw_data.shape[1], max_len_train, 2))\n",
        "for i in range(train_raw_data.shape[0]):\n",
        "    train_data[i, :, :lens_train[i], :] = train_data_all[i]"
      ],
      "metadata": {
        "id": "5TOuAPWwuh0z"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Training"
      ],
      "metadata": {
        "id": "kVALX7t0O9K9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "############### Training without regularization ################\n",
        "\n",
        "seed = 500\n",
        "torch.random.manual_seed(seed)\n",
        "np.random.seed(seed)\n",
        "random.seed(seed)\n",
        "\n",
        "# training configurations\n",
        "batch_size = 4\n",
        "num_epochs = 1000\n",
        "lr         = 1e0\n",
        "\n",
        "# model configurations\n",
        "T          = np.array([0., 50.])\n",
        "data_dim   = 2\n",
        "device     = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "# device     = \"cpu\"\n",
        "n_class    = 11\n",
        "n_state    = train_raw_data.shape[0]\n",
        "\n",
        "# mask = train_data[:, :, 0] > 0\n",
        "# v, c = np.unique(train_data[mask][:, 0], return_counts=True)\n",
        "# mu = c / train_data.shape[0] / 50 / 10\n",
        "# print(mu)\n",
        "# mu = np.zeros(n_class) * 1e-3\n",
        "\n",
        "test_event_num = (test_data[:, :, 0] > 0).sum(1)\n",
        "wo_fed_test_llk = np.zeros((n_state, n_state))\n",
        "\n",
        "dir_path = rootpath + \"/Results/saved_models\"\n",
        "if os.path.exists(dir_path): \n",
        "    print(\"Duplicated folder!\")\n",
        "else:\n",
        "    os.makedirs(dir_path)\n",
        "\n",
        "for i in range(n_state):\n",
        "\n",
        "    alphas = np.random.uniform(low=0.0, high=1.0, size=(n_class, n_class))\n",
        "    beta   = np.random.uniform(low=0.0, high=1.0, size=(n_class))\n",
        "\n",
        "    train_set  = torch.FloatTensor(train_data[i])\n",
        "\n",
        "    # mask = train_set[..., 0] > 0\n",
        "    # v, c = np.unique(train_set[mask][:, 0], return_counts=True)\n",
        "    # mu = c / train_set.shape[0] / 50 \n",
        "    # print(mu)\n",
        "    mu = np.ones(n_class) * 1e-2\n",
        "\n",
        "    init_model = MultivariateExponentialHawkes(T=T, mu=mu, alphas=alphas, beta=beta,\n",
        "                                            data_dim=data_dim, device=device)\n",
        "    # init_model.load_state_dict(torch.load(\"/content/drive/MyDrive/Neural_Kernel/Deep_Fourier_Kernel/Results/saved_models/%s_stone12/%s-99.pth\" % (modelname, modelname)))\n",
        "    # # # define point process generator and generator synthetic data\n",
        "    # # generator  = TemporalPointProcessGenerator(trg_model, upper_bound=1e+2)\n",
        "    # # data, size = generator.generate(T=T, batch_size=2000, min_n_points=5, verbose=False)\n",
        "    # # np.save(\"%s/data-1d-n2000_exponential_targetmodel-%d.npy\" % (rootpath, seed), data)\n",
        "    # # print(data.shape)\n",
        "\n",
        "    # training data preparation\n",
        "    # train_dataset = TensorDataset(train_data)\n",
        "    # train_loader  = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "    # training\n",
        "    modelname  = \"No_Fed-constant-mu%.3f-%d_train-state-%d\" % (0.01, seed, i)\n",
        "    train_llk, test_llk = train(init_model, train_set, device=device,\n",
        "                modelname=modelname, rootpath=rootpath,\n",
        "                num_epochs=num_epochs, lr=lr, batch_size=batch_size,\n",
        "                print_iter=50, tol=1e-3, testing=False, test_data=None, save_model=False)\n",
        "    \n",
        "    modelpath = dir_path + \"/%s.pth\" % modelname\n",
        "    init_model.cpu()\n",
        "    torch.save(init_model.state_dict(), \"%s\" % modelpath)\n",
        "    \n",
        "    # with torch.no_grad():\n",
        "    #     for j in range(n_state):\n",
        "    #         test_set   = torch.FloatTensor(test_data[[j]])\n",
        "    #         test_loglik = init_model(test_set.to(device))\n",
        "    #         test_event_llk = test_loglik / test_event_num[j]\n",
        "    #         print(\"Train/Test State (%d, %d),  testing llk: %.5f\" % (i, j, test_event_llk))\n",
        "    #         wo_fed_test_llk[i, j] = test_event_llk"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "btTPjjHOO-fM",
        "outputId": "a821ef0d-ee69-4142-8c94-8c48cace1912"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Duplicated folder!\n",
            "[2022-11-30T03:38:41.329257+00:00] Epoch: 49\tTrain Loglik: -2.916e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:38:46.230578+00:00] Epoch: 99\tTrain Loglik: -2.245e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:38:51.141478+00:00] Epoch: 149\tTrain Loglik: -1.909e+00\t stag: 6 converge: 0\n",
            "[2022-11-30T03:38:55.982478+00:00] Epoch: 199\tTrain Loglik: -1.810e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:39:01.315187+00:00] Epoch: 249\tTrain Loglik: -1.754e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:39:06.815107+00:00] Epoch: 299\tTrain Loglik: -1.708e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:39:11.658015+00:00] Epoch: 349\tTrain Loglik: -1.666e+00\t stag: 0 converge: 9\n",
            "[2022-11-30T03:39:16.515635+00:00] Epoch: 399\tTrain Loglik: -1.624e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:39:23.087899+00:00] Epoch: 449\tTrain Loglik: -1.584e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:39:28.032346+00:00] Epoch: 499\tTrain Loglik: -1.537e+00\t stag: 0 converge: 24\n",
            "[2022-11-30T03:39:35.253912+00:00] Epoch: 49\tTrain Loglik: -2.400e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:39:40.260033+00:00] Epoch: 99\tTrain Loglik: -2.184e+00\t stag: 0 converge: 34\n",
            "[2022-11-30T03:39:46.531541+00:00] Epoch: 49\tTrain Loglik: -2.498e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:39:51.540083+00:00] Epoch: 99\tTrain Loglik: -1.962e+00\t stag: 4 converge: 0\n",
            "[2022-11-30T03:39:56.478206+00:00] Epoch: 149\tTrain Loglik: -1.902e+00\t stag: 0 converge: 7\n",
            "[2022-11-30T03:40:05.485242+00:00] Epoch: 49\tTrain Loglik: -2.356e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:40:10.385953+00:00] Epoch: 99\tTrain Loglik: -2.175e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:40:15.301858+00:00] Epoch: 149\tTrain Loglik: -2.104e+00\t stag: 0 converge: 19\n",
            "[2022-11-30T03:40:23.319541+00:00] Epoch: 49\tTrain Loglik: -2.975e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:40:28.413094+00:00] Epoch: 99\tTrain Loglik: -2.144e+00\t stag: 1 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:40:33.358639+00:00] Epoch: 149\tTrain Loglik: -2.052e+00\t stag: 0 converge: 24\n",
            "[2022-11-30T03:40:40.844543+00:00] Epoch: 49\tTrain Loglik: -2.637e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:40:45.783783+00:00] Epoch: 99\tTrain Loglik: -2.217e+00\t stag: 0 converge: 18\n",
            "[2022-11-30T03:40:53.878038+00:00] Epoch: 49\tTrain Loglik: -2.394e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:40:58.710206+00:00] Epoch: 99\tTrain Loglik: -2.224e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:41:03.656457+00:00] Epoch: 149\tTrain Loglik: -2.269e+00\t stag: 8 converge: 7\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:41:12.666022+00:00] Epoch: 49\tTrain Loglik: -2.136e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:41:17.537754+00:00] Epoch: 99\tTrain Loglik: -2.058e+00\t stag: 7 converge: 12\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:41:26.505160+00:00] Epoch: 49\tTrain Loglik: -2.394e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:41:32.643673+00:00] Epoch: 99\tTrain Loglik: -2.145e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:41:37.483821+00:00] Epoch: 149\tTrain Loglik: -1.978e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T03:41:42.484667+00:00] Epoch: 199\tTrain Loglik: -1.914e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:41:47.296336+00:00] Epoch: 249\tTrain Loglik: -1.854e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:41:52.268782+00:00] Epoch: 299\tTrain Loglik: -1.800e+00\t stag: 0 converge: 18\n",
            "[2022-11-30T03:41:57.057589+00:00] Epoch: 349\tTrain Loglik: -1.752e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:42:01.992105+00:00] Epoch: 399\tTrain Loglik: -1.702e+00\t stag: 0 converge: 7\n",
            "[2022-11-30T03:42:11.042327+00:00] Epoch: 49\tTrain Loglik: -2.397e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:42:15.898950+00:00] Epoch: 99\tTrain Loglik: -2.501e+00\t stag: 2 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:42:20.815631+00:00] Epoch: 149\tTrain Loglik: -2.115e+00\t stag: 3 converge: 32\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:42:27.347781+00:00] Epoch: 49\tTrain Loglik: -2.526e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:42:32.257659+00:00] Epoch: 99\tTrain Loglik: -2.206e+00\t stag: 4 converge: 0\n",
            "[2022-11-30T03:42:37.146818+00:00] Epoch: 149\tTrain Loglik: -2.139e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T03:42:43.730768+00:00] Epoch: 199\tTrain Loglik: -2.024e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:42:48.546655+00:00] Epoch: 249\tTrain Loglik: -1.964e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:42:53.419832+00:00] Epoch: 299\tTrain Loglik: -1.918e+00\t stag: 0 converge: 18\n",
            "[2022-11-30T03:43:01.249244+00:00] Epoch: 49\tTrain Loglik: -2.621e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:43:06.124112+00:00] Epoch: 99\tTrain Loglik: -2.099e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:11.145195+00:00] Epoch: 149\tTrain Loglik: -2.064e+00\t stag: 6 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:16.102012+00:00] Epoch: 199\tTrain Loglik: -2.053e+00\t stag: 6 converge: 45\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:21.354180+00:00] Epoch: 49\tTrain Loglik: -2.658e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:26.331188+00:00] Epoch: 99\tTrain Loglik: -2.147e+00\t stag: 7 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:31.353222+00:00] Epoch: 149\tTrain Loglik: -2.138e+00\t stag: 7 converge: 46\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:36.466892+00:00] Epoch: 49\tTrain Loglik: -2.145e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:43:41.365957+00:00] Epoch: 99\tTrain Loglik: -2.087e+00\t stag: 2 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:46.268517+00:00] Epoch: 149\tTrain Loglik: -2.147e+00\t stag: 1 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:51.246235+00:00] Epoch: 199\tTrain Loglik: -2.091e+00\t stag: 1 converge: 40\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:43:57.122041+00:00] Epoch: 49\tTrain Loglik: -2.675e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:44:01.978235+00:00] Epoch: 99\tTrain Loglik: -2.025e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:44:06.784627+00:00] Epoch: 149\tTrain Loglik: -1.808e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:44:11.648145+00:00] Epoch: 199\tTrain Loglik: -1.701e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:44:16.464537+00:00] Epoch: 249\tTrain Loglik: -1.643e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:44:21.283594+00:00] Epoch: 299\tTrain Loglik: -1.598e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:44:26.256577+00:00] Epoch: 349\tTrain Loglik: -1.561e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:44:31.253089+00:00] Epoch: 399\tTrain Loglik: -1.526e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:44:36.191712+00:00] Epoch: 449\tTrain Loglik: -1.491e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:44:41.290424+00:00] Epoch: 499\tTrain Loglik: -1.453e+00\t stag: 0 converge: 9\n",
            "[2022-11-30T03:44:47.014629+00:00] Epoch: 549\tTrain Loglik: -1.421e+00\t stag: 0 converge: 5\n",
            "[2022-11-30T03:44:57.493851+00:00] Epoch: 49\tTrain Loglik: -2.507e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:45:02.389864+00:00] Epoch: 99\tTrain Loglik: -2.105e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:45:07.272050+00:00] Epoch: 149\tTrain Loglik: -2.010e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:45:12.198697+00:00] Epoch: 199\tTrain Loglik: -1.925e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:45:17.028016+00:00] Epoch: 249\tTrain Loglik: -1.840e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:45:21.865940+00:00] Epoch: 299\tTrain Loglik: -1.788e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:45:26.824255+00:00] Epoch: 349\tTrain Loglik: -1.747e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:45:31.648892+00:00] Epoch: 399\tTrain Loglik: -1.714e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:45:36.569814+00:00] Epoch: 449\tTrain Loglik: -1.684e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:45:41.510411+00:00] Epoch: 499\tTrain Loglik: -1.654e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:45:46.555773+00:00] Epoch: 549\tTrain Loglik: -1.620e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:45:51.397662+00:00] Epoch: 599\tTrain Loglik: -1.582e+00\t stag: 0 converge: 3\n",
            "[2022-11-30T03:45:56.344344+00:00] Epoch: 649\tTrain Loglik: -1.538e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:46:02.959549+00:00] Epoch: 699\tTrain Loglik: -1.489e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:46:07.815523+00:00] Epoch: 749\tTrain Loglik: -1.445e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:46:12.594665+00:00] Epoch: 799\tTrain Loglik: -1.409e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:46:17.391978+00:00] Epoch: 849\tTrain Loglik: -1.372e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:46:22.184921+00:00] Epoch: 899\tTrain Loglik: -1.341e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:46:26.983708+00:00] Epoch: 949\tTrain Loglik: -1.310e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:46:31.880942+00:00] Epoch: 999\tTrain Loglik: -1.279e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:46:36.649876+00:00] Epoch: 49\tTrain Loglik: -2.275e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:46:41.448168+00:00] Epoch: 99\tTrain Loglik: -1.983e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:46:46.430527+00:00] Epoch: 149\tTrain Loglik: -1.902e+00\t stag: 0 converge: 8\n",
            "[2022-11-30T03:46:55.243125+00:00] Epoch: 49\tTrain Loglik: -2.479e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:00.103853+00:00] Epoch: 99\tTrain Loglik: -2.085e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T03:47:04.878418+00:00] Epoch: 149\tTrain Loglik: -1.945e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:47:09.819890+00:00] Epoch: 199\tTrain Loglik: -1.881e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:14.729486+00:00] Epoch: 249\tTrain Loglik: -1.775e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:19.505296+00:00] Epoch: 299\tTrain Loglik: -1.725e+00\t stag: 0 converge: 2\n",
            "[2022-11-30T03:47:24.333690+00:00] Epoch: 349\tTrain Loglik: -1.693e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:29.347540+00:00] Epoch: 399\tTrain Loglik: -1.655e+00\t stag: 0 converge: 34\n",
            "[2022-11-30T03:47:35.765940+00:00] Epoch: 49\tTrain Loglik: -2.968e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:40.680904+00:00] Epoch: 99\tTrain Loglik: -2.052e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:45.633667+00:00] Epoch: 149\tTrain Loglik: -1.840e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:47:50.499659+00:00] Epoch: 199\tTrain Loglik: -1.657e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:47:55.455502+00:00] Epoch: 249\tTrain Loglik: -1.561e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:48:00.301724+00:00] Epoch: 299\tTrain Loglik: -1.490e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:48:05.702485+00:00] Epoch: 349\tTrain Loglik: -1.435e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:48:13.447625+00:00] Epoch: 399\tTrain Loglik: -1.389e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:48:19.328182+00:00] Epoch: 449\tTrain Loglik: -1.347e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:48:24.152095+00:00] Epoch: 499\tTrain Loglik: -1.308e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:48:29.061273+00:00] Epoch: 549\tTrain Loglik: -1.262e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:48:34.181052+00:00] Epoch: 599\tTrain Loglik: -1.229e+00\t stag: 0 converge: 45\n",
            "[2022-11-30T03:48:39.517507+00:00] Epoch: 49\tTrain Loglik: -2.627e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:48:44.449794+00:00] Epoch: 99\tTrain Loglik: -2.312e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:48:49.499199+00:00] Epoch: 149\tTrain Loglik: -2.122e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:48:54.504602+00:00] Epoch: 199\tTrain Loglik: -2.062e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:48:59.481735+00:00] Epoch: 249\tTrain Loglik: -1.960e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:04.386025+00:00] Epoch: 299\tTrain Loglik: -1.899e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:09.298750+00:00] Epoch: 349\tTrain Loglik: -1.832e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:14.204862+00:00] Epoch: 399\tTrain Loglik: -1.780e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:20.630260+00:00] Epoch: 449\tTrain Loglik: -1.735e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:25.808749+00:00] Epoch: 499\tTrain Loglik: -1.694e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:30.632963+00:00] Epoch: 549\tTrain Loglik: -1.659e+00\t stag: 0 converge: 12\n",
            "[2022-11-30T03:49:39.149676+00:00] Epoch: 49\tTrain Loglik: -2.407e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:44.049041+00:00] Epoch: 99\tTrain Loglik: -2.200e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:48.972771+00:00] Epoch: 149\tTrain Loglik: -2.157e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:53.968256+00:00] Epoch: 199\tTrain Loglik: -2.089e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:49:58.929354+00:00] Epoch: 249\tTrain Loglik: -2.018e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:50:03.999766+00:00] Epoch: 299\tTrain Loglik: -1.979e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:50:08.847767+00:00] Epoch: 349\tTrain Loglik: -1.945e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:50:13.647734+00:00] Epoch: 399\tTrain Loglik: -1.916e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:50:18.482861+00:00] Epoch: 449\tTrain Loglik: -1.886e+00\t stag: 0 converge: 12\n",
            "[2022-11-30T03:50:26.903037+00:00] Epoch: 49\tTrain Loglik: -2.311e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:50:31.795349+00:00] Epoch: 99\tTrain Loglik: -2.116e+00\t stag: 3 converge: 12\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:50:40.427086+00:00] Epoch: 49\tTrain Loglik: -2.670e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:50:45.270395+00:00] Epoch: 99\tTrain Loglik: -2.120e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:50:50.217724+00:00] Epoch: 149\tTrain Loglik: -2.185e+00\t stag: 3 converge: 22\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:50:57.664270+00:00] Epoch: 49\tTrain Loglik: -2.653e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:51:02.634253+00:00] Epoch: 99\tTrain Loglik: -2.173e+00\t stag: 3 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:51:07.472482+00:00] Epoch: 149\tTrain Loglik: -2.066e+00\t stag: 0 converge: 34\n",
            "[2022-11-30T03:51:13.889951+00:00] Epoch: 49\tTrain Loglik: -2.196e+00\t stag: 1 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:51:18.809339+00:00] Epoch: 99\tTrain Loglik: -2.293e+00\t stag: 1 converge: 20\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:51:28.373802+00:00] Epoch: 49\tTrain Loglik: -2.323e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:51:33.200776+00:00] Epoch: 99\tTrain Loglik: -2.201e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:51:38.132173+00:00] Epoch: 149\tTrain Loglik: -2.136e+00\t stag: 4 converge: 0\n",
            "[2022-11-30T03:51:42.982210+00:00] Epoch: 199\tTrain Loglik: -2.092e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:51:47.814151+00:00] Epoch: 249\tTrain Loglik: -2.047e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T03:51:52.871140+00:00] Epoch: 299\tTrain Loglik: -2.011e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:51:57.728748+00:00] Epoch: 349\tTrain Loglik: -1.981e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:52:02.823345+00:00] Epoch: 399\tTrain Loglik: -1.958e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:52:07.649128+00:00] Epoch: 449\tTrain Loglik: -1.937e+00\t stag: 1 converge: 3\n",
            "[2022-11-30T03:52:16.989397+00:00] Epoch: 49\tTrain Loglik: -2.678e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:52:21.827635+00:00] Epoch: 99\tTrain Loglik: -2.190e+00\t stag: 1 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:52:26.851914+00:00] Epoch: 149\tTrain Loglik: -2.065e+00\t stag: 8 converge: 17\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:52:34.927840+00:00] Epoch: 49\tTrain Loglik: -2.387e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:52:41.626678+00:00] Epoch: 99\tTrain Loglik: -2.851e+00\t stag: 7 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:52:46.567779+00:00] Epoch: 149\tTrain Loglik: -2.260e+00\t stag: 7 converge: 36\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:52:52.723785+00:00] Epoch: 49\tTrain Loglik: -2.218e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:52:57.617101+00:00] Epoch: 99\tTrain Loglik: -2.129e+00\t stag: 0 converge: 9\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:53:06.464836+00:00] Epoch: 49\tTrain Loglik: -2.453e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:53:11.292573+00:00] Epoch: 99\tTrain Loglik: -2.063e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T03:53:16.194951+00:00] Epoch: 149\tTrain Loglik: -2.081e+00\t stag: 2 converge: 0\n",
            "[2022-11-30T03:53:21.068155+00:00] Epoch: 199\tTrain Loglik: -1.949e+00\t stag: 2 converge: 0\n",
            "[2022-11-30T03:53:25.962765+00:00] Epoch: 249\tTrain Loglik: -1.817e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:53:30.771839+00:00] Epoch: 299\tTrain Loglik: -1.746e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:53:35.710846+00:00] Epoch: 349\tTrain Loglik: -1.686e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:53:40.739695+00:00] Epoch: 399\tTrain Loglik: -1.638e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:53:45.821952+00:00] Epoch: 449\tTrain Loglik: -1.596e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:53:50.608963+00:00] Epoch: 499\tTrain Loglik: -1.553e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:53:55.492675+00:00] Epoch: 549\tTrain Loglik: -1.520e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:54:00.541227+00:00] Epoch: 599\tTrain Loglik: -1.489e+00\t stag: 0 converge: 7\n",
            "[2022-11-30T03:54:05.473798+00:00] Epoch: 649\tTrain Loglik: -1.461e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:54:10.364196+00:00] Epoch: 699\tTrain Loglik: -1.435e+00\t stag: 0 converge: 39\n",
            "[2022-11-30T03:54:16.082636+00:00] Epoch: 49\tTrain Loglik: -2.943e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:54:20.890881+00:00] Epoch: 99\tTrain Loglik: -2.105e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:54:25.692015+00:00] Epoch: 149\tTrain Loglik: -1.932e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:54:30.930350+00:00] Epoch: 199\tTrain Loglik: -1.892e+00\t stag: 0 converge: 33\n",
            "[2022-11-30T03:54:37.285749+00:00] Epoch: 49\tTrain Loglik: -2.672e+00\t stag: 2 converge: 0\n",
            "[2022-11-30T03:54:42.529502+00:00] Epoch: 99\tTrain Loglik: -2.187e+00\t stag: 5 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:54:48.688378+00:00] Epoch: 149\tTrain Loglik: -2.169e+00\t stag: 0 converge: 32\n",
            "[2022-11-30T03:54:55.381684+00:00] Epoch: 49\tTrain Loglik: -3.228e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:55:00.206751+00:00] Epoch: 99\tTrain Loglik: -2.087e+00\t stag: 6 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:55:05.199203+00:00] Epoch: 149\tTrain Loglik: -2.215e+00\t stag: 6 converge: 35\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:55:11.522802+00:00] Epoch: 49\tTrain Loglik: -2.608e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:55:16.551874+00:00] Epoch: 99\tTrain Loglik: -2.171e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:55:21.466814+00:00] Epoch: 149\tTrain Loglik: -2.124e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:55:26.401013+00:00] Epoch: 199\tTrain Loglik: -2.077e+00\t stag: 0 converge: 25\n",
            "[2022-11-30T03:55:33.737849+00:00] Epoch: 49\tTrain Loglik: -3.019e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:55:38.667442+00:00] Epoch: 99\tTrain Loglik: -2.230e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:55:43.681882+00:00] Epoch: 149\tTrain Loglik: -1.986e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:55:48.533383+00:00] Epoch: 199\tTrain Loglik: -2.002e+00\t stag: 2 converge: 0\n",
            "[2022-11-30T03:55:53.622411+00:00] Epoch: 249\tTrain Loglik: -1.854e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:56:00.104957+00:00] Epoch: 299\tTrain Loglik: -1.763e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:56:05.029520+00:00] Epoch: 349\tTrain Loglik: -1.707e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:56:10.043187+00:00] Epoch: 399\tTrain Loglik: -1.656e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:56:15.055871+00:00] Epoch: 449\tTrain Loglik: -1.616e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:56:19.879341+00:00] Epoch: 499\tTrain Loglik: -1.579e+00\t stag: 0 converge: 11\n",
            "[2022-11-30T03:56:28.436480+00:00] Epoch: 49\tTrain Loglik: -2.535e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:56:33.286651+00:00] Epoch: 99\tTrain Loglik: -2.182e+00\t stag: 9 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:56:38.316667+00:00] Epoch: 149\tTrain Loglik: -2.158e+00\t stag: 9 converge: 47\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:56:43.572404+00:00] Epoch: 49\tTrain Loglik: -2.593e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:56:48.528750+00:00] Epoch: 99\tTrain Loglik: -2.113e+00\t stag: 0 converge: 14\n",
            "[2022-11-30T03:56:56.867275+00:00] Epoch: 49\tTrain Loglik: -2.291e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:57:01.898318+00:00] Epoch: 99\tTrain Loglik: -2.327e+00\t stag: 7 converge: 26\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:57:09.281927+00:00] Epoch: 49\tTrain Loglik: -2.328e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:57:14.229831+00:00] Epoch: 99\tTrain Loglik: -2.423e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T03:57:19.274915+00:00] Epoch: 149\tTrain Loglik: -2.170e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:57:24.165528+00:00] Epoch: 199\tTrain Loglik: -2.106e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:57:29.167648+00:00] Epoch: 249\tTrain Loglik: -2.025e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:57:34.369945+00:00] Epoch: 299\tTrain Loglik: -1.981e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:57:39.462503+00:00] Epoch: 349\tTrain Loglik: -1.939e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:57:44.399353+00:00] Epoch: 399\tTrain Loglik: -1.907e+00\t stag: 1 converge: 1\n",
            "[2022-11-30T03:57:49.374092+00:00] Epoch: 449\tTrain Loglik: -1.860e+00\t stag: 0 converge: 4\n",
            "[2022-11-30T03:57:54.271173+00:00] Epoch: 499\tTrain Loglik: -1.825e+00\t stag: 0 converge: 25\n",
            "[2022-11-30T03:58:01.493710+00:00] Epoch: 49\tTrain Loglik: -2.477e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:58:08.424731+00:00] Epoch: 99\tTrain Loglik: -2.069e+00\t stag: 2 converge: 1\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:58:18.121733+00:00] Epoch: 49\tTrain Loglik: -2.379e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:58:23.034751+00:00] Epoch: 99\tTrain Loglik: -2.195e+00\t stag: 7 converge: 6\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:58:32.296084+00:00] Epoch: 49\tTrain Loglik: -2.479e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:58:37.082958+00:00] Epoch: 99\tTrain Loglik: -2.097e+00\t stag: 3 converge: 2\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:58:46.667873+00:00] Epoch: 49\tTrain Loglik: -2.379e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T03:58:51.521242+00:00] Epoch: 99\tTrain Loglik: -2.293e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T03:58:56.554471+00:00] Epoch: 149\tTrain Loglik: -2.253e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T03:59:01.349521+00:00] Epoch: 199\tTrain Loglik: -2.241e+00\t stag: 1 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:06.181434+00:00] Epoch: 249\tTrain Loglik: -2.238e+00\t stag: 1 converge: 30\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:13.334260+00:00] Epoch: 49\tTrain Loglik: -3.055e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:20.088522+00:00] Epoch: 99\tTrain Loglik: -2.430e+00\t stag: 2 converge: 1\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:29.986934+00:00] Epoch: 49\tTrain Loglik: -2.464e+00\t stag: 1 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:34.849809+00:00] Epoch: 99\tTrain Loglik: -2.292e+00\t stag: 0 converge: 19\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:42.788114+00:00] Epoch: 49\tTrain Loglik: -2.479e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T03:59:47.759868+00:00] Epoch: 99\tTrain Loglik: -2.184e+00\t stag: 0 converge: 17\n",
            "[2022-11-30T03:59:55.797674+00:00] Epoch: 49\tTrain Loglik: -2.354e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:00:00.814751+00:00] Epoch: 99\tTrain Loglik: -2.166e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T04:00:05.717247+00:00] Epoch: 149\tTrain Loglik: -2.087e+00\t stag: 6 converge: 0\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:10.681658+00:00] Epoch: 199\tTrain Loglik: -2.059e+00\t stag: 0 converge: 45\n",
            "[2022-11-30T04:00:16.003458+00:00] Epoch: 49\tTrain Loglik: -2.381e+00\t stag: 2 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:20.958928+00:00] Epoch: 99\tTrain Loglik: -2.351e+00\t stag: 7 converge: 26\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:28.259013+00:00] Epoch: 49\tTrain Loglik: -2.306e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:33.248064+00:00] Epoch: 99\tTrain Loglik: -2.283e+00\t stag: 3 converge: 22\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:40.886320+00:00] Epoch: 49\tTrain Loglik: -2.323e+00\t stag: 0 converge: 0\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:45.973916+00:00] Epoch: 99\tTrain Loglik: -2.653e+00\t stag: 7 converge: 26\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "Learning rate decrease!\n",
            "[2022-11-30T04:00:53.166291+00:00] Epoch: 49\tTrain Loglik: -2.107e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:00:58.034883+00:00] Epoch: 99\tTrain Loglik: -1.971e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T04:01:03.117739+00:00] Epoch: 149\tTrain Loglik: -1.940e+00\t stag: 5 converge: 0\n",
            "[2022-11-30T04:01:08.076509+00:00] Epoch: 199\tTrain Loglik: -1.916e+00\t stag: 3 converge: 0\n",
            "[2022-11-30T04:01:12.995902+00:00] Epoch: 249\tTrain Loglik: -1.898e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T04:01:17.941332+00:00] Epoch: 299\tTrain Loglik: -1.880e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T04:01:22.852029+00:00] Epoch: 349\tTrain Loglik: -1.861e+00\t stag: 0 converge: 5\n",
            "[2022-11-30T04:01:33.775122+00:00] Epoch: 49\tTrain Loglik: -2.936e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:01:38.624275+00:00] Epoch: 99\tTrain Loglik: -2.269e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T04:01:43.577426+00:00] Epoch: 149\tTrain Loglik: -2.071e+00\t stag: 1 converge: 0\n",
            "[2022-11-30T04:01:48.415113+00:00] Epoch: 199\tTrain Loglik: -1.966e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:01:53.336941+00:00] Epoch: 249\tTrain Loglik: -1.901e+00\t stag: 0 converge: 1\n",
            "[2022-11-30T04:01:58.156378+00:00] Epoch: 299\tTrain Loglik: -1.836e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:03.213632+00:00] Epoch: 349\tTrain Loglik: -1.778e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:08.177266+00:00] Epoch: 399\tTrain Loglik: -1.721e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:13.184399+00:00] Epoch: 449\tTrain Loglik: -1.668e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:18.108830+00:00] Epoch: 499\tTrain Loglik: -1.624e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:23.147216+00:00] Epoch: 549\tTrain Loglik: -1.582e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:28.097116+00:00] Epoch: 599\tTrain Loglik: -1.544e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:32.959131+00:00] Epoch: 649\tTrain Loglik: -1.517e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:39.155109+00:00] Epoch: 699\tTrain Loglik: -1.481e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:44.374449+00:00] Epoch: 749\tTrain Loglik: -1.461e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:49.290678+00:00] Epoch: 799\tTrain Loglik: -1.429e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:54.164893+00:00] Epoch: 849\tTrain Loglik: -1.408e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:02:59.168218+00:00] Epoch: 899\tTrain Loglik: -1.381e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:03:04.112905+00:00] Epoch: 949\tTrain Loglik: -1.360e+00\t stag: 0 converge: 0\n",
            "[2022-11-30T04:03:08.993362+00:00] Epoch: 999\tTrain Loglik: -1.336e+00\t stag: 0 converge: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0XtInFK1kjP-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "VG9gJ4wjkjSP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Tb8PddVekjYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.save(dir_path + \"/%s_testing_llk.npy\" % modelname, wo_fed_test_llk)"
      ],
      "metadata": {
        "id": "TKVLCcoQ_p0_"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(init_model.kernel._alphas.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "6uUxChZZQOSn",
        "outputId": "fd718860-1e9e-4f12-a563-e2b5603c16f9"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fe4d804add0>"
            ]
          },
          "metadata": {},
          "execution_count": 21
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANF0lEQVR4nO3df6zddX3H8der90dvb/lRoKWTFmwVZDYspuyqIJlZKIswFfbHYiBinNnWZJmKRmNwS8b+NIsh+gcx6RB1kUBMJRlxRNCqWRah4bYlQlsYpSAttHKpUmqBe3tv3/vjHpfbu15azvd9frj385E0vedHXt937r2v+zk/P8cRIQD//y3q9QAAuoOyA0VQdqAIyg4UQdmBIga7ebBhL44RLU3L81Du+HFsOi3Li4fTsiQpJqdy884eTc3z4ddy8waTf7bTeT/bmfPyfoclaeDQ0bSsN3RUUzHpk13W1bKPaKne7w1peYPLV6ZlSdL0wV+lZQ1ctDYtS5Jm9jybmjf5J+9NzVv8H4+m5g2cuyI1b2ZiIi3rN9dfmZYlSed8++G0rK2xZcHLuBkPFEHZgSIoO1AEZQeKoOxAEY3Kbvta20/Z3mP71qyhAORru+y2ByTdIek6Sesk3WR7XdZgAHI1WdnfJ2lPROyNiClJ90q6IWcsANmalH2VpH1zTu9vnXcC2xttj9seP6bJBocD0ETHH6CLiE0RMRYRY0Na3OnDAVhAk7K/IOnCOadXt84D0IealP1RSZfYXmt7WNKNku7PGQtAtrbfCBMR07Y/LelBSQOS7oqInWmTAUjV6F1vEfGApAeSZgHQQbyCDiiCsgNFUHagCMoOFNHVbak0ukS+7I/S4l5+9xlpWZK07N/ytqXKlr0n26LJ46l5h/4md6umFY/8JjVPidtSZW4j1U2s7EARlB0ogrIDRVB2oAjKDhRB2YEiKDtQBGUHiqDsQBGUHSiCsgNFUHagCMoOFEHZgSIoO1AEZQeKoOxAEZQdKIKyA0V0dQ86v/6G/PjTaXmHP7o+LUuSliVmzSw/MzFN0p7p1Lglz7ycmnd8+PzUPO15LjcPrOxAFZQdKIKyA0VQdqAIyg4UQdmBItouu+0Lbf/U9i7bO23fkjkYgFxNnmeflvSFiNhu+0xJ22z/KCJ2Jc0GIFHbK3tEHIiI7a2vj0jaLWlV1mAAcqW8gs72GknrJW09yWUbJW2UpBEvzTgcgDY0foDO9hmSvi/pcxHx6vzLI2JTRIxFxNiwFjc9HIA2NSq77SHNFv3uiLgvZyQAndDk0XhL+qak3RFxe95IADqhycp+laRPSLra9mOtf3+eNBeAZG0/QBcR/yXJibMA6CBeQQcUQdmBIig7UERXt6XSwIB89llpcW//p4fTstI98ovUuL1fuTI171137EvNO7RuKDVv5A9ytxw7964+/l3pElZ2oAjKDhRB2YEiKDtQBGUHiqDsQBGUHSiCsgNFUHagCMoOFEHZgSIoO1AEZQeKoOxAEZQdKIKyA0VQdqAIyg4UQdmBIrq7B12ENDnZ1UO+FQ+++Fha1thtf5eWJUkX//OO1LzpqWOpeRfcfiA17/Xr/zg1z0PDaVkxM5OWJUk6npy3AFZ2oAjKDhRB2YEiKDtQBGUHiqDsQBGNy257wPYO2z/IGAhAZ2Ss7LdI2p2QA6CDGpXd9mpJH5Z0Z844ADql6cr+NUlfknR8oSvY3mh73Pb4VLze8HAA2tV22W1/RNJLEbHtza4XEZsiYiwixoa9pN3DAWioycp+laTrbT8n6V5JV9v+bspUANK1XfaI+HJErI6INZJulPSTiLg5bTIAqXieHSgi5S2uEfEzST/LyALQGazsQBGUHSiCsgNFUHagiK7uQXd8dLEmL784LW//hrx9xSTpukvPTMta8Y5X0rIk6fgbb6TmpVs0kBq3dEvu2y0iMeuZf3lvYpr0zi8+kpq3EFZ2oAjKDhRB2YEiKDtQBGUHiqDsQBGUHSiCsgNFUHagCMoOFEHZgSIoO1AEZQeKoOxAEZQdKIKyA0VQdqAIyg4UQdmBIrq6B50sHR/O+/uy5h8fTsuSJL3n3WlRi47kfmLtgh+T26aB5eel5s28fCg1z287PzXvwIdWpmW984s/T8vqJlZ2oAjKDhRB2YEiKDtQBGUHiqDsQBGNym57me3Ntp+0vdv2lVmDAcjV9Hn2r0v6YUT8pe1hSaMJMwHogLbLbvtsSR+U9FeSFBFTkqZyxgKQrcnN+LWSJiR9y/YO23faXjr/SrY32h63PX5s6miDwwFooknZByVdLukbEbFe0lFJt86/UkRsioixiBgbGv4/fwsAdEmTsu+XtD8itrZOb9Zs+QH0obbLHhEHJe2zfWnrrA2SdqVMBSBd00fjPyPp7tYj8Xslfar5SAA6oVHZI+IxSWNJswDoIF5BBxRB2YEiKDtQBGUHinBEdO1gZ/nceL83dO14vXT441ek5i09eCw1b/G+V1LzZv77mdS8fnbob3Pf73Xev+btpbg1tujV+LVPdhkrO1AEZQeKoOxAEZQdKIKyA0VQdqAIyg4UQdmBIig7UARlB4qg7EARlB0ogrIDRVB2oAjKDhRB2YEiKDtQBGUHiqDsQBFNPxHmLfHIYg2845K0vMkLzkrLkqTBLdvSspY9eSQtS5Ji287UPF28Njcv2cCKFal5MxMTaVkrf/xiWpYkTaemLYyVHSiCsgNFUHagCMoOFEHZgSIald32523vtP2E7Xtsj2QNBiBX22W3vUrSZyWNRcRlkgYk3Zg1GIBcTW/GD0paYntQ0qik3CcgAaRpu+wR8YKkr0p6XtIBSYcj4qH517O90fa47fGpmdfanxRAI01uxp8j6QZJayVdIGmp7ZvnXy8iNkXEWESMDQ+Mtj8pgEaa3Iy/RtKzETEREcck3SfpAzljAcjWpOzPS7rC9qhtS9ogaXfOWACyNbnPvlXSZknbJT3eytqUNBeAZI3e9RYRt0m6LWkWAB3EK+iAIig7UARlB4qg7EARXd2WStPT0sSv0+JGfvt6WpYkac1FaVHHRobSsiTJqWmSXnk1OzGVh3O/f1MfGkvLWvTzp9KyuomVHSiCsgNFUHagCMoOFEHZgSIoO1AEZQeKoOxAEZQdKIKyA0VQdqAIyg4UQdmBIig7UARlB4qg7EARlB0ogrIDRVB2oIju7kE3NKRYdX5a3PEn96ZlSdKiM5amZR26bFValiSdv20kNc8juXnZYnIyNe/IRXl72g0/9Nu0rG5iZQeKoOxAEZQdKIKyA0VQdqAIyg4Uccqy277L9ku2n5hz3rm2f2T76db/53R2TABNnc7K/m1J184771ZJWyLiEklbWqcB9LFTlj0i/lPS/E9jvEHSd1pff0fSXyTPBSBZu/fZV0bEgdbXByWtXOiKtjfaHrc9PjX9WpuHA9BU4wfoIiIkxZtcvikixiJibHhwtOnhALSp3bL/yvbbJKn1/0t5IwHohHbLfr+kT7a+/qSkf88ZB0CnnM5Tb/dIeljSpbb32/5rSV+R9Ge2n5Z0Tes0gD52yre4RsRNC1y0IXkWAB3EK+iAIig7UARlB4qg7EARnn1NTJcOZk9I+uVpXHW5pJc7PE67+nk2qb/n6+fZpP6e73Rne3tErDjZBV0t++myPR4RY72e42T6eTapv+fr59mk/p4vYzZuxgNFUHagiH4t+6ZeD/Am+nk2qb/n6+fZpP6er/FsfXmfHUC+fl3ZASSj7EARfVV229fafsr2Htt9ta+d7Qtt/9T2Lts7bd/S65nmsz1ge4ftH/R6lvlsL7O92faTtnfbvrLXM/2O7c+3fqZP2L7Hdk8/CK9Tm7z2TdltD0i6Q9J1ktZJusn2ut5OdYJpSV+IiHWSrpD09302nyTdIml3r4dYwNcl/TAi/lDSe9Qnc9peJemzksYi4jJJA5Ju7O1UndnktW/KLul9kvZExN6ImJJ0r2Y3tuwLEXEgIra3vj6i2V/W3I9qbcD2akkflnRnr2eZz/bZkj4o6ZuSFBFTEfFKb6c6waCkJbYHJY1KerGXw3Rqk9d+KvsqSfvmnN6vPirTXLbXSFovaWtvJznB1yR9SdLxXg9yEmslTUj6Vutuxp228z4fu4GIeEHSVyU9L+mApMMR8VBvpzqp097kdSH9VPbfC7bPkPR9SZ+LiFd7PY8k2f6IpJciYluvZ1nAoKTLJX0jItZLOqo++ayB1n3fGzT7B+kCSUtt39zbqd7cqTZ5XUg/lf0FSRfOOb26dV7fsD2k2aLfHRH39XqeOa6SdL3t5zR79+dq29/t7Ugn2C9pf0T87pbQZs2Wvx9cI+nZiJiIiGOS7pP0gR7PdDKNN3ntp7I/KukS22ttD2v2QZL7ezzT/7Jtzd7n3B0Rt/d6nrki4ssRsToi1mj2+/aTiOib1SkiDkraZ/vS1lkbJO3q4UhzPS/pCtujrZ/xBvXJg4fzNN7k9ZR70HVLREzb/rSkBzX7iOhdEbGzx2PNdZWkT0h63PZjrfP+ISIe6OFMv08+I+nu1h/yvZI+1eN5JEkRsdX2ZknbNfuMyw71+GWzrU1e/1TSctv7Jd2m2U1dv9fa8PWXkj72lnN5uSxQQz/djAfQQZQdKIKyA0VQdqAIyg4UQdmBIig7UMT/AFnwB68hOPfIAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.imshow(wo_fed_test_llk)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "l3R9T0t8Xpr6",
        "outputId": "d3dae507-b541-4b81-b99c-03791c96a816"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fe4d7fa0d50>"
            ]
          },
          "metadata": {},
          "execution_count": 22
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD6CAYAAABnLjEDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd4xl133fv7/X2/Q+O7OFZVlFUuSKkkzGoWnJlmVHkmHZsGIbTEKAgREDcpTAlBLAsFMACQhUgAg2CMs24zgqji1QkWQrtBolRKK07HW5vc3MTp8383o5+WPe7pvf99wp3F2+XfL+PsBi57x777nnnvLu+/3Or4hzDoZhvPWJXOkGGIbRGWyxG0ZIsMVuGCHBFrthhARb7IYREmyxG0ZIuKTFLiLvE5FDInJERD5+uRplGMblRy52n11EogBeA/BeAGcA/BTAR5xzL292Taw74+LDvRfKzTp910QC2lKlc6J0PNrU5RqdLwFtp0vAt6VrHJW3vR6A469RbjdfE9UfSMVvuPA11I5mmk5o6jpiRa9K1FP0AQ9Jjar0+t9/+EiV7tulK6mW4vqCOD1II+Ad5I0RfcDtoDqEngNpHkTA1XW7JabrjK5Qnf260ho/F+DPP24nFSM07jz3AMDF9UUb+7u2soh6sRBwFRAL+nCH3A3giHPuGACIyJcAfBDApos9PtyLff/toQvlwlxGNzpb966JnNGzsdGlBynSW1VlOavPbyb8yRijvhAa5CZ1ZjNJg17U50fLft/WuvU1jZxut9CCaPboZ08fS3h18pcML978rbovIqt6eAef9tu5eKsuNzL6JukpvbqrPfq56j0Nr87sSbrv/VOqfPrFUVWW0bIqN5eSXp38xeZo0UToC8Xldf+lZvRzuNtWvVuUF9KqnOjT7er5ZlaV4x85p8ozLw97dfJi53FGTZ+QOaG/MBx/uQIo7dJ1ZE+0+/v4X37av6DFpfyM3wXg9IbymdZnChF5SEQOisjBej7g1WIYRkd4wxV0zrlHnHMHnHMHYt2Z7S8wDOMN4VJ+xp8FMLmhPNH6bFNcOYrqq90XyiMv6Z9iq7t9mSc7rc9xEf39FC/on3zFEf2zqBTwyyo9q89JrOh7pBf1T9OVvbqbus7o42u7/N9avUd1nbW0bnctxz+p9c9OFzAy0bKuM7nMZd1/mXP6556L+j/jJ76r6yiM6htHq/r3cyOu66j0+w1Nz+pr5r47rsojx/XxlX36JZA5FyB6lUh8oP4sjuh25M7o83NTFVWeX+vy7tE3pdu1NpFT5URBj/vq32txpK/gt5tFr5Xr9BjlTuvjmVl9j8yMbjcAnP1Z3V/Dz7TFt9PFzXVwl/Jm/ymA60Vkn4gkAPwmgK9dQn2GYbyBXPSb3TlXF5HfA/AtrOua/9w599Jla5lhGJeVS/kZD+fcNwF88zK1xTCMN5BLWuwXxQbBoTRIcmy3L29USrTNVaEtrYQ+XmcdYNA+JYnY1W7aBovpE4pj+p6JVd3u0rDf7kZSnxMl0Sta1desTerjySW/4dUe/Rk/O289u4ge3sSa307eNy8N0T4vyfnxvD7f26cHUOnRz17P0rOO6+MNqqPSF7CVmeV26eNs17A2wcYRWidSHAmQr8nuo0E7gLUM7bPz1n7Sbze3s5nQQnwtS32V0nUUx/xtyFoP67ECt9U9zFzWMEKCLXbDCAm22A0jJNhiN4yQ0FEFnROgscFWvZnYWukCADWyfWAlVWpBH69nSHkRUGe5X5+Tmtd1Fki5Iw19fqWHbeP9e/B9y/26HCN7erbhF99NACCbI75Hg47HCnR+wFc7K5UqA7od2dN0vG/rNgC+IrBa2H6c1fnd/mfcx6zwrOeo/8hkv0kznf0dAKDWRQpQ0o3xuBdHqa/OBClV6QM6xevvaX281L/9+7i50dBpC12dvdkNIyTYYjeMkGCL3TBCQmeNamIOboPDf3NOC0WNlB9QgGXKCMmy0tjaMKeZ9f2tY0v6sT3Zl+Q5NgqJki96LefLfxz0oUGBJaokwzfp2dcSAd/Dnj/71sYUyWVdZmMWwHcw4XuwzMmybq3Lf/ZiSd+nQjqSWEmfX5nQnRVd8B2iWH5mnUc9S3EOKls7HjVzvlKksbr1uAorPaj7C5MBhjo0/SK7tJt3fU770BfIqarnqD9/V/brGy/e2J7P9Sc3nxP2ZjeMkGCL3TBCgi12wwgJHZXZpSaIzbQdEliulYYvb2TPbP19xMEBohWWq/zrOYgf6wFSC/qEKsUJS5AsHC357Y7RvnC5n/fu9XHe4+W9/6B2eserVK5pGbL3qF/B6oSeAtKkgBiLJKOT01DQs8dpn72eZmcmfb6b0jJ6pObXyf0VX6M6RMu63BfxVYpZt+pv9rMOhJ8ttUCBPnZR/LiZAEcucuopT2tPrdSinp+503pCF8b8dnrjeqTdOdy36rrNDxmG8VbCFrthhARb7IYREjq7z97UMdv7D2kZMlLzm9NzcmtBlR33K920xxsQCIEDIiZXdDlW0gLi0n4d+KD3sBYIywP+vnBqSbe70kNxy7ndvaQXyAckMaCv5tSybicHV8id1Hu6kTJnSgBSs3oDO3+N3vflvuHAFOUA2+2e4xToUijI4oxuN48Z6w0AP2hJhPwV4hRQJEHBH3Mn9eZ+cVTHgAeA3sO6XYVRPWZ9r2mBOF7U8yJW9tudoiAkjnRIXaf0NT1HtEND37O+48XZXxxS5e6fnrnwd7RQ5dMvYG92wwgJttgNIyTYYjeMkNDZffYGEN+QYovlPw48CABNCnjIcinLsaVhzsro18nycp3qjJa17NagIIAups+vegkfgHKfllPjlPkqUqfAmexXzn7QAJIrvH9N7egie/BeLY8n5nw9QLVPy51+YEctt3ad1nJracgftHIf7Q3TGFVz+oO1SX3P5JJXJTKz9OzUX2xvUSF7gHRGT3W2gwD8YI8cx6A0pMeUdQ3Oy7zp40i9w3qXwoTeh0/k/Zx/POdd9wb9w9zm7297sxtGSLDFbhghwRa7YYQEW+yGERI6qqBrZhzyt7cVPHnKwJHsKfAlmBnWxg+NjFZocJACDGjNi2OvFwDVHq24YoVdfJUyw16nDRVqWa00KY0HGP7ESVkTo+AKK+T8MUwGMFFfmVY4ovsiWiQDl2EygOnV7ZSmr+xhpWhhr36WxIJWtq3t0hfIO1a8OhcO6YiRtUEKTrGi6xy99Zwqz+d9g5flvB4TqZASMKvv4cr6eK1LX796s298ko9QkBIKorFyAykvaa5hye9fThszvn9OlU/36TTDqXkyujnpv4+Lu/Q4z94zcOHv2rnNl7S92Q0jJNhiN4yQYIvdMEJCR2X2SFHQ/VxbdvKCAYx38SXomaagf002qmFjFO3IUcv57UjPUQBEcmDg4ADRipbF+l/V8mF+3u/GCAVbYEMITnpQGCdjilV4sDNIkgwyGifJmWZVy9+pWT+yQWVAy7KpeQoCQeqIxCoZgaz61j9di1qmLPdr2Te1oI8vz4/q42SABAD95LzERjXVbv0cPKbZaf0g0vDlaw66wQElOTBEuV/rL5JLQUY1upL5Jf2s4y/o/oyWdTvjBV8fVBjX9x082NabxIp+gMrz2JvdMEKCLXbDCAnbLnYR+XMRmRWRFzd81i8ij4vI4db/fVvVYRjGlWcnMvtfAvjvAP7Hhs8+DuDbzrlPisjHW+WHt6vIRXXSgQRt0Tb9GBAoU/AJceyssHUdQckMOQAiJ4ngrXmuozisu40DZwK+8wwHZOBEg+zIwQESAaCu1RGoZSNbHucEGtHugOQL1E5OpsByLDvbBDkasZzPTj0cyJH7InDM2BmJzmF7AXYsihf0BUFzjT+rkjNNz3E9SCqhIvwxBPxEpBwIszCsJx8HC5GGH3CSn70y3B54d+QSHGGcc08AWKSPPwjg0dbfjwL40Hb1GIZxZblYmX3EOXc+uewMgJHNThSRh0TkoIgcbBR9CznDMDrDJSvonHMOgT/mLhx/xDl3wDl3IJrxzSANw+gMF7vPfk5Expxz0yIyBmB2R1dFgGZiewf/jXA8AE+2pcCCdQq+UA9IPBipU2CDeQpSMEh7+dRLnGSQZTcAkPrWe7aenoC+dln+Xr8vVUKd44SCQKxQQE9uUwAs+ybyW7czKOBIgDsCXcPBQOiEgOu303GwnqBJ/ZtY1Rc0Uv57jnVIXsJP3pqn7gzqC/8c0t2wbqHEdiUBupsMjXtkmw5vcbFv9q8BeKD19wMAHrvIegzD6BA72Xr7IoAfAbhBRM6IyIMAPgngvSJyGMB7WmXDMK5itv0Z75z7yCaHfv4yt8UwjDeQziaJcDppX7RKyRmKvuzB9sacHICTKVS79CNFywGJB2lToNKz9d5yM0EyPtl2rwUk30tyckOSU3lvnmXjILmVA04yzSjZ9Jd1O4P2r2MFfU60pJ8lXmS9gL6ebRYAIEq+BRxsMzNLyS1ydM+8/5xZuoaDlXJSUFJfeDbjsaI/9dNk099MbN0XPIbZGd8uvThEvga1rffdWcYPsnWPlrXyIF5sKyyCZPwL99r0iGEYbylssRtGSLDFbhghobMyOwC3Ic5XaVh/1xTHfHmD/bortD9a7qNEE7QHHkRdu46rxBUAwKHauM4G7avzfiwAlAa2tsVmf3e+h9QDZOGAxIEbYRuDwhglVDzrx10rjOspwP7/FdJ5sMxe6fXbEaHYgnXaf2Y5tpbbft+4niV/hCol2aB7CO27ewkeev2+XBEaSGpGuZeShg7oOop5fyKU+8nuI0ty/4w+n3U7xdGguHZUrG3QNWwxRezNbhghwRa7YYQEW+yGERJssRtGSOiogq6vdw0f/sAPL5SfXZ5Qxz+19++8a377uX+pynt7dYrPVExbNhxbHlDlTNyPLNGf0lY142lt0dJLViCvrmoP3gg5oMwUdFIEAEjGtIbo18ae1veI6nuUKb3nX515l1fnSFprEper2ltmpaK1VGtlrfXL130F0n17XlblfWmdxOB/Hrlble/ZdVyVp4p+wMmZgg4cen2PDocQIS3SLw8+r8px1q4BWG5oj8mnVveocqmh+28spcf0R7P7VPnQ277q3ePzy5Oq/LXp23W7olqr+qv9x1T5i0fv8ur8+7seUeXvFPeq8pmqnq9/9sw9qnz9hO9jFm/oZXt499CFvyv/eXOnGHuzG0ZIsMVuGCHBFrthhARZDzTTGTLXj7kbPvuvLpTzq9q6pdkMCAIxreXQRpeWm1IDJVWunNN1uqQfBTC2qGWeRoaC/JGzQjOtjyfntOwbkG8AdUpmyIkII2SsEhnXzxF5zY/q00iTg8mqrqO0h4xmqDtzr/gNZcOQWq/uX+ExCTD2Ydj56K57D6nyk89dp8oJSpBYnyKrp4A6a326nelB0oEUKanlPD37kJ8wo0nOMWmaW8nvaN1M9gPaImbqyBAYl6C5RWVOQJmY56gmXpWo79P9FT/a1t2c+tNPo3z2dOAg2ZvdMEKCLXbDCAm22A0jJHR0n92txVD54eCF8uhhLXct3BQQBEJvq6Oe0U3O/kDLtoUxDhbp1xnVohhy05QckpIDNJL6nhzkMig5QCOp931jFEiQgz7UTuvnSC36wlqNxHihQB7dx3U7PccXR9kkATRSun8Wbtbtzk6TjMndGSBTxkv6miNHblBlvbMMVPq0903abyYSFLgjVmaHKL23H6f+7T2i9+4Xb/QjembOcUIMLaPnKDhF4TFtfzEUECm92q07jJ2wOHhF12nSSS34diILt+q2973WrmSak1NuvNemRwzDeEthi90wQoItdsMICR2V2SXXQPxn2nbS07dp2WP/+DnvmleOj6tyNKVlr7UVvX+aGdb249cPznt1vnR2TJ8zeUaVz65pe2/OAXju8KAq77mJIhAAODXTr8rvvOaEKk8XtTxYruuhmOxa9uqconYtFXT/La1qm4TFaW0b7/aQsgJAfY2FcC0jFu/S/d1c1v3togEyIu0l75nQ43p2Xke8+O1bfqLKXzzk25jnS3oU2Ga8VtLCMCcXOX27lunHRv15MTOv+7e5pu+5fEDL05Pj2ub/9Ck9LwAg0a33868d1vedzut5MHNY902k6kdjyd6i73t2tJ1EufaC2cYbRuixxW4YIcEWu2GEhI7axqfGJ92ef/2xC+WeI1q2Kw/43z19r2kZsp7W59QyWkap0h55LReQeGKR92x1mfenV67RclPXGd2m/G6W6oEkJTqIVigBAQWtrOb0c2VnfJ/u0qCWr7127tXtTKzpe2anfHtwF9PtWNmn5f44JZGoU8BPTrgIAH2HtJ36zLu1gUD/K7r/Kr0BETuJzIy+pkb71/nJrW0huk7pvlq8yZeFe47rPl++Vtc59oTO/Hjq/VrG73vNT+jA/VUc2TqpaOacbkMi7++zz96l9RNj328bo/z40J9hpThltvGGEWZssRtGSLDFbhghwRa7YYSEjhrVxHI1DLy7bYByyy9Pq+O/0PuSd80fvvDPVPmWEW3AwsEeb+rRRgv9nLIVwNOLOrDg7f1nVfmJs9fSFbqOuYZWDr1r/KR3D4aDWD6/tEuVr89q5U8i4ivovn9CB31AThvJ3Dd6VJW/+qoOmBiN+h471SWtkLv/jhdUmQM5Pn9OGzldM7Dg1XlyqU+V9/Zpo5qZ+7WBy8PXf0vfo6THBwBeymtDqHpTj8GdOd2OJPXfk3N7VXk05iu+bu/T8+Dx09qBp/vX9dx6oEcbA/3j7I1enWMZPa43ZnVffGPqFlWeXdPKt/JZStEDYP/b9Hw7tL8duLX8qc2VnfZmN4yQYIvdMELCtotdRCZF5Lsi8rKIvCQiH2193i8ij4vI4db/fdvVZRjGlWNboxoRGQMw5px7WkS6ADwF4EMA/gWARefcJ0Xk4wD6nHMPb1VXanzS7XmobVTTdUrfuzTo2wLEyHcjQcYqnB21NMQpLv12RMm2hAM0MIs360pyJKIHiNeoUUbVeJGyd9Z0eW2CjGqm/HHhTKVs0JJYJUMeCozAhj2AH3hjdULLfEkKGtEkI5zisN/BXWd0pcvX6WdLLVCQyy4yNDnnPzs/K2fadUIBPOu6Dn6O/F7/PcfBK3g+8nMVKQtxtLK9gVpxVNeZPUNGNfPaMIf7GwDm7tD3nfx2e0IfPPh55PNnLs6oxjk37Zx7uvX3KoBXAOwC8EEAj7ZOexTrXwCGYVylvC6ZXUT2Ang7gCcBjDjnzqvTZwCMbHLNQyJyUEQONooBcXsMw+gIO17sIpID8LcAft85pxJpuXVZIPA3jHPuEefcAefcgWjGj4VuGEZn2NE+u4jEsb7Q/9o5dz774jkRGXPOTbfkej8DHeGiQK27LfdUu8ippYuv8OXhMslRLLMz9UzAd5DTdazt0u3gvIJensHI1jIn4CeO4HM4wQPL49Uev05qticjlvv0c7BeIDPn11np1tc0kvqcci9dQ6+HoGCbnnNSLyXZWNSVVHtI11D22xklZyXf4Ymcm4r6uNfOAKmW9T11ejc1yHmpOE4OVAHmFnXqCw72wfOiGNE6k3hAAElO3FEabk+2IBn/PDvRxguALwB4xTn36Q2HvgbggdbfDwB4bLu6DMO4cuzkzX4PgN8B8IKIPNv67D8A+CSAr4jIgwBOAviNN6aJhmFcDrZd7M65HyLwRw8A4Ocvb3MMw3ij6KhtPBwgjfb3Bu+FekkE4ScliJBJc4RksQbF/o+WAuok4SW5RAkcSM5qJvVxlt24PgAQimMgvE9MugavjqCtf6qD+yZGe/m8t89BGAFfluV2ZWb1CYVRCl4RkNSSSSxz4g4+Tv0dMCtrpKuJUiKJ+Jo+HtOuCN54lAf9Ds6epn1zGhMeo1iB5kHAKzFobmykTvM1Ra4GQTI4y/3JxfaikPrme/1mLmsYIcEWu2GEBFvshhESOpskwmm7dLZzLw/48gknvmMZM7WytUzZ8N2BEc/rMstzLHslF2iPvLD9PrsvT+tytLR1HZwIEvCDZ7L+go8naI82vuYHRIxUyR58SE+JZnTrZ29WAgJ60phU+nRnxKgO3tvn5wL8wIxswsV75DUa9zTZ4yeW/PdclPqCg3Hy3GP7i9Syrwco0/uUE3rysyYowGe05Ne5cr0eo40+D7KFr4u92Q0jJNhiN4yQYIvdMEKCLXbDCAmdNaqBVn41EmSUENAaVuJtBwem2JEhCRmGJJf1RZzFgxU5tQBrivSCvkmJst2Qv4On0AuyWeTMNewkxEo9aVK54XcGOyNxf0W8a7ZWhAWcsq1hCTt78LwAgGoXK7bIEIqMU2JkdMNj1kj5feGi3HBqJynLmgk9aNwmwDcc47kWWWFjIVJuBmTLYUehWL79sEFjfOFemx4xDOMthS12wwgJttgNIyR01qimqYMKpFa0kUdt0ZdPspS9kw0dKpTNk+X+hp+s08vimp3R7WBnguKY9g5JLenz66kAYyCSp3NTFEiQnGkaS2TA4du/eEEsM6tavuOgi0nq39gqWSgB6C7o/o00Mt45G+GglkGOGok89ScldGAHnQr1RVDABjYQYj2A5zDF+gwyVonU/LmWOafbXRij4JtLutLu41vrO4CAQKNVfQ3rWWIl3YbUNFljAVjdo7PHRhZX24VGwMQ5f96mRwzDeEthi90wQoItdsMICR2V2bO9JfzMh567UD62OqCO/+rQEe+av3rpblUeH9CJ8gaSekN15vCEKgcJUtVbteyVp33ynm4d8jrW1N+JK/fqOleO+slw4rt0HZMDy945G7k5u6TK3/vxrf5Jg/pZV4p6+LqG11R5uUgKi1P+pvjuAzqZ4Z70aVX+0XPXq3LfhO7/TML3Wjkxo/vjzmtfU+Wza1rmHE3pTCDX5PxkkYtVrUt4bkYnmOzL6jpyCW1wceiwPv8v3vOn3j3+z/IdqvyDGZ3g88RePV8//HM/UuWvHrrNq/MXrj2kyss1bRAwV9JjcvQZPX+79/vh16tFPc5HutqJMCuf3zyaiL3ZDSMk2GI3jJBgi90wQsK2iR0vJ+nRSXft77QTOybIBr0UlCTw1NZJFzk4JNtQ1wO2jdNznFBAH48X9T052WH3adqPHfa/M9keYDsb/0pAUggmtaTbxXu05X5KoLioz8+d9OW/4oTuoOVrtR6AbRLSZGOwdJ2v9uk6Tf23O7Ll8aX9+nh61u8rTlLJdgicVIP7JjutL1i80W83J3/kucSJHfP0XJmAdrPbxOoe/cHgC5vviwNAvOAfn7lby+W7v9HWo/z41UeQL0xdXGJHwzDeGthiN4yQYIvdMEJCZ/3ZRSc8TOa1DFTP+PbKERJZ2Gc4TrJZtKy/vyqcmBBATG/JIjOn992rZG/v2S+TTJ9c8e8RrelzOLEE+3xvTJ4BAN2nOZskUMts/d28XTul6teZXND75Cnyn+Yx4ucI0kWkqM7CuN7v9wJ6ahMDb3wA31zCiylACUY4GCTbnHtZOxBgk0/FxIquNLm0tc8EABSGaS6Rfqia02PafVw/fD3rt5P7J1Iwf3bDMDZgi90wQoItdsMICbbYDSMkdFRBF8nVkb137kK5dq8+Hg3I4jp9tleV+3dph5JohAIJUh3FQsqrM7+gnRFiA9ppoj5Hx4e0VmVxmepkLSKAwXHtMLK4lFXl5qpW7mRGVlV5uuoPjRzRBjCsjOy595wqn5zSDimZI/1eneURMtQZ04Y31aJuZ/ycNuho7va1aUv36nbduU87wjz1yj5VvufWV1X5pblRr87l6W5VjmS1EjDXpZ2EOEzHsakuVZYsRaQEkP05nSoon9f9vUjz4rbbjqnyodlhr84ajSM7Wc1Oa6eguXdqZWZy1ldaV67TbT+eHWkfe2TzJW1vdsMICbbYDSMkbLvYRSQlIj8RkedE5CUR+ePW5/tE5EkROSIiXxaRzR1pDcO44uxEZq8AuN85tyYicQA/FJG/B/AxAJ9xzn1JRP4UwIMA/mSrigaSBfzO3p9cKH9/UQdGuLHrHF+CH6W1fDeS1rLtcEqXn5zdo8qplB9codyv5fqhXh0MoNGtZfRUTBtTVHq03DW3qOVJAEjQNe+5QculUbISuSN3SpX/4sS7vTrLt2ndwvKCDnwwFtfPOjiiZdD5qtZ/AMD+/VOq3JXQ8uBTr+1V5Z5bdWCJe8a03AoA3z9znSpfm5tX5YV9Wn/xti4dQOPObt0XADC/Wz/rMnk4pSnd70srY6q8uqzP/6N3fs27x3RN6ziey+tAEj9p7FXlD408o8qPVv0xS8f0mNzRe0aV86NaD/CNF3XQksbNvk6E7bM2yvAuIPnFebZ9s7t1zq+GeOufA3A/gP/d+vxRAB/ari7DMK4cO5LZRSQqIs8CmAXwOICjAJadc+dfX2cA7Nrk2odE5KCIHFxbDEi8bRhGR9jRYnfONZxzdwCYAHA3gBt3egPn3CPOuQPOuQO5/vj2FxiG8YbwuoNXiMgfAigBeBjAqHOuLiLvBvBHzrlf3Ora5OSkm/jov71Q7j6qj6/ugUd6butke+z8UemjYBa9AQkHlilJIG3Kxiiwf4F+s+S02IXyQIAjDG3jcvIKdtSo6W1gL0El4Ads2K6cosATsoOxLo5Q4gOKHcLBLArj/rOnFvQ5eR23EZlpsoUY0+cnl/w641o14yX/4CAl3BccMGPlGv8913Vq64SenPCTnaySS37/1nL6nMJu3Y7UOQrcwYFVSn6d83fq8siP2+e88PjnsLZ4+uKCV4jIkIj0tv5OA3gvgFcAfBfAh1unPQDgse3qMgzjyrETbfwYgEdFJIr1L4evOOe+LiIvA/iSiPwXAM8A+MIb2E7DMC6RbRe7c+55AG8P+PwY1uV3wzDeBHQ2eEUEaGTaMktpSNv91nO+jXkjr89hGZJt4Rtk2lPP+QErY2u6zgaZutc4aCXV0Yxr6afaHaAXoOgU9Swl+KtQgsp+slEv+GIXy6GxEgVs4L6h56qnfKmNg0Kw7MsJFDmJZSPtPzvrTRz5L7B+okF7w/W0/+wc8ILv26RxF9r44eCR1T5/XpSK+mGbpE/mdtVyFDil6rebn7WZpGAgVCcnqAxKGuqi1OcSKKJ7mLmsYYQEW+yGERJssRtGSOiozJ5OV3Db205cKB+b0P7Vtzq/BVgAABTgSURBVPUtete80KcT8mXJbzka1YJseZ6EJBb2AJSu0ddITMtRfI8m+SSv0l40pnyf+eqt2qa5l+ztWdewJ6ft8197mRJUAkCXFkSrBd2uCB0v1LQAmDjrGzW567Wdf09OP/vCMW0vHhsmW+2A/i2s6E3wwQkdg2BlTfdXH/mWD2X9ZBaLJUpmsaptytkHIhbRY7rcq/0CfvVnfurdg+3pzyzrawrH9dy67R3aUOSZl7UfBwDsv177HlSbekyWx/VzrKR1f7sx3++ebU3mb2v3Z/0H/unnsTe7YYQEW+yGERJssRtGSLDFbhghoaNZXJN7Jt3Ywx+9UE7NaGVFecw3qskd8wPubYSNEIqjZACT8o0n4sv6ou2cbfI3aK+V/mf19dVuX0nFhjbJZTY00ecXxyjj6mn/e7g8uPVYcZ3pc/qe7BgD+M4cNR0jIsBJSJdXbvTHbOAZ3ZCFt+tni5b0cXb6YWMWwHeeqVK8kGovGauQIVTPy1qZWZjw+yJ3kpxWdulzuk7q8/Okj4sEeHCnFnSd+dt0h6ZOaWugHDnjBM2t/I16Po59r92fL37rs1hbuEhHGMMw3hrYYjeMkGCL3TBCQmeTRMQbyE20gyDecZcONNgb94PrfbNXB+CbGNIpP8/Oa8OHt0/qyBJLFfLsAHBiZkCVo7dqQXTpnBYIr9lHyRd6tDHQSL8O7AgA+3vnVPn5OW2wsbysgy7eMKHvsXYLRWcAsDpNSR7W9PDd/fbDqvzkq9fo69f84XYJLXNzAMq1mpYpp04MqvKea2a9OmfHteB/bZ9OmHH8nO7/f3rdIVUuNXyh/ZUFnTiiRMk/+nN67vSn9Zi+1tBGSr2T2tAHAPbfo8fsyUO6/8o36HvcMqgDaR5b8pNw1Opav3PfhBb8n0jo4Jzzw/rZk3P+mCX7dTsKY21jn4Cuu4C92Q0jJNhiN4yQYIvdMEJCR/fZM0OT7oZfawec7H9FG/nP35bmS9B9Uu8pctBEF9FbiuU+2gPv8rcc0/N6DzZe1OVIjQIm7tFyU/+rut2lwe2T4cTXtGzcoEASpX5yWikEjMs2YxWldqentGwXqQYEB8loIa88rHUFkaquM17Q47F8re8E1Peqlpfnb9d6kxQFZmzEOQiH/5zpeX3fZmLr/uO+6Dqh+2J5v6/L6T6pN/zzu3VfdJ3We+SlIZKvV/z+rWd0O1f26nb2HNfXpOZ0G4LGbO4urRMZfaLtQPajI1/ASmna9tkNI8zYYjeMkGCL3TBCQkf32ZsJYG13u9xIaRm9NOzLatLQMk4zocURTujAgf0bGb/OajfbZusyJwMoDek6V8paTl2b9EWkhN5aRi2nn4NtzKs6ZoGXyALwbd/ZFpvzNqZGtWyXnfX9BFg/sbKXpgR1X2JVHy8NBwTGbGp5mJNsuKh+ELZTTyz776DSsNaLcPDN4ig1lHU5/dquoazNBVrn6HHlYJy1nJbhOblF91F/OZVoPrIPBMA6Jr0mpOHP3+Io1bm7bRfSPLW5L4m92Q0jJNhiN4yQYIvdMEJCR2V2F9HB/ZucVT5gGzlCslmdbH85IYF3z4DDLOsm8pQAke7J/tacVDCpzfUDz+GEfdxuTmrAyREBXz7mdnqJFCjBQLTiy+yFET0FWC8Qp/3+eob88gP6N1ohn+xRvUeeWCG7BGoW+9AHtYvL/NqKUgINivOIRiIgsUdDX1Ma0eeM/1A/x9pu3XeclAPwnyWyTULPBLlZsB3J+n0o6Uau/fAuuvl6sDe7YYQEW+yGERJssRtGSOiozC4OiJbbMgUnInRBvrgkWsVXKaFfTMsoLI9HGgF74FQH34OTBDapl2IUt7/guzEjTrJXLafbEV8juYv2Tqs9frtZ/mP7b453llze2uYf8MeAEztKk+3W9fGgeHFFSqKRe1V3KI9RfJVsIwJk3xiFOmD7Cm8ekGycndENX9vr38NRHYmVrf0uuL+DbPopvycilPwxM03+DAu6nbWMPw9ilIAye7Y9ISNVXy9z4dimRwzDeEthi90wQsKOF7uIREXkGRH5equ8T0SeFJEjIvJlEdnez9MwjCvG63mzfxTAKxvKnwLwGefcdQCWADx4ORtmGMblZUcKOhGZAPDLAP4rgI+JiAC4H8A/b53yKIA/AvAnW1bkoAwoWElVIWeQC9dsbDApx2pZNojZ3lCnQc40ySIbjgS0YwtiftJRJOjZymz8w34bpFDyjEbgK8eErolEdKXRKveN3xn+OaRIJKMaNqKJ1HwFEl9THiDjHjbUyW6tZAXgGd5wXzBe3zS2mScAojS36jQf2WCFx4jvuX4f+oCGgJ+DlajRgP7lOgMVgwHs9M3+WQB/gHaXDwBYds6df7wzAHYFXWgYxtXBtotdRH4FwKxz7qmLuYGIPCQiB0XkYKMQ8Ao0DKMj7ORn/D0APiAi7weQAtAN4HMAekUk1nq7TwA4G3Sxc+4RAI8AQGrXZOcC3hmGodh2sTvnPgHgEwAgIvcB+PfOud8Skb8B8GEAXwLwAIDHdnLDjUYasdLWCf8AIFLfWmZkAxkONBFk9MHUKc4l6xK4zliZjg/6clWN5H6+huXSKBmNBDmYcDAFTmrJshzfsxn3+zde0GPACSiZFAX2CEo86Bm8jGxt8OLpZfxcIUgu6XbWU1vL+XwPnkfJRT/IAxshcVBQnq8cZCNe2t6opqzzY3jOSuwcFl/1A07yODaTG55li+G7lH32h7GurDuCdRn+C5dQl2EYbzCvy1zWOfc9AN9r/X0MwN2Xv0mGYbwRmAWdYYSEzgaviDtUJtreHLN7SI7K+lELVjI6aOJGRxoAqOs4gqgNBG3Saqo9+rFZRoxQAMrSbl1nPUNBCyZ9IbNZ1OckZ7TygANidP8Tndjx3NGAiIg5EkTzNHwkry3Tc6TP+d/tNd29qOzWDRN6DhdludTf8C6N6vs0qN2la8hBZ4WCWE74si/fNzFHgRqHWEgnG45e3f/pd+mkjAAwu0tvrLsoKxd0O2s091b2B7Q7oZ81u2tVX9NL0SsoAGU9HRBA8iZdR/5oexAbL2z+/rY3u2GEBFvshhESbLEbRkjoqMwOANiwz54+oh3lSrv85qRIRmf5Ol7ggA1aNmsE+OI5DkZBCRs4OGGUZEoOthB/2U9IGV/TZU7gwMyc0Buw2TNBwf71Z0lKkMgBMjjIZazk79nm9+k6G3O6w3pf1eev7tXletaXU3sO63asTXLQh61l36A6OYAIB5aINPQJbKeepSARy6u+A0RiXreT7RrYdr4woeXxrmP+u7O4i5KYHO9W5cSafo4UBa8ISkxaPqYVLZnZ9sMGBSi5cGzTI4ZhvKWwxW4YIcEWu2GEhM4GnKwJklNtmbr7hJZPmglfTu06pWUQDkbI+9Uss5QG/e8zlsW6T2gBL79Hd0uCZPRomWXh7QM5xkk2Y/m6+5C+Z27K37/27OWpzPqMjbIcAEQrvszObZ+7XbcjO61tDBoJ3tv3ZcoU6RKalMjR85GnRAjJJb9OjjGQnabAjOQTz34C2WndF0tnfT0LB39k//WuM7qOJvVFet4fs2ZCPzvXGaH5m5vSioHSoK90qvaS7uZcW+kkdQs4aRihxxa7YYQEW+yGERJssRtGSOi8Uc0G7RgbDDSSvqKLs6F6x0l/UenV318cmALwFVmFEa3w4CyklV42fNg+gAMHU/AUSPSoHOyiNODXye1mhR0Hs6j06A+yU35ERM5yEiVfpMIYGauQ/icocw23s07ONpyZd7vnAPyAIo0kZdChMfCUgJTtJVYMivJAgTBp7lRzNLdIWcyBTAHfEKc4ru+RO6mPlwdoQgc0k+d8M7VhjAIUpuexN7thhARb7IYREmyxG0ZI6KxRTROIltoyRXKFggjO+d89HOTPS6YQ3bpcTwcE2ac60otbZxzgAAIscwYlNWA5NDNHBkTUzmaMAk0EGGhUu/Q53BcNduChBBBBASfTi2QoEtdTgvUXnjwekGU0sarbXhjXDfMy2LIDT8Cz10lG98aAY1fQmNTTHDzSu4UXvJT1AqklfVPWwwQFnOT5yMFXOHFHhJN2BCSAcBF22Lm8SSIMw3iTY4vdMEKCLXbDCAkdldmjVaDneFse6z6m00E141m+BP3PLOpzsnrjXWpajqrn9CZkJcCRILmgBbroGm0u01dgtKI3wVOzvOHty0yREgWp7NWbto7k59QiBcgo+nviPS/qiBiNLr3RG12lTd35ZX3PgPRbsm9StzOlo2zkTuh7cn8nVzhgIhAvsAMOBXs8q9uRXtD9Gyv4zx5bChCyN1Dao9sRK+h2Jp47ri94z36vju5XVlQ5tVvXmX1+SpWlMabK6RO6vwGgNqSNDCI1PQ9yZ/XcSx2d1RVE/Pdxf3ZUf/Dj59t/u837yd7shhESbLEbRkiwxW4YIaGjMnsjDqxOtr9f6kktzxTG/T3beEEH7mf7ZE7cyPb2bIMOAAly/m/GtezbdUrL28vX6Ztks7oNtaz/ncn77Gzjz3u65QFdR7QSEHxzRFfCgTqKw7o/U4v9qpydokgJAEpUJ+/lL92iAyRmZ6hvrvUzZ2YogcPytbrOni4tC6/u0ceTS/6zZygQZrlHX8M2+rGSrqMrfZ0qV3r8MVs4oOdajWwIXHSXKs/fqu/R2+Mn9mD7gJXr9fFEgYJv3qH1AGxnAgCrFMCz6x1vaxde/H/e+eexN7thhARb7IYREmyxG0ZI6Pg+e/fJtgySyOu90HjJd2SO1LVcml7goH/sx6y/vyoB/tbxIvkUn9JZIopjei+UbbVzJ/X5jYwvt1b69WeeDTQ9F/tC973m75eWh7TcmljWfSGU/SK5RPYET1HGBwC5G6/R7bhRy+i9z2s7h/K4lrf7DvuOAalpvTffjPXoe5LuoJ7SOpPctL/PzuSKekyKdU5Aqc+PVvX57O8OwPMdZ7+A1Jy2Y0gtaruQ5LLf7nicYjac4vgANL/P6XtE82Q7ASCxRycUiU4ttOurbd539mY3jJBgi90wQsKOfsaLyAkAqwAaAOrOuQMi0g/gywD2AjgB4Decc0tvTDMNw7hUXs+b/eecc3c45w60yh8H8G3n3PUAvt0qG4ZxlXIpCroPAriv9fejAL4H4OGtLmjEgbXx9vdLNxmFbDx2nuyMLsfKFASCAgnWU1tnXgH8wAaFiTQd56wyWtuTnteGKCv7fAVdcpkUQKSA48CN3M7FmwMyw3qZVPR92YAjvkr9ecM+r85ar1aO1SjYx+Kd2jCHAzgEPXsvtHFPuZ8MhqpkyEMBPYsNf1rGKAsPOx9tmymIsuGUhvxIprkpfc7aGAXjJCcWik/pBcgAgHpKf1YaooAYyzR/s7o/OagJABSH9TXNwQ0K0IWg7L/r7PTN7gD8XxF5SkQean024pybbv09A2Bkh3UZhnEF2Omb/V7n3FkRGQbwuIioPRznnBMJMkwFWl8ODwFAvKsv6BTDMDrAjt7szrmzrf9nAXwVwN0AzonIGAC0/p/d5NpHnHMHnHMHomnfX90wjM6w7ZtdRLIAIs651dbfvwDgPwH4GoAHAHyy9f9j29U1NryIT/zuFy+UZ+ra2KI3WuRL8PTaHlXeldQBAo4Uh1X5np7DqjwUy3t1fnH2Xar8e6PfVuUvLb1Tle/MntDHp+9W5Y9PPu7d45nSXlX+xdyLqrzQ1AEbGk5/796c8Dc2vlPUdf7DwttU+fbuM6r8jalbVfmWAX0cAJZJDr09oQ1iKuRpdKaog1v8+tDLXp3fX9SBIX53+KAqf2/lJlX+zPgPVPnRvB5zAFhp6HaOxHSgiV1x3V/LDf1i+Zu5A6r86/2+gVGCIpF+ZVpfszenDYweHHxClb+ev8Ors4uyRPwSzYNDNT1/Hznzs6r86mkKVAHg/Tc9pcpPvKNtGFX9mHf6BXbyM34EwFdlPdNEDMD/cs79g4j8FMBXRORBACcB/MYO6jIM4wqx7WJ3zh0DcHvA5wsAfv6NaJRhGJcfs6AzjJAgbocB5i/LzUTmsP6TfxDAfMdufPFYOy8fb4Y2Am/+du5xzg0FXdDRxX7hpiIHN1jiXbVYOy8fb4Y2Am/tdtrPeMMICbbYDSMkXKnF/sgVuu/rxdp5+XgztBF4C7fzisjshmF0HvsZbxghwRa7YYSEji52EXmfiBwSkSMiclUFuxCRPxeRWRF5ccNn/SLyuIgcbv1/Rd32RGRSRL4rIi+LyEsi8tGrtJ0pEfmJiDzXaucftz7fJyJPtsb/yyLiZ93sfFujIvKMiHz9Km7jCRF5QUSeFZGDrc9e95h3bLGLSBTA5wH8EoCbAXxERG7u1P13wF8CeB99drVF46kD+HfOuZsBvAvAv2n14dXWzgqA+51ztwO4A8D7RORdAD4F4DPOuesALAF48Aq28TwfBfDKhvLV2EbgckSKcs515B+AdwP41obyJwB8olP332Eb9wJ4cUP5EICx1t9jAA5d6TZSex8D8N6ruZ0AMgCeBvBOrFt8xYLmwxVq20RrodwP4OtYDyZ9VbWx1Y4TAAbps9c95p38Gb8LwOkN5TOtz65mrtpoPCKyF8DbATyJq7CdrZ/Hz2I9zsHjAI4CWHbOnfcjvRrG/7MA/gDA+SBhA7j62ghcpkhRHU0S8WbGuc2j8XQaEckB+FsAv++cy7fcjwFcPe10zjUA3CEivVgPeHLjFW6SQkR+BcCsc+4pEbnvSrdnGy46UtRGOvlmPwtgckN5ovXZ1cyOovF0EhGJY32h/7Vz7u9aH1917TyPc24ZwHex/pO4V0TOv2Cu9PjfA+ADrTDpX8L6T/nP4epqI4BLixS1kU4u9p8CuL6l7UwA+E2sR7u5mjkfjQfYYTSeNxJZf4V/AcArzrlPbzh0tbVzqPVGh4iksa5XeAXri/7DrdOuaDudc59wzk045/ZifS5+xzn3W7iK2gisR4oSka7zf2M9UtSLuJgx77Ci4f0AXsO6/PYfr7Tig9r2RQDTAGpYl9UexLoM920AhwH8I4D+K9zGe7Euvz0P4NnWv/dfhe28DcAzrXa+COAPW59fA+AnAI4A+BsAySs97q123Qfg61djG1vtea7176Xz6+ZixtzMZQ0jJJgFnWGEBFvshhESbLEbRkiwxW4YIcEWu2GEBFvshhESbLEbRkj4/0gPoMqYOH8vAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "wo_fed_test_llk = np.load(dir_path + \"/No_Fed_testing.npy\")"
      ],
      "metadata": {
        "id": "dahndzkJaZmE"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "wo_fed_test_llk.mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PuJnqe4Mas03",
        "outputId": "8ed79ef3-37fa-4dcb-b343-e9c3f09a2d0f"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-2.2543094777432553"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "l9NyB1WOVn4y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(init_model.kernel._beta.detach().numpy())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "s8KZdTntSxll",
        "outputId": "3b815d1d-a9c2-47e6-fcf5-34c8bd360405"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7ff806892950>]"
            ]
          },
          "metadata": {},
          "execution_count": 89
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3jb5Xk38O8tyWfJZ1tJnNjOQQoECImbUiilnEoHLYMeKIV1He1bxlhpoYPCVXp1fVm3dRttYYXSbmlpS1teDk1pm7J0lEGA0hFakxMkwbIl4jhOIst2bEvyUdL9/iH9bKFItmz9pN9B9+e6fKHDL9JjEt9+9Dz3c9/EzBBCCGF8Fq0HIIQQQh0S0IUQwiQkoAshhElIQBdCCJOQgC6EECZh0+qNGxsbub29Xau3F0IIQ3rttdcGmbkp3XOaBfT29nZ0dnZq9fZCCGFIRNSb6TlZchFCCJOQgC6EECYhAV0IIUxCAroQQpiEBHQhhDAJCehCCGESEtCFEMIkJKALITTzincIXSeCWg/DNCSgCyE0c/uTe/Gvvz2k9TBMQ7OTokKI4haaiuD46CQsRFoPxTRkhi6E0MRbgTAAoH9kAqGpiMajMQcJ6EIITXgDodnb3X5ZR1eDBHQhhCZ8bwvooXmuFNmSgC6E0IQ3EMaq+gqU2SzwyAxdFbIpKoTQhDcQgqvZgeryEngGZIauBpmhCyEKLhZjvDUYxprGKridDllDV4kEdCFEwfWPTGAqEsPaZjtcTjuOj05ibHJG62EZngR0IUTB+QbjKYtrGqvgbnYAkI1RNUhAF0IUnDexZr622Q63UwnosuySqwUDOhGVE9EfiWgfER0gon9Ic82niChARHsTXzfmZ7hCCDPwDYZQXW5DQ1UpVtZVoKLECo/M0HOWTZbLFIBLmDlERCUAXiai3zLzrpTrnmDmz6k/RCGE2XgHwljbbAcRgQhY12xH94DM0HO14Ayd45RfnSWJL87rqIQQpuYbDGFNo332vstpl1x0FWS1hk5EViLaC2AAwLPM/Gqayz5KRPuJaBsRrcrwOjcRUScRdQYCgRyGLYQwqtBUBP6xKaxpqpp9zO10wD82hdEJyXTJRVYBnZmjzLwJwEoA5xDRmSmX/AZAOzNvBPAsgEcyvM5WZt7CzFuamppyGbcQwqCUI/9rm+Zm6G5n/LZsjOZmUVkuzDwCYCeAy1MeH2LmqcTdHwB4hzrDE0KYjS9RZXFt0gzdlUhdlI3R3GST5dJERLWJ2xUALgPwZso1y5PuXgVAKtaLnDy9/xi+8qvXtR6GyANvIASrhdDaUDn7WEttBSpLrbKOnqNsZujLAewkov0A/oT4GvrTRPQ1Iroqcc2tiZTGfQBuBfCp/AxXFItf7u7Hz3Ydwcj4tNZDESrzBcJYVVeBMpt19jGLheCSTJecLZi2yMz7AWxO8/hXk27fDeBudYcmipkn8YO9p28EF69v1ng0Qk3eQOht6+cKl9OBFz2SLJELOSkqdCc8FUHf8AQAYE/vSY1HI9QUVYpyJa2fK9xOOwLBKflUlgMJ6EJ3epJKqe4+MqLhSITajiWKcq3JMEMHZGM0FxLQhe4oG2PvWdeIvX0jiMbkHJtZeNOkLCrcswFd1tGXSgK60B2PP4gymwVXbVqB0FRENspMRElZTLfksqKmHPYym+Si50ACutAdjz+Edc12vLO9HgCwu1eWXczCGwihpqIEDVWlpzxHRFjXbJcllxxIQBe64/EH4XY60N5QifqqUuw+IhujZuELxDdEiSjt826npC7mQgK60JWxyRkcH52E2+kAEWHzqloJ6CaSKWVR4XY6MBiaxnBYMl2WQgK60BWla41S26OjrQ6+QFhS2UwgODmDgeBU2vVzhUs2RnMiAV3oivKDrGQ8bG6tBQDskfRFw5vdEG2cb4YuRbpyIQFd6IrHH0RlqRUttRUAgLNX1sJCkGUXE/ANxj99rWvOPENfVl0OR5lNNkaXSAK60JVufwiuZjsslvimWVWZDactq5aAbgK+QDhelKs+c0AnIml2kQMJ6EJXuhIZLsk62mqx94gcMDI6byCE1vpKlNrmDztupwPdAzJDXwoJ6EI3ToanEQhOnRrQW+sQno7KrM3gfIEw1jRmnp0rXE4HhsPTGAxNLXiteDsJ6EI3lIDtcr5906yjtQ6ArKMbWTTG8A3GG0MvRNkYlV/giycBXeiGJ/Exe/2yt8/Q25QDRnJi1LCOjUxgOhLLaoaufELrlo3RRZOALnSj2x+Eo8yGZdXlb3uciNDRWos9MkM3rB6lKFcWM/RmRxmqy20yQ18CCehCN7pOBOFy2tMeC9/cWgffYBgn5QShIc3loC88Qyei+MaozNAXLZueouVE9Eci2pdoM/cPaa4pI6IniKiHiF4lovZ8DFaYFzPD4w+estyiUNbR9/TJLN2IfImiXPVpinKl43I64BkIglkymxYjmxn6FIBLmPlsAJsAXE5E56Zc8xkAJ5l5HYD7AfybusMUZjcYmsbJ8ZnZ7u+pzl5VA6uFZB3doOI1XDIX5UrldtoxMj6DgGS6LMqCAZ3jlM8+JYmv1F+bVwN4JHF7G4BLKdu/OSEwd9Q70wy9stSG05Y5JNPFoOJVFhdeP1fIxujSZLWGTkRWItoLYADAs8z8asolLQD6AICZIwBGATSkeZ2biKiTiDoDAWkGK+ZkSllM1tFah33SwchwlKJc81VZTOWS1MUlySqgM3OUmTcBWAngHCI6cylvxsxbmXkLM29pampayksIk+ryh1BbWYIme1nGazraahGejqLrhPyQG8l8XYoyabKXobayRGq6LNKislyYeQTATgCXpzzVD2AVABCRDUANgCE1BiiKQ3fiyP98K3VywMiY5usjmgkRwd3skKqLi5RNlksTEdUmblcAuAzAmymXbQdwQ+L2NQCeZ9meFlli5kQNl/l/4FvrK9EgHYwMZ64oV+Wi/pxSpEtCSfZsWVyzHMAjRGRF/BfAk8z8NBF9DUAnM28H8DCAnxJRD4BhANflbcTCdPxjUwhORrDemX5DVEFE2NxaJ7XRDcY3mF1RrlRupwNjkxEMBKfgTDlsJtJbMKAz834Am9M8/tWk25MAPqbu0ESxmNsQnT+gA/F19P855MdweDrrnGahLe9AGGsXsX6uSN4YlYCeHTkpKjSX2qVoPrMHjGTZxRCiMcZbQ4tLWVS4Z9vRycZotiSgC815/EE02suymnFvXJk4YCQB3RD6T8aLci1lhq78m5CN0exJQBea6/KHFtwQVVSW2rBhebWcGDUIb6Lt3FJm6ADgapbuRYshAV1oipnRk6ZL0Xw6Wmux7+gIItFYHkcm1OAdWHzKYjKlSJdkumRHArrQVP/IBMLT0cUF9LY6jE9H0SUzN93zDYZRW5l9Ua5UbqcdwakIToxNqjwyc5KALjQ1tyGa/Qxu7oCRLLvonXcglFXJ3ExcsjG6KBLQhaaUH9RsUhYVK+sq0Ggvw55e2RjVO99geMnLLUBykS75NJYNCehCUx5/EMuqy1FTUZL1n1E6GEmmi76NTc4gEJxa8oYoANRXlaLRXiobo1mSgC405fEH4c5QMnc+HW11ODw0jiGpl61bSlGupaQsJnM1O2TJJUsS0IVmojFGz0AI7iz6TKaaO2Ak6+h65QvklrKocDvt6BmQTJdsSEAXmukbHsfkTGxRGS6KjStrYJMDRrrmDYRgsxDaGhZXlCuVy+lAaCqCY6OS6bIQCehCM7MZLktYcikvsWLDimoJ6DrmC4TRWl+JEmtuYWauBICsoy9EArrQTHfi0IlrCUsugNLBaFQOGOmUNxDKebkFmEtplUyXhUlAF5rpOhFES20FqsqyqeJ8qs2ttZiYieJN6WCkO9EY4/DQeM4bogBQW1mKJkeZbIxmQQK60IzHH8zYFDobUnlRv5SiXItpOzcft9MuM/QsSEAXmohEY/AFwvM2hV7IyroKNDnK5MSoDi2l7dx8XM0OdA+EEJMG4fOSgC40cXhoHNPR2IJdiuYjB4z0y6tSyqLC7XRgfDqK/pEJVV7PrLLpKbqKiHYS0UEiOkBEt6W55iIiGiWivYmvr6Z7LSEU3YtoajGfjtY69A6NY1AOGOmKNxBGXQ5FuVLNbowOyLLLfLKZoUcA3MHMGwCcC+AWItqQ5rrfM/OmxNfXVB2lMJ0ufxBEuX8k72iTA0Z65FMpw0UhRbqys2BAZ+bjzLw7cTsI4BCAlnwPTJhbtz+EtvpKVJRac3qds1rkgJEeeQNL6yOaSU1FCZzVZZKLvoBFraETUTviDaNfTfP0eUS0j4h+S0RnqDA2YWJd/uCiKixmUl5ixRkrqrFbKi/qxtjkDAZDuRXlSkdpdiEyyzqgE5EdwC8AfIGZx1Ke3g2gjZnPBvAggF9leI2biKiTiDoDgcBSxywMbjoSw+HBcE4bosk2t9Zh/1E5YKQXSlGuXOqgp+NqdqBHMl3mlVVAJ6ISxIP5o8z8VOrzzDzGzKHE7R0ASoioMc11W5l5CzNvaWpqynHowqjeGgwjEuOcUhaTdbTVyQEjHZltO7fEE8CZuJ12TMxEcfSkZLpkkk2WCwF4GMAhZr4vwzXLEteBiM5JvO6QmgMV5tGlUoaLoqO1FgBkHV0nfIPxolyt9bkV5UqlTABkHT2zbGbo5wP4JIBLktISP0BENxPRzYlrrgHwBhHtA/AAgOtYal2KDLr9QVgtpNopwpbaCjQ7ymQdXSe8A2G0NuRelCvVuuZEpoukLma0YBENZn4ZAC1wzXcAfEetQQlz8/iDaG+oRJkttwwXRfyAUR1ekxm6LvgGQ1jTqO5yCxDPdFlWXS4bo/OQk6Ki4Dz+UE41XNLpaKtF3/AEAkE5YKSlaIxxeHAca5vV3RBVuJx2WXKZhwR0UVCTM1H0DoXhalY5oCcKdck6uraOnoyXdFibhxk6EN936RkIISqZLmlJQBcF1TMQQozV2xBVnNlSgxKrHDDS2mzKooqHipK5nXZMRWLoGx7Py+sbnQR0UVBKLY71y9SdwcU7GNVgT6+UANCS2lUWU7mke9G8JKCLgvL4QyixEtoa1J/BdbTWYn//CGbkgJFmlKJcdSoV5UqldLdSul2Jt5OALgrKcyKINY121VPagPg6+uRMDIeOpx5kFoXiDYTyNjsHAEd5CVbUlMsMPQMJ6KKgPAPBJTWFzoZSeVHy0bXjC4Tztn6ucDkdUnUxAwnoomDCUxH0DU/ArfKRcMWKmnI4q6WDkVZGJ+JFufI5QwfiG6PegGS6pCMBXRRMT2LdM18zdOWAkWS6aMOncpeiTFxOB6YjMRyRTJdTSEAXBeNRuYZLOh2tdTh6cgIDwcm8vYdIT0lZVLMOejpuyXTJSAK6KBiPP4gym0X1ok3JOtoShbokfbHgvIF4Ua5Vefz7BZIyXSSgn0ICuigYjz+Edc12WC3zlgbKyRkr4geM9siyS8H5AvkpypWqqsyGltoK2RhNQwK6KBiPP5jX5RZA6WBUI+voGsh3ymIyt9R0SUsCuiiIsckZHB+dzHtAB+Lr6PuPjmI6IgeMCiUSjaF3aDzvKYsKt9MBXyAsXapSSEAXBaGUPHWr1KVoPh1ttZiKyAGjQjp6ciJelKtAM3SX04HpaAy9kunyNhLQRUEUIsNFIZUXC883qNRwKdQMXTZG05GALgrC4w+istSKltqKvL/XitoKLKsulwNGBeQdUBpDF2aGvq5ZaUcnG6PJJKCLguj2h+BqtsOSxwyXZB1ttVICoIB8gyHUV5XmrShXqspSG1bVV8jGaIpsmkSvIqKdRHSQiA4Q0W1priEieoCIeohoPxF15Ge4wqi6CpDhkqyjtQ79IxMYGJMDRoXgDYSxprEwyy0Kd7ND2tGlyGaGHgFwBzNvAHAugFuIaEPKNVcAcCW+bgLwPVVHKQztZHgageBUQQP6ZllHLyhfAVMWFS6nA77BkJRLTrJgQGfm48y8O3E7COAQgJaUy64G8BOO2wWgloiWqz5aYUjKx2JXATJcFGe2VKPUapF19AIYHZ/BYGi6YCmLCrfTjpkoo3coXND31bNFraETUTuAzQBeTXmqBUBf0v2jODXog4huIqJOIuoMBAKLG6kwLE+iKJfajaHnU2az4oyWallHLwDvYH67FGUyV9NFll0UWQd0IrID+AWALzDzkhJ8mXkrM29h5i1NTU1LeQlhQN3+IBxlNiyrLi/o+3a01mF/vxwwyrd89xHNZG2THURSpCtZVgGdiEoQD+aPMvNTaS7pB7Aq6f7KxGNCoOtEEC6nHUSFyXBRdLTWYToSw0E5YJRX3kC8rWC+i3Klqii1orW+UjZGk2ST5UIAHgZwiJnvy3DZdgB/lch2ORfAKDMfV3GcwqCYGR5/sKDLLYq5youy7JJPvkAIrfX5L8qVjqvZITP0JNn8DZwP4JMALiGivYmvDxDRzUR0c+KaHQB8AHoAfB/AZ/MzXGE0g6FpnByfgau58AF9eU0FlteUS6ZLnsXbzhV2/Vzhdtrx1mBYltUSbAtdwMwvA5j3szIzM4Bb1BqUMA/laLYWM3QgvuyyRzJd8iYSjeHwUBiXnu7U5P3dTgciMcbhoXBB02L1Sk6KirzSImUx2ebWWvSPTMAvB4zy4ujJCcxEueAbogrl35Usu8RJQBd51eUPobayBE32Mk3ev6MtccBI1tHzwhvQJmVRsbbJDgtJ6qJCArrIq+7Ekf9CZ7gozlihHDCSgJ4Pheojmkl5iRVtDVVSdTFBArrIG2ZO1HDRZvYGxA8YndlSLSdG88QbCKGhqhS1lYUpypWOq1m6FykkoIu88Y9NITgZwXqNN6s6Wuvwuhwwyot4hos2s3OF2+nA4aFxTEWimo5DDySgi7yZ2xDVOKC3xQ8YHTg2quk4zMg3WPiiXKlcTjuiMcZbg1LTRQK6yJtCdimaz1wHI1l2UZNWRblSSU2XORLQRd54/EE02stQX6CmB5ksqynHCjlgpDqlKFehuhRlsqapClYLycYoJKCLPPL4Q5puiCbb3FaHPZK6qCpvoorm2mZt/47LbFa0NVTKxigkoIs8YebZlEU96Gitw7HRSZwYlQNGavENhuNFuery3yd2IdK9KE4CusiL/pEJhKejOgroiUJdsuyiGu9ACG0NVbBpUJQrldtpx+GhMCZnijvTRfu/CWFKcxui+lhyOWNFDUptFjkxqiLfYOH7iGbicjoQ47mDTsVKArrICyXjQOuURUWpzYKzWmpkhq6SSDSG3qGw5uvnCuWTYPdAca+jS0AXeeHxB7Gsuhw1FSVaD2VWR2st3ugfkwMoKuhTinLpZIa+urEKNgsV/caoBHSRFx5/EG6NSuZm0tFah+loDAeOSQejXPkSRbm0qoOeqtRmQXtjVdHnoktAF6qLxhg9AyG4dfJxXCGVF9UzV2VRHzN0IL5fU+y56BLQher6hscxORPTTYaLwlldjpbaCml4oQJfIKx5Ua5UrmYHeofHizrTJZueoj8kogEieiPD8xcR0WhSe7qvqj9MYSSzGS46W3IB4g0vZGM0d96A9jVcUrmdDjADPQPFu+ySzQz9xwAuX+Ca3zPzpsTX13IfljCy7sQPlEtnSy5AfB39+Ogkjo9OaD0UQ9NDlcVUSopsMWe6LBjQmfklAMMFGIswia4TQbTUVqCqbMGWtQU3t44uyy5LNTI+jaHwtO5m6O2NVSixUlFvjKq1hn4eEe0jot8S0RmZLiKim4iok4g6A4GASm8t9MbjD2rWFHohG5ZXo8wmHYxy4U0c3tHbDL3EasHqxuLuXqRGQN8NoI2ZzwbwIIBfZbqQmbcy8xZm3tLU1KTCWwu9iURj8AXCmjWFXogcMMqd3lIWk7mcDpmh54KZx5g5lLi9A0AJETXmPDJhSIeHxjEdjWnepWg+HW11OCAHjJbMG9BPUa5U7mYH+k6OY2K6OP9ucw7oRLSMEh2AieicxGsO5fq6wpi6ddLUYj4drbWYjsbwRr8cMFoKX0A/RblSuZ32os50ySZt8TEArwBYT0RHiegzRHQzEd2cuOQaAG8Q0T4ADwC4jpk5f0MWetblD4IIutswS6Z0MNojyy5LEk9Z1Nf6ucI1272oONfRF0xDYObrF3j+OwC+o9qIhKF1+0Noq69ERalV66Fk1Jw4YCTr6Is3E43hyPA43n/GMq2HklZ7QyVKrRZ4ijR1UX+fmYShefxB3VRYnE9HW52kLi5B3/A4ZqKs209gNqsFa5qqirbZhQR0oZrpSAxvDYZ1vSGq6GitxYmxSRwbkQNGi+HTacpisnimi8zQhcjJW4NhRGKs25TFZMo6uiy7LI4v0Rh6rcaNoefjbrbj6MkJhKciWg+l4CSgC9V0GSDDRXG6csBIll0WxTsQRqO9FDWV+qlzn0pZ8ivGTBcJ6EI13f4grBbS9cdxRanNgo0r5YDRYvkGQ1ij49k5MFfTpRiXXSSgC9V4/EG0N1SizKbfDJdkHa11OHBstKjLrS6WNxDG2mZ9/8Jua6hCqc0yWySumEhAF6rx+EO6reGSzubWOsxEGQeOjWo9FEM4GZ7GcHha9zN0q4WwtskuM3QhlmpyJoreoTBczcYJ6B1ttQCk8mK2ZjdEdT5DB5TuRTJDF2JJegZCiLExNkQVzY5yrKqXA0bZmq2yqPMZOhD/d9g/MoFQkWW6SEAXqlCaCqxfpv8f9mQdrXXYfeQkpFrFwnyBMEqtFqzUYVGuVEpzlWIrpSsBXajC4w+hxEpoa9D/x/FkHa118I9N4djopNZD0T1vIIS2hkpdFuVKpXxSLLZlF/3/zQhD8JwIYk2jHSUG+GFPNnvAqFeWXRbiC4QMkZIKAKvqK1FmsxTdxqixfvqEbnkGgrpsCr2Q05Y7UF4iHYwWMhONoXdoXLc1XFJZLYR1zXZ4iix1UQK6yFl4KoK+4Qm4ddgUeiElVgs2rqzF7iOS6TKfvuFxRGKsyy5FmbidDllDF2KxlCPWRpyhA/Fll4NywGheSoaLXuugp+Ny2nF8dBJjkzNaD6VgJKCLnHkMVMMlnY7WWsxEGW/0ywGjTPTcRzQTd3PxbYxKQBc58/iDKLNZ0FpfqfVQlqSjLb4x+ppsjGbkC4TRaC9DTYV+i3Klmst0KZ5ll2xa0P2QiAaI6I0MzxMRPUBEPUS0n4g61B+m0DOPP4R1zXZYLaT1UJak0V6G1vpK2Ridh9dAGS6KlXUVqCixwiMz9Lf5MYDL53n+CgCuxNdNAL6X+7CEkXT7g4ZdblF0tMY3RuWAUXq+wbCh1s8BwJLIdOkuonZ0CwZ0Zn4JwPA8l1wN4CcctwtALREtV2uAQt/GJmdwbHTS+AG9rQ6B4BSOnpQORqmUolxGSVlM5nIWV5EuNdbQWwD0Jd0/mnjsFER0ExF1ElFnIBBQ4a2F1pQNJ7cBuhTNRzoYZaYU5TLakgsQX0f3j01hdKI4Ml0KuinKzFuZeQszb2lqairkW4s8MXqGi+K0ZQ5UlFixR/LRT+EdUFIWjfdLW5loFMvGqBoBvR/AqqT7KxOPiSLg8QdRWWpFS63+CzbNx2aVDkaZeAdDiaJcxstiUso5F8vGqBoBfTuAv0pku5wLYJSZj6vwusIAuv0huJrtsBg0wyVZR1sdDh4bkwNGKbwDYbQ3Vhoyi6mltgKVpdaiWUfPJm3xMQCvAFhPREeJ6DNEdDMR3Zy4ZAcAH4AeAN8H8Nm8jVboTpcJMlwUHa11iMQY+4/KAaNkRugjmonFQnAVUaaLbaELmPn6BZ5nALeoNiJhGCfD0wgEp0wT0De3JjoYHTmJc1bXazwafZiJxnBkaBxXnLlM66EsmcvpwIue4kjCkJOiYsmUj7Eug2e4KBrtZWhrqJRSukmOKEW5DDpDB+Ibo4HgFEbGp7UeSt5JQBdLppQmNVJj6IXEOxjJASOFT2k7Z8CURYXLWTwboxLQxZJ1+4NwlNmwrLpc66GopqO1FoMhOWCk8BqwKFcq92xAN/86ugR0sWRdJ+JNLYiMl/2QyTva4mvnv94rmbdAvMqi0YpypVpRUw57ma0octEloIslYWZ4/EHDnxBNdfpyBy4/Yxm+/Vy3lNNFvA660Wq4pCJKdC+SJRch0hsMTePk+MzswQ2zICL8y0fOQn1VKW57fA8mpos7Jz3eR9T4v7TdzuJIXZSALpZE+fhqpg1RRV1VKb71sU3wBsL4+o5DWg9HM8Ph+C9to8/Qgfg6+mAoXmTMzCSgiyUxW8piqve4GnHje1bjp7t68dwhv9bD0YTSpciINVxSuYpkY1QCuliSLn8ItZUlaLKXaT2UvLnz8vU4bZkDd23bj0BwSuvhFJwZUhYVxVKkSwK6WBKlqYWZMlxSldmseOD6zQhNRXDXtn1Fl5vuDRi3KFeqZdXlcJTZTL8xKgFdLJpZM1zScTsduPuK07CzK4Cf7urVejgF5Q0YtyhXKiIqimYXEtDFovnHpjA2GcF6k9RwWcgN727Hhe4m/PN/HTL9R/ZkvkDIFOvnCrfTge4BmaEL8TZzG6LFEdCJCN/42EZUldlw2+N7MRUxfyrjTDSGI8Pjplg/V6xrtmM4PI3BkHn3QySgi0UzS5eixWh2lOPej27EweNj+NbvPFoPJ++Uolxmm6ED5s50kYAuFs3jD6LRXob6qlKth1JQ79vgxCfe1Yrv/96H/+0Z1Ho4eeUdMH4Nl1RKQO828caoBHSxaB5/qCg2RNP5ygc3YHVjFW5/cp+py7H6Bs2TsqhwVpfBUW6TGboQCmaeTVksRhWlVjxw3WYMhafw5V++btpURu9ACE2OMlSXG7coVyoiim+MFvsMnYguJ6IuIuohoi+lef5TRBQgor2JrxvVH6rQg/6RCYSno0Ub0AHgzJYa3H7Zeux4/QS2vXZU6+HkhW8wjDWN5pmdK9xOOzwDQdP+Is6mp6gVwEMArgCwAcD1RLQhzaVPMPOmxNcPVB6n0Im5DdHiXHJR3PTeNXjX6nrcs/0AeofCWg9Hdd5ACGubzfd37Gp2YGR8BgGTZrpkM0M/B0APM/uYeRrA4wCuzu+whF4pJ+2KJWUxE6uFcP/HN8FqIXzhib2IRGNaD0k1w+FpjIzPmHSGbu6N0WwCeguAvqT7RxOPpfooEe0nom1EtDoRnm0AAA79SURBVEqV0Qnd8fiDWFZdbuiGB2pZUVuBf/7wWdhzZAQPPt+j9XBUM1uUy4QzdOWTpVk3RtXaFP0NgHZm3gjgWQCPpLuIiG4iok4i6gwEiqMLt9l4/PEuRSLuz89egY9sbsGDz3fjtd5hrYejCqXt3FoDN4bOpMkR775k1pou2QT0fgDJM+6VicdmMfMQMyuLUj8A8I50L8TMW5l5CzNvaWpqWsp4hYaiMUbPQAhuE87ccvEPV5+BlroKfOGJvQhOzmg9nJz5AmGU2ixoqavQeiiqi2e62E1bwiGbgP4nAC4iWk1EpQCuA7A9+QIiWp509yoAxdsVwMT6hscxORMr6gyXdBzlJbj/2k3oPzmBe7Yf1Ho4OfMGQljdUGWKolzpuJwOePzmzHRZMKAzcwTA5wA8g3igfpKZDxDR14joqsRltxLRASLaB+BWAJ/K14CFdmYzXGTJ5RRb2uvxuYvX4Re7j+Lp/ce0Hk5OfIGwqQ4UpXI32zE2GcGACWvc27K5iJl3ANiR8thXk27fDeBudYcm9EapVOeSJZe0Pn+pCy91D+LLT72OjtY6rKg13pLFdCSG3uFxfOCs5QtfbFDJNV2c1eUaj0ZdclJUZK3rRBAttRWoKstqHlB0SqwW/PvHNyESY9zx5D7EYsb7SH9keBzRGJt6hj7Xjs58G6MS0EXWPP6gKZtCq6m9sQr3XHUGXvEN4fu/92k9nEUzUx/RTBrtpairLDHlxqjhAvrEdBQ/7+xD1ICzHyOLRGPwBcKmbQqtpo+9YyWuOHMZvvm7LrzRP6r1cBbFa6I+opnEuxc5TJmLbriA/uu9/bhz235c+eDLeMU7pPVwisbhoXFMR2NF06UoF0SEr3/4LNRXleK2x/dgYto4DTF8gRCaHWVwmKgoVzrx1MWQ6TJdDBfQP/7OVfjOX2zG2MQMrv/+Lvztz15D3/C41sMyve4ibGqRi7qqUtx37SZ4A2F8fYdxsni9gZCpZ+eKs1fWIjgVwfXf34V9fSNaD0c1hgvoRIQrN67Ac3dciDsuc+OFrgAuve9F3PvfbyI0FdF6eKbV5Q+CyNxrq2o7f10j/vqC1fjprl48d8iv9XAWxMzwBsKmamqRyUc6VuKeP98Ajz+Eqx/6A255dDfeGjR+kTXDBXRFeYkVn7/UhZ1fvAhXnrUc333Bi4u/+QJ+3tlnyOwCvev2h9BWX4mKUqvWQzGUL/7Zepy+vBp3bduPgM7znofD0xidmCmKX9pWC+FT56/Gi3dehFsvdWFn1wAuu+9FfOVXr2MgOKn18JbMsAFdsaymHPd9fBN++dl3o6W2Andu248PffcPpqmroRcef7DoKywuRZnNim9ftwmhqQju2rZP12u2ZuxStBBHeQluv8yNF+68CNef04rH/9iHi77xAu571mPIT/yGD+iKza11eOpv3437P342/GOT+Oj3XsGtj+3BsZEJrYdmeNORGN4aDMuG6BK5nQ58+QOnY2dXAD/d1av1cDJS+oiuK4IZeqpmRzn+8UNn4tnbL8TF65vxwHPduPDenfjxH97CdMQ4pZFNE9ABwGIhfHjzSuz84kW49ZJ1eObACVzyrRdw/7MeQ2Ua6M1bg2FEYiwpizn4q/PacNH6Jvzzfx3Sbf6zbzBelMuIJ1zVsrqxCg99ogO/uuV8uJx23PObg3jffS9i+75jhljKNVVAV1SW2nD7+9fjuTsuxPtOd+Lbz3Xjkm+9gF/v7df1R1696pIMl5wREe69ZiPsZTbc9vheTEX0N8HwBUJY02jeolyLsWlVLR7763Px40+/E5WlVtz62B5c9dDLeLl7UOuhzcuUAV2xsq4S3/mLDjz5N+ehwV6K2x7fi2v+4xVTpSkVQrc/CKuFimptNR+aHeW495qNOHh8DN/6nUfr4ZzCa/KiXItFRLhofTN23HoB7rv2bJwMz+AvH34Vn3z4Vd0eGDN1QFecs7oe2295D+796Eb0Do3j6of+gDue3Af/mHF3swvJ4w+ivaESZTbJcMnVpac78Yl3tWLrSz78oUc/s73pSAxHhsexxoRNLXJlsRA+0rESz91xIb7ywdPxev8ornzwZdz2+B4cGdLXGZiiCOhA/C/l2neuws4vXoibL1yL3+w7hou/+QIe2tmDyRn9ffzVE48/JDVcVPSVD27AmqYq3PHkPoyMT2s9HADAkeEwojHG2maZoWdSXmLFjReswUt3XYxbLl6LZw6cwKX3vYB7th/AkE6aThdNQFc4ykvwpStOw7O3vxcXuBrxjWe68L77XsSO14/L+noakzNR9A6F4WqWgK6WilIrHrhuM4bCU/jyL1/Xxb+72RouMkNfUHV5Ce78s9Pw4p0X45p3rMRPXjmMC7/xAh58rhvj09qmOhZdQFe0NVThPz+5Bf/vxnfBXmbDZx/djeu27sKBY/pcG9NKz0AIMZYNUbWd2VKDO96/HjteP4Ftrx3VejizfURlDT17zupy/MtHNuJ3f3chzl/XgG8968F7730BP9vVi5moNqmORRvQFe9e14inP/8e/NOHzkT3QAhXPvgy7n5qPwZ18hFKa90D8QyX9ctk5qa2v75gDc5dU497th9A75C2x859gXBRFOXKh3XNdvznJ7fgF3/7bqxurMRXfvUG3n//S5p86i/6gA4ANqsFf3luG3Z+8SL8n/NX4+edR3HxN17A1pe8hjpUkA8efwglVkJbg8zc1Ga1EO67dhOsFsIXntiLiEazOiCeslgMR/7z6R1tdXjyb87DwzdsQYmV8NlHd+ND3/1f7PIVripsVgGdiC4noi4i6iGiL6V5voyInkg8/yoRtas90EKoqSjB31+5Ac/83Xuxpb0OX9/xJt5//4v4n4N+XaxzasFzIog1jXaUWOV3fz6sqK3A1z9yFvYcGcGDz/doMoa5olzySztXRIRLT3fit7e9F/desxEDY5O4busufPpHf8Sh42N5f/8Fe4kRkRXAQwAuA3AUwJ+IaDszJ7c3/wyAk8y8joiuA/BvAD6ejwEXwtomO3706XPwQtcA/vHpg7jxJ524wNWIv79yg2HWkmMxRiTGiHH8v9HEVyQWQywGRGKx2ceiMUaUGZHo3O1oLH7/4PExbGmv1/rbMbUrN67A828O4MHnu3HaMgdaGyphIYKFCFZLPEhYiGAlAlE8Y8tKBAvFn7NaTr1tSVxrTfxZSjyfTjEV5SoUq4Vw7ZZVuOrsFXjkfw/joZ09+MADv8eHN7fg9svcWFlXmZf3pYVmnkR0HoB7mPnPEvfvBgBm/peka55JXPMKEdkAnADQxPO8+JYtW7izs1OFbyG/ZqIx/GxXL+5/1oPwdBSrG6ugxTm6GDNijLQBORLj2QCuBGQ1P1DcfcVp+JsL16r3guIUwckZfOCB36NvOH+1h5RAbyGCxTJ3GwBCUxH8+NPvxEXrm/P2/sVsdHwG332xBz/6w2GAgbsuX48bL1izpNcioteYeUu657Lp9tsCoC/p/lEA78p0DTNHiGgUQAOAt52cIKKbANwEAK2trVkNXmslVgs+ff5qfGhTC/7jJa9mzTQI8dnX7BcRrFaCzRL/obRZ4vetidsWS+IxiwVWC+L/JcBqtcQfT8zmbNa5Pz/3Z+a+Sq0WbFxZq8n3XEwc5SX49S3vwWu9JxFjBjMjGlN+kSe+EveZgejs4/FPY2lvMyfup7xOmusqS604d02D1v8bTKumsgR3X3E6bjivHfc/68Gq+vzM0Avavp2ZtwLYCsRn6IV871zVVZXi7itO13oYwsTqq0px2Qan1sMQebSitgLf+NjZeXv9bHa6+gGsSrq/MvFY2msSSy41AKThpxBCFFA2Af1PAFxEtJqISgFcB2B7yjXbAdyQuH0NgOfnWz8XQgihvgWXXBJr4p8D8AwAK4AfMvMBIvoagE5m3g7gYQA/JaIeAMOIB30hhBAFlNUaOjPvALAj5bGvJt2eBPAxdYcmhBBiMeS0iBBCmIQEdCGEMAkJ6EIIYRIS0IUQwiQWPPqftzcmCgDoXeIfb0TKKdQiIN9zcZDvuTjk8j23MXNTuic0C+i5IKLOTLUMzEq+5+Ig33NxyNf3LEsuQghhEhLQhRDCJIwa0LdqPQANyPdcHOR7Lg55+Z4NuYYuhBDiVEadoQshhEghAV0IIUzCcAF9oYbVZkNEq4hoJxEdJKIDRHSb1mMqBCKyEtEeInpa67EUChHVEtE2InqTiA4l2j+aFhH9XeLf9BtE9BgRlWs9pnwgoh8S0QARvZH0WD0RPUtE3Yn/1qnxXoYK6EkNq68AsAHA9US0QdtR5V0EwB3MvAHAuQBuKYLvGQBuA3BI60EU2LcB/DcznwbgbJj4+yeiFgC3AtjCzGciXprbrGW3fwzg8pTHvgTgOWZ2AXgucT9nhgroAM4B0MPMPmaeBvA4gKs1HlNeMfNxZt6duB1E/Ie8RdtR5RcRrQTwQQA/0HoshUJENQDei3hvATDzNDOPaDuqvLMBqEh0OasEcEzj8eQFM7+EeJ+IZFcDeCRx+xEAH1LjvYwW0NM1rDZ1cEtGRO0ANgN4VduR5N2/A7gLQEzrgRTQagABAD9KLDX9gIiqtB5UvjBzP4BvAjgC4DiAUWb+nbajKignMx9P3D4BQJVmskYL6EWLiOwAfgHgC8w8pvV48oWIrgQwwMyvaT2WArMB6ADwPWbeDCAMlT6G61FizfhqxH+RrQBQRUR/qe2otJFo16lK/rjRAno2DatNh4hKEA/mjzLzU1qPJ8/OB3AVER1GfEntEiL6mbZDKoijAI4ys/LpaxviAd6s3gfgLWYOMPMMgKcAvFvjMRWSn4iWA0DivwNqvKjRAno2DatNhYgI8XXVQ8x8n9bjyTdmvpuZVzJzO+J/v88zs+lnbsx8AkAfEa1PPHQpgIMaDinfjgA4l4gqE//GL4WJN4HT2A7ghsTtGwD8Wo0XzaqnqF5kalit8bDy7XwAnwTwOhHtTTz25USfV2EunwfwaGKy4gPwaY3HkzfM/CoRbQOwG/FMrj0waQkAInoMwEUAGonoKID/C+BfATxJRJ9BvIz4taq8lxz9F0IIczDakosQQogMJKALIYRJSEAXQgiTkIAuhBAmIQFdCCFMQgK6EEKYhAR0IYQwif8PRxbC41tL/mEAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "init_model.kernel._beta.detach().numpy()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d7r1iibES9CN",
        "outputId": "9f31fa59-2fd3-487a-af55-422792048a96"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([3.33075412e-01, 1.00000000e-05, 1.00000000e-05, 8.17423661e-03,\n",
              "       3.42687872e+00, 8.24890602e-01, 6.96250672e-03, 1.75350690e-03,\n",
              "       3.76325095e+00, 3.12796746e-01, 1.00000000e-05])"
            ]
          },
          "metadata": {},
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "yY3UPpw6UaTP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Results"
      ],
      "metadata": {
        "id": "Lq94DcJdkmPH"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "National Data Baseline"
      ],
      "metadata": {
        "id": "jIBbvKEuk-Vd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "init_model = MultivariateExponentialHawkes(T=T, mu=mu, alphas=alphas, beta=beta,\n",
        "                                            data_dim=data_dim, device=device)\n",
        "\n",
        "dir_path = rootpath + \"/Results/saved_models\"\n",
        "modelname  = \"No_Fed-constant-mu%.3f-%d_train-state-%d\" % (0.01, seed, 51)\n",
        "modelpath = dir_path + \"/%s.pth\" % modelname\n",
        "\n",
        "init_model.load_state_dict(torch.load(\"%s\" % modelpath))\n",
        "# init_model._mu = torch.nn.Parameter(torch.ones(11) * 1e-2, requires_grad=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyi1-ow7km5a",
        "outputId": "5e05e73d-132c-4b03-ac90-10382dad0218"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nation_test_llk = []\n",
        "\n",
        "with torch.no_grad():\n",
        "    for j in range(51):\n",
        "        test_set   = torch.FloatTensor(test_data[[j]])\n",
        "        test_loglik = init_model(test_set.to(\"cpu\"))\n",
        "        test_event_llk = test_loglik / test_event_num[j]\n",
        "        nation_test_llk.append(test_event_llk.item())"
      ],
      "metadata": {
        "id": "9tOg3jB5lQZL"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.mean(nation_test_llk)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I6LexDr_ldvw",
        "outputId": "6254a083-a481-4d4e-fd2a-2a2a202bf11e"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-1.4892034649421042"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model ensemble"
      ],
      "metadata": {
        "id": "IjDktr-ulnd8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "alphas = []\n",
        "betas = []\n",
        "\n",
        "dir_path = rootpath + \"/Results/saved_models\"\n",
        "\n",
        "for i in range(51):\n",
        "    modelname  = \"No_Fed-constant-mu%.3f-%d_train-state-%d\" % (0.01, seed, i)\n",
        "    modelpath = dir_path + \"/%s.pth\" % modelname\n",
        "    paras = torch.load(\"%s\" % modelpath)\n",
        "\n",
        "    alphas.append(paras[\"kernel._alphas\"].numpy())\n",
        "    betas.append(paras[\"kernel._beta\"].numpy())"
      ],
      "metadata": {
        "id": "UT4HL9unlpI6"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "alphas = np.stack(alphas, axis = 0).mean(0)\n",
        "beta = np.stack(betas, axis = 0).mean(0)\n",
        "mu = np.ones(11) * 0.01\n",
        "\n",
        "init_model = MultivariateExponentialHawkes(T=T, mu=mu, alphas=alphas, beta=beta,\n",
        "                                            data_dim=data_dim, device=device)"
      ],
      "metadata": {
        "id": "RxS032eZmmY4"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ens_test_llk = []\n",
        "\n",
        "with torch.no_grad():\n",
        "    for j in range(51):\n",
        "        test_set   = torch.FloatTensor(test_data[[j]])\n",
        "        test_loglik = init_model(test_set.to(\"cpu\"))\n",
        "        test_event_llk = test_loglik / test_event_num[j]\n",
        "        ens_test_llk.append(test_event_llk.item())"
      ],
      "metadata": {
        "id": "gsvpysyom80H"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "np.mean(ens_test_llk)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BDjWPxLvnBnA",
        "outputId": "c79f351e-020c-4a0a-918f-50359908fd8f"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-3.06227251545091"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "t99gyjhlnDDR"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}