ATRAF_ecg_4.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"gpuType":"T4","authorship_tag":"ABX9TyN0qOnP7AQhC0ILVNp9ZOtB"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"VRoxcbu_BBE_","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1689605795364,"user_tz":-120,"elapsed":22613,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}},"outputId":"ef732800-4252-484f-a046-ccb765d56330"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","source":["%cd '/content/drive/MyDrive/PhDResearch/ATRAF'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"I-kykokNBDWE","executionInfo":{"status":"ok","timestamp":1689605799942,"user_tz":-120,"elapsed":355,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}},"outputId":"2ea686cc-aa66-414d-f50d-4e34e8d329a9"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/drive/MyDrive/PhDResearch/ATRAF\n"]}]},{"cell_type":"markdown","source":["## Training Transformer"],"metadata":{"id":"WAp-TZK34Y7O"}},{"cell_type":"code","source":["import copy\n","import math\n","\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","from torch.autograd import Variable\n","from torch.nn.modules.normalization import LayerNorm\n","\n","# Autoencoder\n","class Encoder(nn.Module):\n","    def __init__(self, in_seq_len, out_seq_len, d_model, dropout=0.1):\n","        super().__init__()\n","        self.in_seq_len = in_seq_len\n","        self.out_seq_len = out_seq_len\n","        self.d_model = d_model\n","        self.input_dims = self.in_seq_len * self.d_model\n","        self.output_dims = self.out_seq_len * self.d_model\n","        self.dims_1 = (self.input_dims - self.output_dims) // 4 * 3\n","        self.dims_2 = (self.input_dims - self.output_dims) // 4 * 2\n","\n","        linear1 = nn.Linear(self.input_dims, self.dims_1)\n","        linear2 = nn.Linear(self.dims_1, self.dims_2)\n","        linear3 = nn.Linear(self.dims_2, self.output_dims)\n","\n","        self.flatten = nn.Flatten()\n","        self.linears = nn.ModuleList([linear1, linear2, linear3])\n","        self.dropout = dropout\n","        self.unflatten = nn.Unflatten(1, (self.out_seq_len, self.d_model))\n","\n","    def forward(self, x):\n","        x = self.flatten(x)\n","        for i, l in enumerate(self.linears):\n","            x = F.relu(l(x))\n","            x = nn.Dropout(p=self.dropout)(x)\n","        x = self.unflatten(x)\n","        return x\n","\n","\n","class Decoder(nn.Module):\n","    def __init__(self, in_seq_len, out_seq_len, d_model, dropout=0.1):\n","        super().__init__()\n","        self.in_seq_len = in_seq_len\n","        self.out_seq_len = out_seq_len\n","        self.d_model = d_model\n","        self.input_dims = self.in_seq_len * self.d_model\n","        self.output_dims =  self.out_seq_len * self.d_model\n","        self.dims_1 = (self.input_dims - self.output_dims) // 4 * 3\n","        self.dims_2 = (self.input_dims - self.output_dims) // 4 * 2\n","\n","        linear1 = nn.Linear(self.output_dims, self.dims_2)\n","        linear2 = nn.Linear(self.dims_2, self.dims_1)\n","        linear3 = nn.Linear(self.dims_1, self.input_dims)\n","\n","        self.linears = nn.ModuleList([linear1, linear2, linear3])\n","        self.dropout = dropout\n","        self.flatten = nn.Flatten()\n","        self.unflatten = nn.Unflatten(1, (self.in_seq_len, self.d_model))\n","\n","    def forward(self, x):\n","        x = self.flatten(x)\n","        for i, l in enumerate(self.linears):\n","            x = F.relu(l(x))\n","            x = nn.Dropout(p=self.dropout)(x)\n","        x = self.unflatten(x)\n","        return x\n","\n","\n","class Autoencoder(nn.Module):\n","    def __init__(self, encoder, decoder):\n","        super().__init__()\n","        self.encoder = encoder\n","        self.decoder = decoder\n","\n","    def forward(self, input):\n","        output = self.decoder(self.encoder(input))\n","        return output\n","\n","def make_autoencoder_model(in_seq_len, out_seq_len, d_model, dropout=0.1):\n","    encoder = Encoder(in_seq_len, out_seq_len, d_model, dropout)\n","    decoder = Decoder(in_seq_len, out_seq_len, d_model, dropout)\n","    model = Autoencoder(encoder, decoder)\n","    for p in model.parameters():\n","        if p.dim() > 1:\n","            nn.init.xavier_uniform_(p)\n","    return model\n","\n","\n"],"metadata":{"id":"oq9N8S6pS17l","executionInfo":{"status":"ok","timestamp":1689605805720,"user_tz":-120,"elapsed":4547,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["\"\"\" This code is a slightly modified version of The Annotated Transformer.\"\"\"\n","\n","class TransformerModel(nn.Module):\n","    def __init__(self, encoder, src_embed, linear):\n","        super().__init__()\n","        self.encoder = encoder\n","        self.src_embed = src_embed\n","        self.linear = linear\n","\n","    def forward(self, src, src_mask):\n","        output = F.relu(self.linear(\n","            self.encoder(self.src_embed(src), src_mask)))\n","        return output\n","\n","\n","def clones(module, N):\n","    \"\"\"\n","    Produce N identical layers\n","    \"\"\"\n","    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])\n","\n","\n","class TransformerEncoder(nn.Module):\n","    \"Core encoder is a stack of N layers\"\n","\n","    def __init__(self, layer, N):\n","        super().__init__()\n","        self.layers = clones(layer, N)\n","        self.norm = LayerNorm(layer.size)\n","\n","    def forward(self, x, mask):\n","        \"Pass the input (and mask) through each layer in turn\"\n","        for layer in self.layers:\n","            x = layer(x, mask)\n","        return self.norm(x)\n","\n","\n","class SublayerConnection(nn.Module):\n","    \"\"\"\n","    A residual connection followed by a layer norm.\n","    Note for code simplicity the norm is first as opposed to last.\n","    \"\"\"\n","\n","    def __init__(self, size, dropout):\n","        super().__init__()\n","        self.norm = LayerNorm(size)\n","        self.dropout = nn.Dropout(dropout)\n","\n","    def forward(self, x, sublayer):\n","        \"\"\"\n","        Apply residual connection to any sublayer with the same size.\n","        \"\"\"\n","        return x + self.dropout(sublayer(self.norm(x)))\n","\n","\n","class TransformerEncoderLayer(nn.Module):\n","    \"\"\"\n","    Encoder is made up of self-attn and feed foward\n","    \"\"\"\n","\n","    def __init__(self, size, self_attn, feed_forward, dropout):\n","        super().__init__()\n","        self.self_attn = self_attn\n","        self.feed_forward = feed_forward\n","        self.sublayer = clones(SublayerConnection(size, dropout), 2)\n","        self.size = size\n","\n","    def forward(self, x, mask):\n","        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n","        return self.sublayer[1](x, self.feed_forward)\n","\n","\n","def attention(query, key, value, mask=None, dropout=0.0):\n","    \"\"\"Compute the Scaled Dot-Product Attention\"\"\"\n","    d_k = query.size(-1)\n","    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n","    if mask is not None:\n","        scores = scores.masked_fill(mask == 0, -1e9)\n","    p_attn = F.softmax(scores, dim=-1)\n","    p_attn = F.dropout(p_attn, p=dropout)\n","    return torch.matmul(p_attn, value), p_attn\n","\n","\n","class MultiHeadAttention(nn.Module):\n","    def __init__(self, h, d_model, dropout=0.1):\n","        \"\"\"\n","        Take in model size and number of heads.\n","        \"\"\"\n","        super().__init__()\n","        assert d_model % h == 0\n","        self.d_k = d_model // h\n","        self.h = h\n","        self.p = dropout\n","        self.linears = clones(nn.Linear(d_model, d_model), 4)\n","        self.attn = None\n","\n","    def forward(self, query, key, value, mask=None):\n","        if mask is not None:\n","            # Same mask applied to all h heads.\n","            mask = mask.unsqueeze(1)\n","        nbatches = query.size(0)\n","        # 1) Do all the linear projections in batch from d_model => h x d_k\n","        query, key, value = [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n","                             for l, x in zip(self.linears, (query, key, value))]\n","        # 2) Apply attention on all the projected vectors in batch\n","        x, self.attn = attention(query, key, value, mask=mask, dropout=self.p)\n","        # 3) \"Concat\" using a view and apply a final linear\n","        x = x.transpose(1, 2).contiguous().view(\n","            nbatches, -1, self.h * self.d_k)\n","        return self.linears[-1](x)\n","\n","\n","class PositionwiseFeedForward(nn.Module):\n","    \"Implements FFN equation.\"\n","\n","    def __init__(self, d_model, d_ff, dropout=0.1):\n","        super().__init__()\n","        # Torch linears have a \"b\" by default.\n","        self.w_1 = nn.Linear(d_model, d_ff)\n","        self.w_2 = nn.Linear(d_ff, d_model)\n","        self.dropout = nn.Dropout(dropout)\n","\n","    def forward(self, x):\n","        return self.w_2(self.dropout(F.relu(self.w_1(x))))\n","\n","\n","class PositionalEncoding(nn.Module):\n","    \"\"\" Implement the PE function. \"\"\"\n","\n","    def __init__(self, d_model, dropout, max_len=5000):\n","        super().__init__()\n","        self.dropout = nn.Dropout(p=dropout)\n","\n","        pe = torch.zeros(max_len, d_model)\n","        position = torch.arange(0, max_len).unsqueeze(1)\n","        div_term = torch.exp(torch.arange(0, d_model, 2)\n","                             * (-math.log(10000.0) / d_model))\n","\n","        pe[:, 0::2] = torch.sin(div_term * position)\n","        pe[:, 1::2] = torch.cos(div_term * position)\n","        pe = pe.unsqueeze(0)\n","        self.register_buffer(\"pe\", pe)\n","\n","    def forward(self, x):\n","        x = x + Variable(self.pe[:, :x.size(1), :], requires_grad=False)\n","        return self.dropout(x)\n","\n","class FNetEncoderLayer(nn.Module):\n","    \"\"\"\n","    Encoder is made up of a Fourier Mixing Layer and a FF Layer\n","    \"\"\"\n","\n","    def __init__(self, size, fft, feed_forward, dropout):\n","        super().__init__()\n","        self.fft = fft\n","        self.feed_forward = feed_forward\n","        self.sublayer = clones(SublayerConnection(size, dropout), 2)\n","        self.size = size\n","\n","    def forward(self, x):\n","        x = self.sublayer[0](x, lambda x: self.fft(x, x, x))\n","        return self.sublayer[1](x, self.feed_forward)\n","\n","\n","class FourierFFTLayer(nn.Module):\n","    def __init__(self):\n","        super().__init__()\n","\n","    def forward(self, query, key, value):\n","        assert query is key\n","        assert key is value\n","\n","        x = query\n","        return torch.real(torch.fft.fft(torch.fft.fft(x, dim=-1), dim=-2))\n","\n","\n","class FNetHybridModel(nn.Module):\n","    def __init__(self, trans, src_embed, linear, fnet):\n","        super().__init__()\n","        self.trans = trans\n","        self.src_embed = src_embed\n","        self.linear = linear\n","        self.fnet = fnet\n","\n","    def forward(self, src, src_mask):\n","        output = F.relu(self.linear(self.fnet(\n","            self.trans(self.src_embed(src), src_mask))))\n","        return output\n","\n","\n","class FNetEncoder(nn.Module):\n","    \"Core Fnet is a stack of N layers\"\n","\n","    def __init__(self, layer, N):\n","        super().__init__()\n","        self.layers = clones(layer, N)\n","        self.norm = LayerNorm(layer.size)\n","\n","    def forward(self, x):\n","        \"Pass the input through each layer in turn\"\n","        for layer in self.layers:\n","            x = layer(x)\n","        return self.norm(x)\n","\n","\n","\n","class FNetEncoderLayer(nn.Module):\n","    \"\"\"\n","    Encoder is made up of a Fourier Mixing Layer and a FF Layer\n","    \"\"\"\n","\n","    def __init__(self, size, fft, feed_forward, dropout):\n","        super().__init__()\n","        self.fft = fft\n","        self.feed_forward = feed_forward\n","        self.sublayer = clones(SublayerConnection(size, dropout), 2)\n","        self.size = size\n","\n","    def forward(self, x):\n","        x = self.sublayer[0](x, lambda x: self.fft(x, x, x))\n","        return self.sublayer[1](x, self.feed_forward)\n","\n","\n","class FourierFFTLayer(nn.Module):\n","    def __init__(self):\n","        super().__init__()\n","\n","    def forward(self, query, key, value):\n","        assert query is key\n","        assert key is value\n","\n","        x = query\n","        return torch.real(torch.fft.fft(torch.fft.fft(x, dim=-1), dim=-2))\n","\n","def make_transformer_model(N, d_model, l_win, d_ff=0, h=8, dropout=0.1):\n","    if (d_ff == 0):\n","        d_ff = d_model * 4\n","    c = copy.deepcopy\n","    attn = MultiHeadAttention(h, d_model, dropout)\n","    ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n","    position = PositionalEncoding(d_model, dropout, l_win)\n","    final_linear = nn.Linear(d_model, d_model)\n","    model = TransformerModel(\n","        TransformerEncoder(TransformerEncoderLayer(\n","            d_model, c(attn), c(ff), dropout), N),\n","        nn.Sequential(position),\n","        final_linear\n","    )\n","\n","    for p in model.parameters():\n","        if p.dim() > 1:\n","            nn.init.xavier_uniform_(p)\n","    return model\n","\n","def create_fnet_hybrid(N, d_model, l_win, d_ff=0, h=8, dropout=0.1):\n","    if (d_ff == 0):\n","        d_ff = d_model * 4\n","    # if (d_ff == 0):\n","    # d_ff = d_model * 4\n","    c = copy.deepcopy\n","    attn = MultiHeadAttention(h, d_model, dropout)\n","    ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n","    position = PositionalEncoding(d_model, dropout, l_win)\n","    final_linear = nn.Linear(d_model, d_model)\n","    fft = FourierFFTLayer()\n","    model = FNetHybridModel(\n","        TransformerEncoder(TransformerEncoderLayer(\n","            d_model, c(attn), c(ff), dropout), 1),\n","        nn.Sequential(position),\n","        final_linear,\n","        FNetEncoder(FNetEncoderLayer(d_model, c(fft), c(ff), dropout), N - 1)\n","    )\n","\n","    for p in model.parameters():\n","        if p.dim() > 1:\n","            nn.init.xavier_uniform_(p)\n","    return model\n","\n","\n","\n"],"metadata":{"id":"JaXnnJUP40pD","executionInfo":{"status":"ok","timestamp":1689605805721,"user_tz":-120,"elapsed":9,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["import argparse\n","import json\n","import os\n","\n","\n","def get_config_from_json(json_file):\n","    \"\"\"\n","    Get the config from a json file\n","    :param: json_file\n","    :return: config(dictionary)\n","    \"\"\"\n","    # parse the configurations from the config json file provided\n","    with open(json_file, \"r\") as config_file:\n","        config_dict = json.load(config_file)\n","        config_file.close()\n","    return config_dict\n","\n","\n","def save_config(config):\n","    filename = config[\"result_dir\"] + config[\"auto_dataset\"] + \"/\" + \\\n","        \"training_config_l_win_{}_auto_dims_{}.json\".format(\n","            config[\"l_win\"], config[\"autoencoder_dims\"])\n","    with open(filename, 'w', encoding='utf-8') as f:\n","        json.dump(config, f, indent=4)\n","\n","def process_config(json_file):\n","    config = get_config_from_json(json_file)\n","\n","    # create directories to save experiment results and trained models\n","    if config[\"load_dir\"] == \"default\":\n","        save_dir = \"../experiments/{}/{}/{}\".format(\"Transformer\",\n","            config[\"experiment\"], config[\"auto_dataset\"])\n","    else:\n","        save_dir = config[\"load_dir\"]\n","    config[\"summary_dir\"] = os.path.join(save_dir, \"summary/\")\n","    config[\"result_dir\"] = os.path.join(save_dir, \"result/\")\n","    config[\"checkpoint_dir\"] = os.path.join(\n","        save_dir, \"checkpoints/\")\n","    return config\n","\n","\n","def create_dirs(*dirs):\n","    \"\"\"\n","    dirs - a list of directories to create if these directories are not found\n","    :param dirs:\n","    :return exit_code: 0:success -1:failed\n","    \"\"\"\n","    try:\n","        for dir_ in dirs:\n","            if not os.path.exists(dir_):\n","                os.makedirs(dir_)\n","        return 0\n","    except Exception as err:\n","        print(\"Creating directories error: {0}\".format(err))\n","        exit(-1)\n","\n","\n","def get_args():\n","    argparser = argparse.ArgumentParser(description=__doc__)\n","    argparser.add_argument(\n","        \"-c\", \"--config\",\n","        metavar=\"C\",\n","        default=\"None\",\n","        help=\"The Configuration file\")\n","    # argparser.add_argument(\n","    #     \"-n\", \"--num-client\",\n","    #     default=1,\n","    #     help=\"The number of clients participating in Federated Learning\")\n","    args = argparser.parse_args()\n","    return args\n"],"metadata":{"id":"gq3wRpIk43EH","executionInfo":{"status":"ok","timestamp":1689605805722,"user_tz":-120,"elapsed":7,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["import numpy as np\n","from torch.utils.data import Dataset\n","\n","\n","class CustomDataset(Dataset):\n","    def __init__(self, config, train=True, model_type=\"autoencoder\"):\n","        super().__init__()\n","        self.config = config\n","        self.train = train\n","        self.model_type = model_type\n","        if model_type == \"autoencoder\":\n","            self.load_dataset(self.config[\"auto_dataset\"])\n","        else:\n","            self.load_dataset(self.config[\"trans_dataset\"])\n","\n","    def __len__(self):\n","        return self.rolling_windows.shape[0]\n","\n","    def __getitem__(self, index):\n","        if (self.train) or (self.model_type == \"autoencoder\"):\n","            inp = target = self.rolling_windows[index, :, :]\n","        else:\n","            inp = self.rolling_windows[index, :, :]\n","            target = self.rolling_windows[index,\n","                                          self.config[\"pre_mask\"]:self.config[\"post_mask\"], :]\n","        sample = {\"input\": inp, \"target\": target}\n","        return sample\n","\n","    def load_dataset(self, dataset):\n","        data_dir = self.config[\"data_dir\"]\n","        # data_dir = \"../data/{}/\".format(self.config[\"data_dir\"])\n","        self.data = np.load(data_dir + dataset + \".npz\")\n","\n","        # slice training set into rolling windows\n","        if self.model_type == \"autoencoder\":\n","            if self.train:\n","                self.rolling_windows = np.lib.stride_tricks.sliding_window_view(\n","                    self.data[\"training\"], self.config[\"autoencoder_dims\"], axis=0, writeable=True\n","                ).transpose(0, 2, 1)\n","            else:\n","                self.rolling_windows = np.lib.stride_tricks.sliding_window_view(\n","                    self.data[\"test\"], self.config[\"autoencoder_dims\"], axis=0, writeable=True\n","                ).transpose(0, 2, 1)\n","        else:\n","            if self.train:\n","                self.rolling_windows = np.lib.stride_tricks.sliding_window_view(\n","                    self.data[\"training\"], self.config[\"l_win\"], axis=0, writeable=True\n","                ).transpose(0, 2, 1)\n","            else:\n","                self.rolling_windows = np.lib.stride_tricks.sliding_window_view(\n","                    self.data[\"test\"], self.config[\"l_win\"], axis=0, writeable=True\n","                ).transpose(0, 2, 1)\n"],"metadata":{"id":"fkJhM7Ok5Alo","executionInfo":{"status":"ok","timestamp":1689605806147,"user_tz":-120,"elapsed":3,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["import time\n","\n","import torch\n","from torch.utils.data.dataloader import DataLoader\n","\n","\n","device = torch.device(\"cuda\")\n","# device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n","\n","\n","def create_dataloader(dataset, config):\n","    return DataLoader(dataset,\n","                      batch_size=config[\"batch_size\"],\n","                      shuffle=bool(config[\"shuffle\"]),\n","                      num_workers=config[\"dataloader_num_workers\"])\n","\n","\n","def loss_backprop(criterion, out, targets):\n","    assert out.size(1) == targets.size(1)\n","    loss = criterion(out, targets)\n","    loss.backward()\n","    return loss\n","\n","\n","def create_mask(config):\n","    mask = torch.ones(1, config[\"l_win\"], config[\"l_win\"])\n","    mask[:, config[\"pre_mask\"]:config[\"post_mask\"], :] = 0\n","    mask[:, :, config[\"pre_mask\"]:config[\"post_mask\"]] = 0\n","    mask = mask.float().masked_fill(mask == 0, float(\n","        \"-inf\")).masked_fill(mask == 1, float(0))\n","    return mask\n","\n","\n","min_trans_loss = float(\"inf\")\n","best_trans_model = None\n","epoch_trans_loss = list()\n","\n","def trans_train_epoch(train_iter, model, autoencoder, criterion, mask, opt, epoch, config):\n","    global min_trans_loss, best_trans_model, epoch_trans_loss\n","    model.train()\n","    model.to(device)\n","    autoencoder.eval()\n","    encoder = autoencoder.encoder\n","    encoder.to(device)\n","    batch_loss = list()\n","    for i, batch in enumerate(train_iter):\n","        src = batch[\"input\"].float()\n","        # src.to('cuda')\n","        src = encoder(src.cuda())\n","        trg = batch[\"target\"].float()\n","        # trg.to('cuda')\n","        trg = encoder(trg.cuda())\n","        # trg.to(device)\n","        out = model(src.cuda(), src_mask=mask.cuda())\n","\n","        opt.zero_grad()\n","        out.is_cuda\n","        trg.is_cuda\n","        loss = loss_backprop(criterion, out, trg)\n","        opt.step()\n","        batch_loss.append(loss.item())\n","\n","    if len(batch_loss) > 0:\n","        epoch_trans_loss.append(sum(batch_loss)/len(batch_loss))\n","        print('TRANSFORMER. Epoch: {} \\tTotal Loss: {:.6f}'.format(epoch,\n","                                                                          epoch_trans_loss[-1]))\n","\n","    if epoch_trans_loss[-1] < min_trans_loss:\n","        torch.save(model.state_dict(),\n","                   config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" + f\"best_trans_{epoch}.pth\")\n","        torch.save(opt.state_dict(),\n","                   config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" + f\"optimizer_trans_{epoch}.pth\")\n","        min_trans_loss = epoch_trans_loss[-1]\n","        best_trans_model = f\"best_trans_{epoch}.pth\"\n","    if best_trans_model != None:\n","        return best_trans_model\n","\n","\n","min_auto_loss = float(\"inf\")\n","best_auto_model = None\n","epoch_auto_loss = list()\n","\n","def autoencoder_train_epoch(train_iter, model, criterion, opt, epoch, config):\n","    global min_auto_loss, best_auto_model, epoch_auto_loss\n","    model.train()\n","    model.to(device)\n","    batch_loss = list()\n","    for i, batch in enumerate(train_iter):\n","        src = batch[\"input\"].float()\n","        src.to(device)\n","        trg = batch[\"target\"].float()\n","        trg.to(device)\n","        out = model(src.cuda())\n","        # out = model(src)\n","        opt.zero_grad()\n","        loss = loss_backprop(criterion, out, trg.cuda())\n","        opt.step()\n","        batch_loss.append(loss.item())\n","\n","    if len(batch_loss) > 0:\n","        epoch_auto_loss.append(sum(batch_loss)/len(batch_loss))\n","        print('AUTOENCODER. Epoch: {} \\tTotal Loss: {:.6f}'.format(epoch, epoch_auto_loss[-1]))\n","\n","    if epoch_auto_loss[-1] < min_auto_loss:\n","        torch.save(model.state_dict(),\n","                   config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" + f\"best_autoencoder_{epoch}.pth\")\n","        torch.save(opt.state_dict(),\n","                   config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" + f\"optimizer_autoencoder_{epoch}.pth\")\n","        min_auto_loss = epoch_auto_loss[-1]\n","        best_auto_model = f\"best_autoencoder_{epoch}.pth\"\n","    if best_auto_model != None:\n","        return best_auto_model"],"metadata":{"id":"OEhQbELy4gKX","executionInfo":{"status":"ok","timestamp":1689605806700,"user_tz":-120,"elapsed":5,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["config = {\n","    \"experiment\": \"ECG\",\n","    \"data_dir\": \"/content/drive/MyDrive/PhDResearch/ATRAF/Anomaly-Detection-with-Transformer-Encoder/dataset/\",\n","    \"auto_dataset\": \"ecg_4\",\n","    \"trans_dataset\": \"ecg_4\",\n","    \"num_stacks\": 2,\n","    \"d_model\": 2,\n","    \"d_ff\": 8,\n","    \"num_heads\": 1,\n","    \"dropout\": 0.1,\n","    \"autoencoder_dims\": 200,\n","    \"l_win\":100,\n","    \"pre_mask\": 40,\n","    \"post_mask\": 60,\n","    \"batch_size\": 64,\n","    \"shuffle\": 1,\n","    \"dataloader_num_workers\": 2,\n","    \"auto_num_epoch\": 50,\n","    \"trans_num_epoch\": 50,\n","    \"load_dir\": \"default\",\n","    \"checkpoint_dir\": \"/content/drive/MyDrive/PhDResearch/ATRAF/Anomaly-Detection-with-Transformer-Encoder/experiments/\",\n","    \"result_dir\": \"/content/drive/MyDrive/PhDResearch/ATRAF/Anomaly-Detection-with-Transformer-Encoder/result/\",\n","}"],"metadata":{"id":"R10oeVA18BTn","executionInfo":{"status":"ok","timestamp":1689605807261,"user_tz":-120,"elapsed":22,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","source":["result_dir = config[\"result_dir\"] + config[\"auto_dataset\"] + \"/\"\n","checkpoint_dir = config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\"\n","\n","create_dirs(result_dir, checkpoint_dir)\n","# create_dirs(config[\"result_dir\"], config[\"checkpoint_dir\"])\n","\n","dataset = CustomDataset(config)\n","dataloader = create_dataloader(dataset, config)\n","\n","# Training the autoencoder\n","start = time.time()\n","autoencoder_model = make_autoencoder_model(in_seq_len=config[\"autoencoder_dims\"],\n","                                            out_seq_len=config[\"l_win\"],\n","                                            d_model=config[\"d_model\"])\n","autoencoder_model.float()\n","model_opt = torch.optim.Adam(autoencoder_model.parameters())\n","criterion = torch.nn.MSELoss()\n","for epoch in range(config[\"auto_num_epoch\"]):\n","    config[\"best_auto_model\"] = autoencoder_train_epoch(dataloader,\n","                                                        autoencoder_model,\n","                                                        criterion,\n","                                                        model_opt,\n","                                                        epoch,\n","                                                        config)\n","print(\"COMPLETED TRAINING THE AUTOENCODER\")\n","config[\"auto_train_time\"] = (time.time() - start) / 60\n","\n","# Training the transformer model\n","start = time.time()\n","mask = create_mask(config)\n","# trans_model = make_transformer_model(N=config[\"num_stacks\"],\n","#                                       d_model=config[\"d_model\"],\n","#                                       l_win=config[\"l_win\"],\n","#                                       d_ff=config[\"d_ff\"],\n","#                                       h=config[\"num_heads\"],\n","#                                       dropout=config[\"dropout\"])\n","\n","trans_model = create_fnet_hybrid(N=config[\"num_stacks\"],\n","                                  d_model=config[\"d_model\"],\n","                                  l_win=config[\"l_win\"],\n","                                  d_ff=config[\"d_ff\"],\n","                                  h=config[\"num_heads\"],\n","                                  dropout=config[\"dropout\"])\n","\n","trans_model.float()\n","model_opt = torch.optim.Adam(trans_model.parameters())\n","autoencoder_model.load_state_dict(\n","    torch.load(config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" + config[\"best_auto_model\"]))\n","for epoch in range(config[\"trans_num_epoch\"]):\n","    config[\"best_trans_model\"] = trans_train_epoch(dataloader,\n","                                                    trans_model,\n","                                                    autoencoder_model,\n","                                                    criterion,\n","                                                    mask,\n","                                                    model_opt,\n","                                                    epoch,\n","                                                    config)\n","print(\"COMPLETED TRAINING THE TRANSFORMER\")\n","config[\"trans_train_time\"] = (time.time() - start) / 60\n","save_config(config)\n"],"metadata":{"id":"DUrKvNcr49NW","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1689605851033,"user_tz":-120,"elapsed":41655,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}},"outputId":"b985729b-bd67-41a3-85ed-a09a54a0ba66"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stdout","text":["AUTOENCODER. Epoch: 0 \tTotal Loss: 0.925686\n","AUTOENCODER. Epoch: 1 \tTotal Loss: 0.748897\n","AUTOENCODER. Epoch: 2 \tTotal Loss: 0.693349\n","AUTOENCODER. Epoch: 3 \tTotal Loss: 0.675248\n","AUTOENCODER. Epoch: 4 \tTotal Loss: 0.662870\n","AUTOENCODER. Epoch: 5 \tTotal Loss: 0.653609\n","AUTOENCODER. Epoch: 6 \tTotal Loss: 0.648247\n","AUTOENCODER. Epoch: 7 \tTotal Loss: 0.642004\n","AUTOENCODER. Epoch: 8 \tTotal Loss: 0.639307\n","AUTOENCODER. Epoch: 9 \tTotal Loss: 0.636510\n","AUTOENCODER. Epoch: 10 \tTotal Loss: 0.635793\n","AUTOENCODER. Epoch: 11 \tTotal Loss: 0.632906\n","AUTOENCODER. Epoch: 12 \tTotal Loss: 0.630668\n","AUTOENCODER. Epoch: 13 \tTotal Loss: 0.627632\n","AUTOENCODER. Epoch: 14 \tTotal Loss: 0.625243\n","AUTOENCODER. Epoch: 15 \tTotal Loss: 0.624409\n","AUTOENCODER. Epoch: 16 \tTotal Loss: 0.624697\n","AUTOENCODER. Epoch: 17 \tTotal Loss: 0.623578\n","AUTOENCODER. Epoch: 18 \tTotal Loss: 0.621455\n","AUTOENCODER. Epoch: 19 \tTotal Loss: 0.617572\n","AUTOENCODER. Epoch: 20 \tTotal Loss: 0.615347\n","AUTOENCODER. Epoch: 21 \tTotal Loss: 0.613873\n","AUTOENCODER. Epoch: 22 \tTotal Loss: 0.612100\n","AUTOENCODER. Epoch: 23 \tTotal Loss: 0.614166\n","AUTOENCODER. Epoch: 24 \tTotal Loss: 0.610808\n","AUTOENCODER. Epoch: 25 \tTotal Loss: 0.611207\n","AUTOENCODER. Epoch: 26 \tTotal Loss: 0.610186\n","AUTOENCODER. Epoch: 27 \tTotal Loss: 0.608532\n","AUTOENCODER. Epoch: 28 \tTotal Loss: 0.609167\n","AUTOENCODER. Epoch: 29 \tTotal Loss: 0.609881\n","AUTOENCODER. Epoch: 30 \tTotal Loss: 0.607541\n","AUTOENCODER. Epoch: 31 \tTotal Loss: 0.608072\n","AUTOENCODER. Epoch: 32 \tTotal Loss: 0.606114\n","AUTOENCODER. Epoch: 33 \tTotal Loss: 0.607538\n","AUTOENCODER. Epoch: 34 \tTotal Loss: 0.606585\n","AUTOENCODER. Epoch: 35 \tTotal Loss: 0.605252\n","AUTOENCODER. Epoch: 36 \tTotal Loss: 0.603658\n","AUTOENCODER. Epoch: 37 \tTotal Loss: 0.603447\n","AUTOENCODER. Epoch: 38 \tTotal Loss: 0.604018\n","AUTOENCODER. Epoch: 39 \tTotal Loss: 0.603649\n","AUTOENCODER. Epoch: 40 \tTotal Loss: 0.603337\n","AUTOENCODER. Epoch: 41 \tTotal Loss: 0.602638\n","AUTOENCODER. Epoch: 42 \tTotal Loss: 0.602562\n","AUTOENCODER. Epoch: 43 \tTotal Loss: 0.601566\n","AUTOENCODER. Epoch: 44 \tTotal Loss: 0.599477\n","AUTOENCODER. Epoch: 45 \tTotal Loss: 0.598481\n","AUTOENCODER. Epoch: 46 \tTotal Loss: 0.598229\n","AUTOENCODER. Epoch: 47 \tTotal Loss: 0.597795\n","AUTOENCODER. Epoch: 48 \tTotal Loss: 0.598879\n","AUTOENCODER. Epoch: 49 \tTotal Loss: 0.595569\n","COMPLETED TRAINING THE AUTOENCODER\n","TRANSFORMER. Epoch: 0 \tTotal Loss: 0.961870\n","TRANSFORMER. Epoch: 1 \tTotal Loss: 0.753667\n","TRANSFORMER. Epoch: 2 \tTotal Loss: 0.591952\n","TRANSFORMER. Epoch: 3 \tTotal Loss: 0.505414\n","TRANSFORMER. Epoch: 4 \tTotal Loss: 0.450588\n","TRANSFORMER. Epoch: 5 \tTotal Loss: 0.405701\n","TRANSFORMER. Epoch: 6 \tTotal Loss: 0.372923\n","TRANSFORMER. Epoch: 7 \tTotal Loss: 0.347088\n","TRANSFORMER. Epoch: 8 \tTotal Loss: 0.326796\n","TRANSFORMER. Epoch: 9 \tTotal Loss: 0.313434\n","TRANSFORMER. Epoch: 10 \tTotal Loss: 0.302494\n","TRANSFORMER. Epoch: 11 \tTotal Loss: 0.293589\n","TRANSFORMER. Epoch: 12 \tTotal Loss: 0.287391\n","TRANSFORMER. Epoch: 13 \tTotal Loss: 0.283948\n","TRANSFORMER. Epoch: 14 \tTotal Loss: 0.276687\n","TRANSFORMER. Epoch: 15 \tTotal Loss: 0.273968\n","TRANSFORMER. Epoch: 16 \tTotal Loss: 0.269912\n","TRANSFORMER. Epoch: 17 \tTotal Loss: 0.269108\n","TRANSFORMER. Epoch: 18 \tTotal Loss: 0.269157\n","TRANSFORMER. Epoch: 19 \tTotal Loss: 0.266489\n","TRANSFORMER. Epoch: 20 \tTotal Loss: 0.266815\n","TRANSFORMER. Epoch: 21 \tTotal Loss: 0.263128\n","TRANSFORMER. Epoch: 22 \tTotal Loss: 0.265082\n","TRANSFORMER. Epoch: 23 \tTotal Loss: 0.264883\n","TRANSFORMER. Epoch: 24 \tTotal Loss: 0.264274\n","TRANSFORMER. Epoch: 25 \tTotal Loss: 0.262757\n","TRANSFORMER. Epoch: 26 \tTotal Loss: 0.263264\n","TRANSFORMER. Epoch: 27 \tTotal Loss: 0.262496\n","TRANSFORMER. Epoch: 28 \tTotal Loss: 0.261354\n","TRANSFORMER. Epoch: 29 \tTotal Loss: 0.263785\n","TRANSFORMER. Epoch: 30 \tTotal Loss: 0.262939\n","TRANSFORMER. Epoch: 31 \tTotal Loss: 0.261880\n","TRANSFORMER. Epoch: 32 \tTotal Loss: 0.264171\n","TRANSFORMER. Epoch: 33 \tTotal Loss: 0.263448\n","TRANSFORMER. Epoch: 34 \tTotal Loss: 0.263395\n","TRANSFORMER. Epoch: 35 \tTotal Loss: 0.263467\n","TRANSFORMER. Epoch: 36 \tTotal Loss: 0.263852\n","TRANSFORMER. Epoch: 37 \tTotal Loss: 0.263758\n","TRANSFORMER. Epoch: 38 \tTotal Loss: 0.262016\n","TRANSFORMER. Epoch: 39 \tTotal Loss: 0.260950\n","TRANSFORMER. Epoch: 40 \tTotal Loss: 0.262870\n","TRANSFORMER. Epoch: 41 \tTotal Loss: 0.261537\n","TRANSFORMER. Epoch: 42 \tTotal Loss: 0.263136\n","TRANSFORMER. Epoch: 43 \tTotal Loss: 0.262677\n","TRANSFORMER. Epoch: 44 \tTotal Loss: 0.262794\n","TRANSFORMER. Epoch: 45 \tTotal Loss: 0.261033\n","TRANSFORMER. Epoch: 46 \tTotal Loss: 0.262627\n","TRANSFORMER. Epoch: 47 \tTotal Loss: 0.260652\n","TRANSFORMER. Epoch: 48 \tTotal Loss: 0.261620\n","TRANSFORMER. Epoch: 49 \tTotal Loss: 0.261939\n","COMPLETED TRAINING THE TRANSFORMER\n"]}]},{"cell_type":"markdown","source":["# **Testing**"],"metadata":{"id":"R5Agw6-hMXlC"}},{"cell_type":"code","source":["import math\n","import time\n","import os\n","\n","import matplotlib.pyplot as plt\n","import numpy as np\n","import torch\n","from scipy.stats import norm\n","from sklearn import metrics\n","\n","def load_model(config):\n","    autoencoder_model = make_autoencoder_model(in_seq_len=config[\"autoencoder_dims\"],\n","                                               out_seq_len=config[\"l_win\"],\n","                                               d_model=config[\"d_model\"])\n","    autoencoder_model.load_state_dict(torch.load(\n","        config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" +  config[\"best_auto_model\"]))\n","    autoencoder_model.float()\n","    autoencoder_model.eval()\n","\n","    # trans_model = make_transformer_model(N=config[\"num_stacks\"],\n","    #                                      d_model=config[\"d_model\"],\n","    #                                      l_win=config[\"l_win\"],\n","    #                                      d_ff=config[\"d_ff\"],\n","    #                                      h=config[\"num_heads\"],\n","    #                                      dropout=config[\"dropout\"])\n","\n","    trans_model = create_fnet_hybrid(N=config[\"num_stacks\"],\n","                                  d_model=config[\"d_model\"],\n","                                  l_win=config[\"l_win\"],\n","                                  d_ff=config[\"d_ff\"],\n","                                  h=config[\"num_heads\"],\n","                                  dropout=config[\"dropout\"])\n","\n","    trans_model.load_state_dict(torch.load(\n","        config[\"checkpoint_dir\"] + config[\"auto_dataset\"] + \"/\" + config[\"best_trans_model\"]))\n","    trans_model.float()\n","    trans_model.eval()\n","\n","    return autoencoder_model.encoder, trans_model\n","\n","\n","def create_labels(idx_anomaly_test, n_test, config):\n","    anomaly_index = []\n","    test_labels = np.zeros(n_test)\n","    for i in range(len(idx_anomaly_test)):\n","        idx_start = idx_anomaly_test[i] - (config[\"l_win\"] + 1)\n","        idx_end = idx_anomaly_test[i] + (config[\"l_win\"] + 1)\n","        if idx_start < 0:\n","            idx_start = 0\n","        if idx_end > n_test:\n","            idx_end = n_test\n","        anomaly_index.append(np.arange(idx_start, idx_end))\n","        test_labels[idx_start:idx_end] = 1\n","    return anomaly_index, test_labels\n","\n","\n","def return_anomaly_idx_by_threshold(test_anomaly_metric, threshold):\n","    # test_list = np.squeeze(np.ndarray.flatten(test_anomaly_metric))\n","    idx_error = np.squeeze(np.argwhere(test_anomaly_metric > threshold))\n","\n","    if len(idx_error.shape) == 0:\n","        idx_error = np.expand_dims(idx_error, 0)\n","\n","    return list(idx_error)\n","\n","\n","def augment_detected_idx(idx_detected_anomaly, anomaly_index):\n","    n_anomaly = len(anomaly_index)\n","    idx_detected_anomaly_extended = list(idx_detected_anomaly)\n","    for i in range(n_anomaly):\n","        # print(idx_detected_anomaly)\n","        for j in idx_detected_anomaly:\n","            if j in anomaly_index[i]:\n","                in_original_detection = set(idx_detected_anomaly_extended)\n","                currect_anomaly_win = set(anomaly_index[i])\n","                idx_detected_anomaly_extended = idx_detected_anomaly_extended + \\\n","                    list(currect_anomaly_win - in_original_detection)\n","                # print(j)\n","                break\n","    return list(np.sort(idx_detected_anomaly_extended))\n","\n","\n","def count_TP_FP_FN(idx_detected_anomaly, test_labels):\n","    n_TP = 0\n","    n_FP = 0\n","    #n_detection = len(idx_detected_anomaly)\n","    # for i in range(n_detection):\n","    for i in idx_detected_anomaly:\n","        # if test_labels[idx_detected_anomaly[i]] == 1:\n","        if test_labels[i] == 1:\n","            n_TP = n_TP + 1\n","        else:\n","            n_FP = n_FP + 1\n","\n","    idx_undetected = list(set(np.arange(len(test_labels))\n","                              ) - set(idx_detected_anomaly))\n","    n_FN = 0\n","    for i in idx_undetected:\n","        if test_labels[i] == 1:\n","            n_FN = n_FN + 1\n","    n_TN = len(test_labels) - n_TP - n_FP - n_FN\n","    return n_TP, n_FP, n_FN, n_TN\n","\n","\n","def compute_precision_and_recall(idx_detected_anomaly, test_labels):\n","    # compute true positive\n","    n_TP, n_FP, n_FN, n_TN = count_TP_FP_FN(idx_detected_anomaly, test_labels)\n","\n","    if n_TP + n_FP == 0:\n","        precision = 1\n","    else:\n","        precision = n_TP / (n_TP + n_FP)\n","    recall = n_TP / (n_TP + n_FN)\n","    if precision + recall == 0:\n","        F1 = 0\n","    else:\n","        F1 = 2 * (precision * recall)/(precision + recall)\n","    fpr = n_FP/(n_FP + n_TN)\n","\n","    return precision, recall, F1, fpr, n_TP, n_FP, n_FN\n","\n","\n","def KQp(data, q):\n","    data2 = np.sort(data)  # sap xep tang dan\n","    n = np.shape(data2)[0]  # kich thuoc\n","    p = 1-q  # q tu xet, dat bang smth 0.05 0.025 0.01\n","    h = math.sqrt((p*q)/(n+1))\n","    KQ = 0\n","    for i in range(1, n+1):\n","        a = ((i/n)-p)/h\n","        b = (((i-1)/n)-p)/h\n","        TP = (norm.cdf(a)-norm.cdf(b))*data2[i-1]  # normcdf thu trong matlab\n","        KQ = KQ+TP\n","    # KQp = KQ;\n","    return KQ\n","\n","\n","def plot_roc_curve(fpr_aug, recall_aug, config, n_threshold=20):\n","    tpr = np.insert(recall_aug, [0, n_threshold], [0, 1])\n","    fpr = np.insert(fpr_aug, [0, n_threshold], [0, 1])\n","    auc = metrics.auc(fpr, tpr)\n","    print(\"AUC =\", auc)\n","    lw = 2\n","    plt.plot(fpr, tpr, color=\"darkorange\", lw=lw,\n","             label=\"ROC curve (area = %0.4f)\" % auc)\n","    plt.plot([0, 1], [0, 1], color=\"navy\", lw=lw, linestyle=\"--\")\n","    plt.xlim([0.0, 1.0])\n","    plt.ylim([0.0, 1.05])\n","    plt.xlabel(\"False Positive Rate\")\n","    plt.ylabel(\"True Positive Rate\")\n","    plt.title(\"Augmented Receiver operating characteristic of \" +\n","              config[\"auto_dataset\"])\n","    plt.legend(loc=\"lower right\")\n","    plt.savefig(config[\"result_dir\"] + config[\"auto_dataset\"] + \"/\" + \"augmentedroc.pdf\")\n","    return auc\n","\n","\n","def select_threshold(recon_loss, anomaly_index, test_labels, config, n_threshold=20):\n","    \"\"\"\n","    Select best threshold based on best F1-score\n","    \"\"\"\n","    precision_aug = np.zeros(n_threshold)\n","    recall_aug = np.zeros(n_threshold)\n","    F1_aug = np.zeros(n_threshold)\n","    fpr_aug = np.zeros(n_threshold)\n","    i = 0\n","    threshold_list = np.linspace(np.amin(recon_loss), np.amax(\n","        recon_loss), n_threshold, endpoint=False)\n","    threshold_list = np.flip(threshold_list)\n","\n","    for threshold in threshold_list:\n","        # print(threshold_list[i])\n","        idx_detection = return_anomaly_idx_by_threshold(recon_loss, threshold)\n","        # augment the detection using the ground truth labels\n","        # a method to discount the factor one anomaly appears in multiple consecutive windows\n","        # introduced in \"Unsupervised anomaly detection via variational auto-encoder for seasonal kpis in web applications\"\n","        idx_detection_augmented = augment_detected_idx(\n","            idx_detection, anomaly_index)\n","        precision_aug[i], recall_aug[i], F1_aug[i], fpr_aug[i], _, _, _ = compute_precision_and_recall(idx_detection_augmented,\n","                                                                                                       test_labels)\n","        i = i + 1\n","        # print(precision, recall, F1)\n","\n","    auc = plot_roc_curve(fpr_aug, recall_aug, config)\n","\n","    print(\"\\nAugmented detection:\")\n","    print(\"Best F1 score is {}\".format(np.amax(F1_aug)))\n","    idx_best_threshold = np.squeeze(np.argwhere(F1_aug == np.amax(F1_aug)))\n","    print(\"Best threshold is {}\".format(threshold_list[idx_best_threshold]))\n","    best_thres = np.min(threshold_list[idx_best_threshold])\n","    print(\"At this threshold, precision is {}, recall is {}\".format(precision_aug[idx_best_threshold],\n","                                                                    recall_aug[idx_best_threshold]))\n","    return best_thres, auc\n","\n","\n","def select_KQp_threshold(threshold, recon_loss):\n","    q_list = [0.99, 0.9, 0.1]\n","    temp = math.inf\n","    q_best = 0\n","    for q in q_list:\n","        temp_thres = KQp(recon_loss, q)\n","        # print(temp_thres,abs(temp_thres - threshold))\n","        if abs(temp_thres - threshold) < temp:\n","            temp = abs(temp_thres - threshold)\n","            q_best = q\n","            KQp_thres = temp_thres\n","    print(\"Closest KQp threshold is {} at q = {}\".format(KQp_thres, q_best))\n","    return KQp_thres, q_best\n"],"metadata":{"id":"JLj3DaeGEsyP","executionInfo":{"status":"ok","timestamp":1689605851836,"user_tz":-120,"elapsed":831,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["# Using Transformer-Fourier\n","\n","testing_config_path = '/content/drive/MyDrive/PhDResearch/ATRAF/Anomaly-Detection-with-Transformer-Encoder/result'\n","\n","dataset = CustomDataset(config, train=False)\n","data_loader = create_dataloader(dataset, config)\n","encoder, trans_model = load_model(config)\n","mask = create_mask(config)\n","loss = torch.nn.MSELoss()\n","n_test = len(dataset)\n","recon_loss = np.zeros(n_test)\n","\n","start = time.time()\n","for i, batch in enumerate(data_loader):\n","    src = encoder(batch[\"input\"].float())\n","    trg = encoder(batch[\"target\"].float())\n","    out = trans_model(src, src_mask=mask)\n","    for j in range(config[\"batch_size\"]):\n","        try:\n","            recon_loss[i * config[\"batch_size\"] + j] = loss(\n","                out[j, config[\"pre_mask\"]:config[\"post_mask\"], :], trg[j, config[\"pre_mask\"]:config[\"post_mask\"], :])\n","        except:\n","            pass\n","\n","idx_anomaly_test = dataset.data[\"idx_anomaly_test\"]\n","anomaly_index, test_labels = create_labels(idx_anomaly_test,\n","                                            n_test,\n","                                            config)\n","\n","# Now select a threshold\n","threshold, auc = select_threshold(recon_loss,\n","                                  anomaly_index,\n","                                  test_labels,\n","                                  config)\n","config[\"AUC\"] = auc\n","\n","KQp_thres, q_best = select_KQp_threshold(threshold, recon_loss)\n","config[\"q_best\"] = q_best\n","idx_detection = return_anomaly_idx_by_threshold(recon_loss, KQp_thres)\n","# print(idx_detection)\n","idx_detection_augmented = augment_detected_idx(idx_detection, anomaly_index)\n","# print(anomaly_index_lstm)\n","# print(idx_detection_augmented)\n","precision, recall, F1, _, n_TP, n_FP, n_FN = compute_precision_and_recall(idx_detection_augmented,\n","                                                                          test_labels)\n","config[\"precision\"] = precision\n","config[\"recall\"] = recall\n","config[\"F1\"] = F1\n","config[\"inference_time\"] = (time.time() - start) / 60\n","save_config(config)\n","print(\"\\nPR evaluation using KQE:\")\n","print(\"Precision: {}\".format(precision))\n","print(\"Recall: {}\".format(recall))\n","print(\"F1: {}\".format(F1))\n","print(\"TP: {}\".format(n_TP))\n","print(\"FP: {}\".format(n_FP))\n","print(\"FN: {}\".format(n_FN))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":732},"id":"Jnuau8Y1FHRm","executionInfo":{"status":"ok","timestamp":1689605854727,"user_tz":-120,"elapsed":2896,"user":{"displayName":"Ha Do Thu","userId":"03677637701219815341"}},"outputId":"ba56ca83-d46b-4dfa-a1c8-3c27f1d533b5"},"execution_count":11,"outputs":[{"output_type":"stream","name":"stdout","text":["AUC = 0.9966216216216217\n","\n","Augmented detection:\n","Best F1 score is 0.9969788519637462\n","Best threshold is [1.20785824 1.11601067]\n","At this threshold, precision is [0.9939759 0.9939759], recall is [1. 1.]\n","Closest KQp threshold is 0.46936760240977965 at q = 0.1\n","\n","PR evaluation using KQE:\n","Precision: 0.8638743455497382\n","Recall: 1.0\n","F1: 0.9269662921348314\n","TP: 330\n","FP: 52\n","FN: 0\n"]},{"output_type":"display_data","data":{"text/plain":["<Figure size 640x480 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/NElEQVR4nO3dd1QU198G8GdZ6U0UEESUYo8dRcWCHeyoUexi1Ni7SWyxxBZ7iz2xBn/2FhuJRIwFG/YGKmIFFUWa1N37/uHLJCugLC4s5fmc40n27pRnZ9jd7965MyMTQggQERERFUI62g5AREREpC0shIiIiKjQYiFEREREhRYLISIiIiq0WAgRERFRocVCiIiIiAotFkJERERUaLEQIiIiokKLhRAREREVWiyEKF/bvHkzZDIZwsLCtB3li8lkMsyYMUPbMQqFJk2aoEmTJtqOkSkHBwe0a9dO2zHyHE2/32fMmAGZTKaRZanj0qVLcHNzg7GxMWQyGa5du5brGehfLIRy0OrVqyGTyVC3bl1tR9Gq9+/fY8aMGQgICNBahrQPvLR/urq6cHBwwKhRo/Du3Tut5aKcc+fOHcyYMaNAFMn5RV54r38sr2VKSUlB165d8fbtWyxduhTbtm1DmTJltB3rizx8+BAGBgaQyWS4fPmytuOorYi2AxRkvr6+cHBwwMWLF/HgwQOULVtW25G04v3795g5cyYAaP1X+Jo1a2BiYoL4+Hj4+/tj5cqVuHLlCs6cOaPVXACQkJCAIkX4ltSUO3fuYObMmWjSpAkcHBxUnvvzzz+1E6qAy+n3ep8+fdC9e3fo6+trJNPUqVMxceJETUb8rIcPH+Lx48fYsGEDBg4cmKvrziljx45FkSJFkJSUpO0o2cIeoRzy6NEjnDt3DkuWLIGVlRV8fX21HYkAfP311+jduzcGDx6MXbt2wdvbG2fPnsXFixe1HQ0GBga5XgjFx8fn6vq+hCaz6unpQU9PT2PLy48SExOhVCq1HSNL0va9XC6Xeh40oUiRIjAwMNDIsrLq1atXAICiRYvm6npzip+fH/z8/DB27FhtR8k2FkI5xNfXFxYWFmjbti2+/vrrDAuhgIAAyGSydF22YWFhkMlk2Lx5s0r77t27UblyZRgYGKBKlSrYv38/fHx8VH7tps27aNEirFq1Ck5OTjAyMkKrVq3w9OlTCCEwa9YslCpVCoaGhujYsSPevn2bLtuxY8fQqFEjGBsbw9TUFG3btsXt27dVpvHx8YGJiQmeP38OLy8vmJiYwMrKChMmTIBCoZDyWFlZAQBmzpwpHZr671iYe/fu4euvv0axYsVgYGCA2rVr49ChQ+ky3b59G82aNYOhoSFKlSqF2bNnf/EHeaNGjQB8+JX2XxcuXICnpyfMzc1hZGQEd3d3nD17Nt38z58/x4ABA1CyZEno6+vD0dERQ4cORXJysjTNu3fvMGbMGNjb20NfXx9ly5bF/Pnz02X/73bZs2cPZDIZTp06lW6d69atg0wmw61bt6S2rGzDtPEVp06dwrBhw2BtbY1SpUp9cvu8evUKAwYMQIkSJWBgYIDq1atjy5YtKtP8929u6dKlKFOmDAwNDeHu7q6SUVNZHz9+jGHDhqFChQowNDRE8eLF0bVrV5VDYJs3b0bXrl0BAE2bNpX+7tLeax+PEUp7L+7atQtz5sxBqVKlYGBggObNm+PBgwfpXkPae8vQ0BCurq44ffq0WuOOfv/9d7i6usLIyAgWFhZo3Lhxhr1UZ86cgaurKwwMDODk5IStW7eqPP/27VtMmDABVatWhYmJCczMzNC6dWtcv35dZbq017djxw5MnToVdnZ2MDIyQkxMTJaXAXwonmbMmIHy5cvDwMAAtra26Ny5Mx4+fKix9/qn9n1GY4QuX74MDw8PWFpawtDQEI6Ojvjmm28AfP7zJ7MxQlndPx/7+++/pc/NokWLomPHjrh79670vI+PD9zd3QEAXbt2hUwm++zfTFY/P5RKJZYvX46qVavCwMAAVlZW8PT0VDlUlZCQgFGjRsHS0hKmpqbo0KEDnj9/nu3xiSkpKRg9ejRGjx4NZ2dntefPK9gPn0N8fX3RuXNn6OnpoUePHlizZg0uXbqEOnXqZGt5R44cgbe3N6pWrYp58+YhKioKAwYMgJ2dXabrT05OxsiRI/H27VssWLAA3bp1Q7NmzRAQEIAffvgBDx48wMqVKzFhwgRs3LhRmnfbtm3o168fPDw8MH/+fLx//x5r1qxBw4YNcfXqVZXCS6FQwMPDA3Xr1sWiRYtw4sQJLF68GM7Ozhg6dCisrKywZs0aDB06FJ06dULnzp0BANWqVQPwobhp0KAB7OzsMHHiRBgbG2PXrl3w8vLC3r170alTJwBAREQEmjZtitTUVGm69evXw9DQMFvbM03aB6qFhYXU9vfff6N169ZwcXHB9OnToaOjg02bNqFZs2Y4ffo0XF1dAQAvXryAq6sr3r17h2+//RYVK1bE8+fPsWfPHrx//x56enp4//493N3d8fz5cwwePBilS5fGuXPnMGnSJISHh2PZsmUZ5mrbti1MTEywa9cu6YMzzc6dO/HVV1+hSpUqam3DNMOGDYOVlRWmTZv2yV6WhIQENGnSBA8ePMCIESPg6OiI3bt3w8fHB+/evcPo0aNVpt+6dStiY2MxfPhwJCYmYvny5WjWrBlu3ryJEiVKaCzrpUuXcO7cOXTv3h2lSpVCWFgY1qxZgyZNmuDOnTswMjJC48aNMWrUKKxYsQKTJ09GpUqVAED6b2Z+/vln6OjoYMKECYiOjsaCBQvQq1cvXLhwQZpmzZo1GDFiBBo1aoSxY8ciLCwMXl5esLCw+GxhCXz4Qp4xYwbc3Nzw008/QU9PDxcuXMDff/+NVq1aSdM9ePAAX3/9NQYMGIB+/fph48aN8PHxgYuLC7766isAQGhoKA4cOICuXbvC0dERL1++xLp16+Du7o47d+6gZMmSKuueNWsW9PT0MGHCBCQlJUFPTw937tzJ0jIUCgXatWsHf39/dO/eHaNHj0ZsbCz++usv3Lp1Cy1atNDIe/1T+/5jr169QqtWrWBlZYWJEyeiaNGiCAsLw759+wDgs58/X7J/PnbixAm0bt0aTk5OmDFjBhISErBy5Uo0aNAAV65cgYODAwYPHgw7OzvMnTsXo0aNQp06daT3RkbU+fwYMGAANm/ejNatW2PgwIFITU3F6dOncf78edSuXRvAh0Js165d6NOnD+rVq4dTp06hbdu2ma7/c5YtW4aoqChMnTpV2ub5kiCNu3z5sgAg/vrrLyGEEEqlUpQqVUqMHj1aZbqTJ08KAOLkyZMq7Y8ePRIAxKZNm6S2qlWrilKlSonY2FipLSAgQAAQZcqUSTevlZWVePfundQ+adIkAUBUr15dpKSkSO09evQQenp6IjExUQghRGxsrChatKgYNGiQSqaIiAhhbm6u0t6vXz8BQPz0008q09asWVO4uLhIj1+/fi0AiOnTp6fbVs2bNxdVq1aV1p+2vdzc3ES5cuWktjFjxggA4sKFC1Lbq1evhLm5uQAgHj16lG7Z/zV9+nQBQAQHB4vXr1+LsLAwsXHjRmFoaCisrKxEfHy8tO5y5coJDw8PoVQqpfnfv38vHB0dRcuWLaW2vn37Ch0dHXHp0qV060ubd9asWcLY2FiEhISoPD9x4kQhl8vFkydPpLaPt1GPHj2EtbW1SE1NldrCw8OFjo6OyjbP6jbctGmTACAaNmyosszMLFu2TAAQv//+u9SWnJws6tevL0xMTERMTIwQ4t+/OUNDQ/Hs2TNp2gsXLggAYuzYsRrN+v79+3RZAwMDBQCxdetWqW337t0Zvr+EEMLd3V24u7tLj9Pei5UqVRJJSUlS+/LlywUAcfPmTSGEEElJSaJ48eKiTp06Ku+jzZs3CwAqy8zI/fv3hY6OjujUqZNQKBQqz/33761MmTICgPjnn3+ktlevXgl9fX0xfvx4qS0xMTHdch49eiT09fVV/kbSXp+Tk1O67ZfVZWzcuFEAEEuWLEn3utKya+K9/ql9n/Zc2vt9//79AkCG78E0n8qU9rmQJqv7JyM1atQQ1tbW4s2bN1Lb9evXhY6Ojujbt6/UlrYvdu/e/cnlCZH1z4+///5bABCjRo1Kt4y03EFBQQKAGDNmjMrzPj4+mW6fTwkPDxempqZi3bp1Qoh/982n9kVexUNjOcDX1xclSpRA06ZNAXw45OHt7Y0dO3ZIh4zU8eLFC9y8eRN9+/aFiYmJ1O7u7o6qVatmOE/Xrl1hbm4uPU47c613794q41Dq1q2L5ORkPH/+HADw119/4d27d+jRowciIyOlf3K5HHXr1sXJkyfTrWvIkCEqjxs1aoTQ0NDPvq63b9/i77//Rrdu3RAbGyut682bN/Dw8MD9+/elXEePHkW9evWk3hjgw6+9Xr16fXY9/1WhQgVYWVnBwcEB33zzDcqWLYtjx47ByMgIAHDt2jXcv38fPXv2xJs3b6RM8fHxaN68Of755x8olUoolUocOHAA7du3l35t/Vdad/vu3bvRqFEjWFhYqGzPFi1aQKFQ4J9//sk0q7e3N169eqVy6HTPnj1QKpXw9vZWexumGTRoEORy+We31dGjR2FjY4MePXpIbbq6uhg1ahTi4uLSHbbz8vJS6aF0dXVF3bp1cfToUY1m/W8vYEpKCt68eYOyZcuiaNGiuHLlymdf16f0799fZexQ2qHTtL/ny5cv482bNxg0aJDK+6hXr14qvYqZOXDgAJRKJaZNmwYdHdWP348P0VSuXFlaP/Dh771ChQoq7y19fX1pOQqFAm/evIGJiQkqVKiQ4bbo169ful7UrC5j7969sLS0xMiRI9Mt93NjdnLq7zRtnM3hw4eRkpLyyWmzQp3981/h4eG4du0afHx8UKxYMam9WrVqaNmypfQeUFdWPz/27t0LmUyG6dOnp1tGWu7jx48D+NDT9l8Z7c+s+OGHH+Dk5FQgBnzz0JiGKRQK7NixA02bNsWjR4+k9rp162Lx4sXw9/f/ZPdqRh4/fgwAGZ51VrZs2Qw/8EqXLq3yOK0osre3z7A9KioKAHD//n0AQLNmzTLMYmZmpvI47Vj0f1lYWEjL+5QHDx5ACIEff/wRP/74Y4bTvHr1CnZ2dnj8+HGGlyGoUKHCZ9fzX3v37oWZmRlev36NFStW4NGjRypfDGmvv1+/fpkuIzo6GsnJyYiJiZEOT2Xm/v37uHHjRrptlCZt4GRG0sYo7dy5E82bNwfw4bBYjRo1UL58eQDqbcM0jo6On8yc5vHjxyhXrly6L4S0w0tpf5dpypUrl24Z5cuXx65duzSaNSEhAfPmzcOmTZvw/PlzCCGk56Kjo7P02jLz8fsmrbhJ+3vO7L1YpEiRdGemZeThw4fQ0dFB5cqV1c6Slue/7620cSGrV6/Go0ePVH5oFS9ePN38GW3PrC7j4cOHqFChQrYG9OfU36m7uzu6dOmCmTNnYunSpWjSpAm8vLzQs2dPtc4sS6PO/vmvtL+LjD6PKlWqBD8/P8THx8PY2Fit5Wb18+Phw4coWbKkShGWUUYdHZ102zU7ZzOfP38e27Ztg7+/f7rPh/yIhZCG/f333wgPD8eOHTuwY8eOdM/7+vpKhVBmvzCy02v0scx+SWXWnvZlkjYAb9u2bbCxsUk33ccfglnpWchM2romTJgADw+PDKfR9CUHGjduDEtLSwBA+/btUbVqVfTq1QtBQUHQ0dGRMi1cuBA1atTIcBkmJiYZDjDPiFKpRMuWLfH9999n+HxaQZMRfX19eHl5Yf/+/Vi9ejVevnyJs2fPYu7cuSrLB9Tbhl86riq7NJV15MiR2LRpE8aMGYP69evD3NwcMpkM3bt3/+LB8597f+SmrGSZO3cufvzxR3zzzTeYNWsWihUrBh0dHYwZMybDbZHR9lR3GdmRU3+nMpkMe/bswfnz5/HHH3/Az88P33zzDRYvXozz58+r9KDnR1/y+ZGTvv/+ezRq1AiOjo7SOMvIyEgAH3rHnjx5kmEhn1exENIwX19fWFtbY9WqVeme27dvH/bv34+1a9fC0NBQ+rX58QX9Pv6lnXaxrYzOXsmo7Uukjfy3trZGixYtNLLMzAo+JycnAB8Ot3xuXWXKlJF6a/4rODg427lMTEwwffp09O/fH7t27UL37t2l129mZvbJTFZWVjAzM8vwrKj/cnZ2RlxcXLa3pbe3N7Zs2QJ/f3/cvXsXQgjpsBig3jZUV5kyZXDjxg0olUqVX3337t2Tnv+vjPZPSEiI1FOiqax79uxBv379sHjxYqktMTEx3fsoJ64Y/N/3YtqhbwBITU1FWFjYJwfhAh/+HpRKJe7cuZNpoa2OPXv2oGnTpvjtt99U2t+9eycV/JpahrOzMy5cuICUlBTo6upmuCxNvNezo169eqhXrx7mzJmD7du3o1evXtixYwcGDhyo1t9BdvdP2t9FRp9H9+7dg6Wlpdq9QWl5svL54ezsDD8/P7x9+zbTXqEyZcpAqVTi0aNHKr232fkOefLkCR4/fpxhr12HDh1gbm6ery5Um//7tPKQhIQE7Nu3D+3atcPXX3+d7t+IESMQGxsrnS5apkwZyOXydONEVq9erfK4ZMmSqFKlCrZu3Yq4uDip/dSpU7h586ZGX4OHhwfMzMwwd+7cDI+5v379Wu1lpo2/+fiNYW1tjSZNmmDdunUIDw//5LratGmD8+fPq1zv5/Xr1198faZevXqhVKlSmD9/PgDAxcUFzs7OWLRokcq2/jiTjo4OvLy88Mcff2R4JdW0X+3dunVDYGAg/Pz80k3z7t07pKamfjJfixYtUKxYMezcuRM7d+6Eq6uryoePOttQXW3atEFERAR27twptaWmpmLlypUwMTFJdzbbgQMHVMZ5XLx4ERcuXEDr1q01mlUul6froVm5cmW6ntS0Lx5NfiDXrl0bxYsXx4YNG1T2na+vb5YOB3t5eUFHRwc//fRTut6W7PQ6ZbQtdu/enW68jSaW0aVLF0RGRuKXX35Jt4y0+TXxXldHVFRUuuxpBUzaxf0yy5SR7O4fW1tb1KhRA1u2bFFZz61bt/Dnn3+iTZs2WXg16WX186NLly4QQkgXjswod1pP3MffLytXrlQ71/r167F//36Vf2ljjRYtWpTvrpvHHiENOnToEGJjY9GhQ4cMn69Xr550cUVvb2+Ym5uja9euWLlyJWQyGZydnXH48OEMx43MnTsXHTt2RIMGDdC/f39ERUXhl19+QZUqVTL8ws4uMzMzrFmzBn369EGtWrXQvXt3WFlZ4cmTJzhy5AgaNGiQ4QfhpxgaGqJy5crYuXMnypcvj2LFiqFKlSqoUqUKVq1ahYYNG6Jq1aoYNGgQnJyc8PLlSwQGBuLZs2fStUy+//57bNu2DZ6enhg9erR0+nxar0V26erqYvTo0fjuu+9w/PhxeHp64tdff0Xr1q3x1VdfoX///rCzs8Pz589x8uRJmJmZ4Y8//gDwYZ/8+eefcHd3x7fffotKlSohPDwcu3fvxpkzZ1C0aFF89913OHToENq1ayed+hwfH4+bN29iz549CAsL++Qvd11dXXTu3Bk7duxAfHw8Fi1alG6arG5DdX377bdYt24dfHx8EBQUBAcHB+zZswdnz57FsmXLYGpqqjJ92bJl0bBhQwwdOhRJSUlYtmwZihcvrtKtr4ms7dq1w7Zt22Bubo7KlSsjMDAQJ06cSDcmpkaNGpDL5Zg/fz6io6Ohr6+PZs2awdraOlvbA/hwIcYZM2Zg5MiRaNasGbp164awsDBs3rwZzs7On+19KFu2LKZMmYJZs2ahUaNG6Ny5M/T19XHp0iWULFkS8+bNUytPu3bt8NNPP6F///5wc3PDzZs34evrK/XAaHIZffv2xdatWzFu3DhcvHgRjRo1Qnx8PE6cOIFhw4ahY8eOGnmvq2PLli1YvXo1OnXqBGdnZ8TGxmLDhg0wMzOTio9PZfrYl+yfhQsXonXr1qhfvz4GDBggnT5vbm6e7XsIZvXzo2nTpujTpw9WrFiB+/fvw9PTE0qlEqdPn0bTpk0xYsQIuLi4oEuXLli2bBnevHkjnT4fEhICQL0e1IzGuaYVgO7u7hmeQJKn5fJZagVa+/bthYGBgXQqdkZ8fHyErq6uiIyMFEJ8OLWzS5cuwsjISFhYWIjBgweLW7dupTt9XgghduzYISpWrCj09fVFlSpVxKFDh0SXLl1ExYoVpWnSTmVeuHChyryZnbKZ2SmPJ0+eFB4eHsLc3FwYGBgIZ2dn4ePjIy5fvixN069fP2FsbJzuNX58SqoQQpw7d064uLgIPT29dKdqPnz4UPTt21fY2NgIXV1dYWdnJ9q1ayf27NmjsowbN24Id3d3YWBgIOzs7MSsWbPEb7/9ptbp869fv073XHR0tDA3N1c59fnq1auic+fOonjx4kJfX1+UKVNGdOvWTfj7+6vM+/jxY9G3b19hZWUl9PX1hZOTkxg+fLjKKdixsbFi0qRJomzZskJPT09YWloKNzc3sWjRIpGcnCxN9/F2SfPXX38JAEImk4mnT59m+Pqysg2zc3rry5cvRf/+/YWlpaXQ09MTVatWTfd3+d+/ucWLFwt7e3uhr68vGjVqJK5fv67xrFFRUVImExMT4eHhIe7duyfKlCkj+vXrpzLthg0bhJOTk5DL5Sqn0md2+vzH74+MLmUhhBArVqwQZcqUEfr6+sLV1VWcPXtWuLi4CE9Pz89vVPHhVPSaNWsKfX19YWFhIdzd3aXLbQjx4fT5tm3bppvv49yJiYli/PjxwtbWVhgaGooGDRqIwMDALL8+dZYhxIdLF0yZMkU4OjoKXV1dYWNjI77++mvx8OFDaZovfa9/at9/fPr8lStXRI8ePUTp0qWFvr6+sLa2Fu3atVP5nPpUpow+q4T4/P7JzIkTJ0SDBg2EoaGhMDMzE+3btxd37txRmUad0+eFyPrnR2pqqli4cKGoWLGi0NPTE1ZWVqJ169YiKChImiY+Pl4MHz5cFCtWTJiYmAgvLy8RHBwsAIiff/45S3kyk59Pn5cJoYVRgKQxNWrUgJWVFf766y9tR6FCKiwsDI6Ojli4cCEmTJig7ThaoVQqYWVlhc6dO2PDhg3ajkOUZdeuXUPNmjXx+++/q305koKCY4TyiZSUlHTjSQICAnD9+nWt38iUqDBJTExMN15k69atePv2Ld+LlKclJCSka1u2bBl0dHTQuHFjLSTKGzhGKJ94/vw5WrRogd69e6NkyZK4d+8e1q5dCxsbm3QXNCSinHP+/HmMHTsWXbt2RfHixXHlyhX89ttvqFKlinR/M6K8aMGCBQgKCkLTpk1RpEgRHDt2DMeOHcO3334Le3t7KBSKzw5cNzExyfeXJfgYC6F8wsLCAi4uLvj111/x+vVrGBsbo23btvj5558zvHAaEeUMBwcH2NvbY8WKFdLpyn379sXPP/9c6O9oT3mbm5sb/vrrL8yaNQtxcXEoXbo0ZsyYgSlTpgAAnj59+tkLWU6fPj3bg7/zKo4RIiIiIiQmJuLMmTOfnMbJyUmtsxLzAxZCREREVGhxsDQREREVWoVujJBSqcSLFy9gamqaI5fgJyIiIs0TQiA2NhYlS5bU6M1eC10h9OLFi3R3YCciIqL84enTpyhVqpTGllfoCqG02wI8ffoUZmZmWk5DREREWRETEwN7e/t0t/f5UoWuEEo7HGZmZsZCiIiIKJ/R9LAWDpYmIiKiQouFEBERERVaLISIiIio0GIhRERERIUWCyEiIiIqtFgIERERUaHFQoiIiIgKLRZCREREVGixECIiIqJCi4UQERERFVpaLYT++ecftG/fHiVLloRMJsOBAwc+O09AQABq1aoFfX19lC1bFps3b87xnERERFQwabUQio+PR/Xq1bFq1aosTf/o0SO0bdsWTZs2xbVr1zBmzBgMHDgQfn5+OZyUiIiICiKt3nS1devWaN26dZanX7t2LRwdHbF48WIAQKVKlXDmzBksXboUHh4eORWTiIiICqh8NUYoMDAQLVq0UGnz8PBAYGCglhIRERFRTlMqBW7ffpUjy9Zqj5C6IiIiUKJECZW2EiVKICYmBgkJCTA0NEw3T1JSEpKSkqTHMTExH/7n/n7gxs9AcmyOZiYiIqLsC482RP8t7jgVUixHlp+vCqHsmDdvHmbOnJn+iQtzgIT7uR+IiIiIsuTgrQoYuLsDIuONASTmyDryVSFkY2ODly9fqrS9fPkSZmZmGfYGAcCkSZMwbtw46XFMTAzs7e2B5LgPDTIdwNg2xzITERGR+l7HGqDX/75GfJIuAMDaNAGvcuAgTr4qhOrXr4+jR4+qtP3111+oX79+pvPo6+tDX18/84Ua2wKDn2kqIhEREWmAFYBlRa9g0KA/4OVVEUuWuMPJabnG16PVQiguLg4PHjyQHj969AjXrl1DsWLFULp0aUyaNAnPnz/H1q1bAQBDhgzBL7/8gu+//x7ffPMN/v77b+zatQtHjhzR1ksgIiIiDVAolEhNVUJf/9/SZMCAmrC3N0OrVs6Ijc2ZMb1aPWvs8uXLqFmzJmrWrAkAGDduHGrWrIlp06YBAMLDw/HkyRNpekdHRxw5cgR//fUXqlevjsWLF+PXX3/lqfNERET52NOn0WjRYhsmTPhTpV0mk8HDoyxkMlmOrVsmhBA5tvQ8KCYmBubm5oheagszRThgYsdDY0RERFqya9dtDB58GO/efRgMfeRIT7RpUy7ddNL3d3Q0zMzMNLb+fDVGiIiIiAqGmJgkjBp1DFu2XJfa7O3NYGqql6s5WAgRERFRrgoMfIrevfcjNDRKavP2/gpr1rSFhUXGZ4HnFBZCRERElCtSU5WYM+cfzJr1DxSKDyNzTE31sGpVG/TuXS1HxwJlhoUQERER5bg3b96jffv/ITDw33G5bm72+P33TnB0tNBarnx1rzEiIiLKn4oWNUCRIh/KDrlchpkzm+DUKR+tFkEACyEiIiLKBXK5DrZt64RatWxx5sw3mDbNXSqMtImHxoiIiEjjTp0Kg6GhLlxd7aS2MmWK4vLlQVoZC5QZ7ZdiREREVGAkJyswadIJNG26BT167EVsbJLK83mpCAJYCBEREZGGBAdHon793/Dzz2chBBAaGoU1ay5rO9Yn8dAYERERfREhBDZsuIIxY44jISEVAKCrq4M5c5ph/Hg3Laf7NBZCRERElG2vX8dj0KA/cPBgsNRWoUJxbN/eBbVq2WoxWdawECIiIqJs8fN7AB+fg4iIiJPahgxxweLFHjAy0tVisqxjIURERERqe/kyDl5eO5GY+OFQmKWlETZu7ID27StoOZl6OFiaiIiI1FaihAl+/rk5AMDDwxk3bw7Nd0UQwB4hIiIiygKlUkChUEJXVy61jRxZF6VKmaFTp0rQ0clbp8VnFXuEiIiI6JPCw2PRurUvpk79W6VdR0eGLl0q59siCGAhRERERJ9w8OA9VK26Bn/++RALF57D338/0nYkjeKhMSIiIkonPj4Z48f/iXXrgqS2EiVMtJgoZ7AQIiIiIhVBQS/Qs+c+hIS8kdo6dqyAX3/tAEtLIy0m0zwWQkRERAQAUCiUWLToHKZOPYnUVCUAwMhIF8uWeWDgwFp57j5hmsBCiIiIiBAZ+R5du+5GQECY1ObiYovt27ugfPni2guWwzhYmoiIiGBuro+4uGQAgEwGTJrUEOfODSjQRRDAQoiIiIgA6OrK4evbGZUqWeLkyX6YO7c59PTkn58xn+OhMSIiokIoMPApjIx0Ub26jdRWvnxx3Lo1LF9fF0hd7BEiIiIqRFJTlZg5MwCNGm1Cjx578f59isrzhakIAlgIERERFRqhoVFo3HgTZsw4BYVC4O7dSKxefUnbsbSKh8aIiIgKOCEEtm27gREjjiI29sOAaLlchunT3TFmTD0tp9MuFkJEREQFWFRUAoYMOYJdu25Lbc7OFvj9986oV6+UFpPlDSyEiIiICqiAgDD06bMfz57FSG39+9fA8uWeMDXV12KyvIOFEBERUQEUHh4LD4/fkZysAABYWBhg3bp26Nr1Ky0ny1s4WJqIiKgAsrU1xfTp7gCApk0dcOPGUBZBGWCPEBERUQEghIBSKSCX/9vH8cMPDWBvb4ZevaoVutPis4o9QkRERPnc69fx6NRpJ2bP/kelXS7XQZ8+1VkEfQJ7hIiIiPIxP78H8PE5iIiIOBw+HIJWrZxRv769tmPlGyyEiIiI8qHExFRMmnQCy5ZdkNosLAyl6wRR1rAQIiIiymdu3nyJXr324ebNV1Kbh4czNm/2go2NiRaT5T8shIiIiPIJpVJg5coL+OGHE0hK+nBavL6+HAsWtMSIEa4cC5QNLISIiIjygTdv3qNXr33w83sotVWtao3t27ugShVrLSbL33jWGBERUT5gbKyH589jpcdjx9bDxYuDWAR9IRZCRERE+YCBQRFs394Zjo5F4efXG0uWeMDAgAd2vhS3IBERUR4UFPQCxsZ6qFjRUmqrWrUEQkJGokgR9mNoCrckERFRHqJQKDF//hnUq/cbevTYi6SkVJXnWQRpFrcmERFRHvH0aTSaN9+KiRP9kZqqxLVrEVi9+pK2YxVoPDRGRESUB+zadRuDBx/Gu3eJAACZDJg4sSGGD3fVcrKCjYUQERGRFsXEJGHUqGPYsuW61GZvb4Zt2zrB3d1Be8EKCRZCREREWhIY+BS9e+9HaGiU1Obt/RXWrGkLCwtDLSYrPFgIERERacHz5zFo0mQLkpM/XCHa1FQPq1a1Qe/e1SCT8QrRuYWDpYmIiLTAzs4MEybUBwC4udnj+vUh6NOnOougXMYeISIiolwghAAAlUJnxowmKF3aHAMG1OJp8VrCrU5ERJTDoqIS0L37XixeHKjSrqsrx+DBtVkEaRF7hIiIiHJQQEAY+vTZj2fPYrB//100b+6ImjVttR2L/h9LUCIiohyQnKzAxIkn0KzZFjx7FgMAMDHRQ0REnJaT0X+xR4iIiEjDgoMj0bPnPly5Ei61NW3qgK1bO6FUKTMtJqOPsRAiIiLSECEE1q8PwtixfkhI+HCPMF1dHcyZ0wzjx7tBR4dnhOU1LISIiIg04O3bBPTvfxCHDgVLbRUqFMf27V1QqxbHBOVVLISIiIg0QF9fjnv3IqXHQ4fWxqJFrWBkpKvFVPQ5HCxNRESkAcbGevD17YySJU1x6FB3rF7dlkVQPsAeISIiomy4efMljI314ORkIbXVrl0SoaGjoK/Pr9f8gj1CREREalAqBZYvP486dTagV699SE1VqjzPIih/YSFERESUReHhsWjd2hdjxvghKUmB8+efYc2aS9qORV9A64XQqlWr4ODgAAMDA9StWxcXL1785PTLli1DhQoVYGhoCHt7e4wdOxaJiYm5lJaIiAqrgwfvoWrVNfjzz4dS29ix9TBokIsWU9GX0mr/3c6dOzFu3DisXbsWdevWxbJly+Dh4YHg4GBYW1unm3779u2YOHEiNm7cCDc3N4SEhMDHxwcymQxLlizRwisgIqKCLj4+GePH/4l164KkNltbE2ze7IVWrZy1mIw0Qas9QkuWLMGgQYPQv39/VK5cGWvXroWRkRE2btyY4fTnzp1DgwYN0LNnTzg4OKBVq1bo0aPHZ3uRiIiIsiMo6AVq1VqvUgR5eVXEjRtDWQQVEForhJKTkxEUFIQWLVr8G0ZHBy1atEBgYGCG87i5uSEoKEgqfEJDQ3H06FG0adMm0/UkJSUhJiZG5R8REdHnPH0aDTe3jQgJeQMAMDLSxYYN7bFvXzdYWhppOR1pitYKocjISCgUCpQoUUKlvUSJEoiIiMhwnp49e+Knn35Cw4YNoaurC2dnZzRp0gSTJ0/OdD3z5s2Dubm59M/e3l6jr4OIiAome3tzDBtWGwDg4mKLq1cHY+DAWpDJeJuMgkTrg6XVERAQgLlz52L16tW4cuUK9u3bhyNHjmDWrFmZzjNp0iRER0dL/54+fZqLiYmIKD8RQqg8njevBZYsaYVz5wagfPniWkpFOUlrg6UtLS0hl8vx8uVLlfaXL1/CxsYmw3l+/PFH9OnTBwMHDgQAVK1aFfHx8fj2228xZcoU6Oikr+v09fWhr6+v+RdAREQFRkxMEkaNOgZXVzsMG1ZHajcwKIKxY+trMRnlNK31COnp6cHFxQX+/v5Sm1KphL+/P+rXz/iP7v379+mKHblcDiB9FU9ERJQVgYFPUaPGWmzZch3jx/+Ju3dfazsS5SKtnj4/btw49OvXD7Vr14arqyuWLVuG+Ph49O/fHwDQt29f2NnZYd68eQCA9u3bY8mSJahZsybq1q2LBw8e4Mcff0T79u2lgoiIiCgrUlOVmD37H8ye/Q8Uig8/pnV1dfDwYRQqVbLScjrKLVothLy9vfH69WtMmzYNERERqFGjBo4fPy4NoH7y5IlKD9DUqVMhk8kwdepUPH/+HFZWVmjfvj3mzJmjrZdARET5UGhoFHr33ofAwGdSm5ubPX7/vRMcHS0+MScVNDJRyI4pxcTEwNzcHNFLbWGmCAdM7IDBzz4/IxER5XtCCGzdeh0jRhxDXFwyAEAul2HaNHdMntwIRYrkq3OIChXp+zs6GmZmZhpbLu8MR0REhcK7d4kYPPgwdu26LbU5OVnA17cz6tUrpcVkpE0shIiIqFCQyYALF/49AuDjUwMrVnjC1JRnFhdm7AMkIqJCwdzcANu2dYKlpRF27foamzZ1ZBFE7BEiIqKCKTg4EsbGeihV6t/xJI0alUFY2GgYG+tpMRnlJewRIiKiAkUIgXXrLqNmzXXo23c/lErVc4JYBNF/sRAiIqIC4/XreHh57cSQIUeQkJCKkyfDsH590OdnpEKLh8aIiKhA8PN7AB+fg4iIiJPahgxxQd++1bWYivI6FkJERJSvJSamYtKkE1i27ILUZmlphI0bO6B9+wpaTEb5AQshIiLKt27efIlevfbh5s1XUpuHhzM2b/aCjY2JFpNRfsFCiIiI8qXHj9+hTp0NSEpSAAD09eVYsKAlRoxwhY6OTMvpKL/gYGkiIsqXypQpKo3/qVrVGpcvf4tRo+qyCCK1sEeIiIjyraVLPVCmjDnGj3eDgQG/0kh97BEiIqI8Lz4+GUOGHMbmzddU2o2N9TBlSmMWQZRt/MshIqI8LSjoBXr12ofg4Dfw9b2JRo1Kw9m5mLZjUQHBHiEiIsqTFAol5s8/g3r1fkNw8BsAgFIpcOvWq8/MSZR17BEiIqI85+nTaPTpsx+nTj2W2lxcbLF9exeUL19ci8mooGEhREREecquXbcxePBhvHuXCACQyYCJExtixowm0NOTazkdFTQshIiIKE+IjU3CyJHHsGXLdanN3t4M27Z1gru7g/aCUYHGQoiIiPKEpCQF/vzzofTY2/srrFnTFhYWhlpMRQUdB0sTEVGeYGlphC1bvGBmpo+tW73wv/91YRFEOY49QkREpBWhoVEwNtZFiRL/3hOsZUtnPH48BkWLGmgxGRUm7BEiIqJcJYTAli3XUL36WnzzzSEIIVSeZxFEuYmFEBER5ZqoqAR0774XPj4HEReXjKNH72PTpmvajkWFGA+NERFRrggICEOfPvvx7FmM1ObjUwNdu1bWYioq7FgIERFRjkpOVmDatJNYsOAs0o6CWVgYYN26duja9SvthqNCj4UQERHlmHv3ItGr1z5cuRIutTVt6oCtWzuhVCkzLSYj+oCFEBER5YjQ0CjUqrUOCQmpAABdXR3MmdMM48e7QUdHpuV0RB9wsDQREeUIJycLdO5cCQBQoUJxnD8/EN9914BFEOUp7BEiIqIcs2pVG5QpY44pUxrDyEhX23GI0vmiHqHExERN5SAionwsMTEVY8cex+7dt1Xazc0NMGdOcxZBlGepXQgplUrMmjULdnZ2MDExQWhoKADgxx9/xG+//abxgERElLfdvPkSrq4bsGzZBXz77WE8fRqt7UhEWaZ2ITR79mxs3rwZCxYsgJ6entRepUoV/PrrrxoNR0REeZdSKbB8+XnUqbMBN2++AgAkJKTg8uUXWk5GlHVqF0Jbt27F+vXr0atXL8jlcqm9evXquHfvnkbDERFR3hQeHos2bXwxZowfkpIUAICqVa1x+fK36NSpkpbTEWWd2oOlnz9/jrJly6ZrVyqVSElJ0UgoIiLKuw4evIeBA/9AZOR7qW3s2HqYO7c5DAx4Dg7lL2r/xVauXBmnT59GmTJlVNr37NmDmjVraiwYERHlLfHxyRg//k+sWxcktdnammDzZi+0auWsxWRE2ad2ITRt2jT069cPz58/h1KpxL59+xAcHIytW7fi8OHDOZGRiIjygJiYJOzde1d67OVVERs2tIelpZEWUxF9GbXHCHXs2BF//PEHTpw4AWNjY0ybNg13797FH3/8gZYtW+ZERiIiygNsbU3x66/tYWSkiw0b2mPfvm4sgijfkwmRdgu8wiEmJgbm5uaIXmoLM0U4YGIHDH6m7VhERHnO06fRMDbWQ7Fihirtr17Fw9raWEupqLCSvr+jo2Fmprn71KndI+Tk5IQ3b96ka3/37h2cnJw0EoqIiLRr167bqFZtLQYPPoyPfy+zCKKCRO1CKCwsDAqFIl17UlISnj9/rpFQRESkHTExSfDxOQBv7z149y4Re/bcwfbtN7UdiyjHZHmw9KFDh6T/9/Pzg7m5ufRYoVDA398fDg4OGg1HRES5JzDwKXr12odHj95Jbd7eX6FNm3LaC0WUw7JcCHl5eQEAZDIZ+vXrp/Kcrq4uHBwcsHjxYo2GIyKinJeaqsScOf9g1qx/oFB8OAxmaqqHVavaoHfvapDJeLd4KriyXAgplUoAgKOjIy5dugRLS8scC0VERLkjNDQKvXvvQ2DgvyeNuLnZ4/ffO8HR0UKLyYhyh9rXEXr06FFO5CAiolz24MFb1Kq1DrGxyQAAuVyGadPcMXlyIxQpovYQUqJ8KVvXQo+Pj8epU6fw5MkTJCcnqzw3atQojQQjIqKc5exsgebNnXDgwD04OVnA17cz6tUrpe1YRLlK7ULo6tWraNOmDd6/f4/4+HgUK1YMkZGRMDIygrW1NQshIqJ8QiaTYcOG9ihTxhyzZjWFqam+tiMR5Tq1+z7Hjh2L9u3bIyoqCoaGhjh//jweP34MFxcXLFq0KCcyEhHRF0pOVmDixBM4ciREpd3S0gjLlnmyCKJCS+1C6Nq1axg/fjx0dHQgl8uRlJQEe3t7LFiwAJMnT86JjERE9AWCgyNRv/5vmD//LL755hBevozTdiSiPEPtQkhXVxc6Oh9ms7a2xpMnTwAA5ubmePr0qWbTERFRtgkhsG7dZdSsuQ5XroQDAKKiEnD2LD+ridKoPUaoZs2auHTpEsqVKwd3d3dMmzYNkZGR2LZtG6pUqZITGYmISE2vX8dj4MA/cOhQsNRWoUJxbN/eBbVq2WoxGVHeonaP0Ny5c2Fr++FNNGfOHFhYWGDo0KF4/fo11q1bp/GARESkHj+/B6hWba1KETR0aG1cuTKYRRDRR9TuEapdu7b0/9bW1jh+/LhGAxERUfYkJqZi0qQTWLbsgtRmaWmEjRs7oH37ClpMRpR3aeyKWVeuXEG7du00tTgiIlLTq1fx2LTpmvTY07Msbt4cyiKI6BPUKoT8/PwwYcIETJ48GaGhoQCAe/fuwcvLC3Xq1JFuw0FERLmvdGlzrFnTFvr6cqxY4YmjR3vCxsZE27GI8rQsHxr77bffMGjQIBQrVgxRUVH49ddfsWTJEowcORLe3t64desWKlWqlJNZiYjoP8LDY2FsrAczs3+vAdSjR1U0bFga9vbmWkxGlH9kuUdo+fLlmD9/PiIjI7Fr1y5ERkZi9erVuHnzJtauXcsiiIgoFx08eA/Vqq3FqFHH0j3HIogo67JcCD18+BBdu3YFAHTu3BlFihTBwoULUaoU70tDRJRb4uOTMWTIYXh57URk5Hts2XIde/fe0XYsonwry4fGEhISYGRkBODD/Wn09fWl0+iJiCjnBQW9QM+e+xAS8kZq8/KqCHd3B+2FIsrn1Dp9/tdff4WJyYeBd6mpqdi8eTMsLS1VpuFNV4mINEuhUGLRonOYOvUkUlM/nJRiZKSL5cs9MWBATchkMi0nJMq/ZEIIkZUJHRwcPvtmk8lk0tlkWbVq1SosXLgQERERqF69OlauXAlXV9dMp3/37h2mTJmCffv24e3btyhTpgyWLVuGNm3aZGl9MTExMDc3R/RSW5gpwgETO2DwM7UyExHllqdPo9Gnz36cOvVYanNxscX27V1QvnxxLSYjyl3S93d0NMzMzDS23Cz3CIWFhWlspWl27tyJcePGYe3atahbty6WLVsGDw8PBAcHw9raOt30ycnJaNmyJaytrbFnzx7Y2dnh8ePHKFq0qMazERFpW0jIG9St+yvevUsEAMhkwMSJDTFjRhPo6cm1nI6oYFD7ytKatGTJEgwaNAj9+/cHAKxduxZHjhzBxo0bMXHixHTTb9y4EW/fvsW5c+egq6sL4ENPFRFRQVS2bDHUrWsHP7+HsLc3w7ZtnTgeiEjDNHZlaXUlJycjKCgILVq0+DeMjg5atGiBwMDADOc5dOgQ6tevj+HDh6NEiRKoUqUK5s6dC4VCkVuxiYhyjY6ODJs2dcS339bC9etDWAQR5QCt9QhFRkZCoVCgRIkSKu0lSpTAvXv3MpwnNDQUf//9N3r16oWjR4/iwYMHGDZsGFJSUjB9+vQM50lKSkJSUpL0OCYmRnMvgohIQ1JTlZgz5x80alQGzZo5Su22tqZYt669FpMRFWxaPTSmLqVSCWtra6xfvx5yuRwuLi54/vw5Fi5cmGkhNG/ePMycOTOXkxIRZV1oaBR6996HwMBnsLMzxY0bQ1GsmKG2YxEVClo7NGZpaQm5XI6XL1+qtL98+RI2NjYZzmNra4vy5ctDLv93kGClSpUQERGB5OTkDOeZNGkSoqOjpX9Pnz7V3IsgIvoCQghs3XodNWqsRWDgh7NXIyLicPLkIy0nIyo8slUIPXz4EFOnTkWPHj3w6tUrAMCxY8dw+/btLC9DT08PLi4u8Pf3l9qUSiX8/f1Rv379DOdp0KABHjx4oHJz15CQENja2kJPTy/DefT19WFmZqbyj4hI26KiEtC9+17063cAsbEffsg5OVngzJlv0KVLZS2nIyo81C6ETp06hapVq+LChQvYt28f4uLiAADXr1/P9PBUZsaNG4cNGzZgy5YtuHv3LoYOHYr4+HjpLLK+ffti0qRJ0vRDhw7F27dvMXr0aISEhODIkSOYO3cuhg8fru7LICLSmoCAMFSrtha7dv3749HHpwauXRuMevV42yKi3KT2GKGJEydi9uzZGDduHExNTaX2Zs2a4ZdfflFrWd7e3nj9+jWmTZuGiIgI1KhRA8ePH5cGUD958gQ6Ov/Wavb29vDz88PYsWNRrVo12NnZYfTo0fjhhx/UfRlERLkuOVmB6dNPYv78s0i7lG3RogZYv74dunb9SrvhiAqpLF9ZOo2JiQlu3rwJR0dHmJqa4vr163ByckJYWBgqVqyIxMTEnMqqEbyyNBFpS2hoFKpVW4P4+BQAQJMmDti61Yt3iyfKgpy6srTah8aKFi2K8PDwdO1Xr16FnZ2dRkIRERVETk4WWL7cE7q6OliwoAX8/fuyCCLSMrUPjXXv3h0//PADdu/eDZlMBqVSibNnz2LChAno27dvTmQkIsqXIiPfw8hIF0ZGulLbN9/UhLu7A8qWLabFZESURu0eoblz56JixYqwt7dHXFwcKleujMaNG8PNzQ1Tp07NiYxERPmOn98DVK26Bt9996dKu0wmYxFElIeoPUYozZMnT3Dr1i3ExcWhZs2aKFeunKaz5QiOESKinJSYmIpJk05g2bILUtvhwz3Qtm15LaYiyv+0fvf5NGfOnEHDhg1RunRplC5dWmNBiIjyu5s3X6JXr324efOV1ObpWRYuLiW1mIqIPkXtQ2PNmjWDo6MjJk+ejDt37uREJiKifEWpFFi+/Dzq1NkgFUH6+nKsWOGJo0d7wsbGRMsJiSgzahdCL168wPjx43Hq1ClUqVIFNWrUwMKFC/HsGQ8vEVHhEx4eizZtfDFmjB+SkhQAgKpVrXH58rcYObIuZDKZlhMS0aeoXQhZWlpixIgROHv2LB4+fIiuXbtiy5YtcHBwQLNmzXIiIxFRnhQcHIlq1dbCz++h1DZ2bD1cvDgIVapYazEZEWXVF9101dHRERMnTsTPP/+MqlWr4tSpU5rKRUSU55UtWwyVK1sBAGxtTeDn1xtLlnjAwEDt4ZdEpCXZLoTOnj2LYcOGwdbWFj179kSVKlVw5MgRTWYjIsrT5HIdbNvWCX36VMONG0PRqpWztiMRkZrU/tkyadIk7NixAy9evEDLli2xfPlydOzYEUZGRjmRj4goT1AolFi06BwaNSoDNzd7qb10aXNs3dpJi8mI6EuoXQj9888/+O6779CtWzdYWlrmRCYiojzl6dNo9OmzH6dOPYajY1FcuzYEZmb62o5FRBqgdiF09uzZnMhBRJQn7dp1G4MHH8a7dx9uKB0W9g5//vkQX39dWcvJiEgTslQIHTp0CK1bt4auri4OHTr0yWk7dOigkWBERNoUE5OEUaOOYcuW61Kbvb0Ztm3rBHd3B+0FIyKNylIh5OXlhYiICFhbW8PLyyvT6WQyGRQKhaayERFpRWDgU/TuvR+hoVFSm7f3V1izpi0sLAy1mIyINC1LhZBSqczw/4mICpLUVCXmzPkHs2b9A4Xiw20YTU31sGpVG/TuXY0XRyQqgNQ+fX7r1q1ISkpK156cnIytW7dqJBQRkTY8fPgW8+adkYogNzd7XL8+BH36VGcRRFRAqV0I9e/fH9HR0enaY2Nj0b9/f42EIiLShgoVLLFgQUvI5TLMnNkEp075wNHRQtuxiCgHqX3WmBAiw19Gz549g7m5uUZCERHlhqioBBgZ6UJf/9+PwpEjXdGsmSNvkUFUSGS5EKpZsyZkMhlkMhmaN2+OIkX+nVWhUODRo0fw9PTMkZBERJoWEBCGPn32o3v3r7BwYSupXSaTsQgiKkSyXAilnS127do1eHh4wMTERHpOT08PDg4O6NKli8YDEhFpUnKyAtOnn8T8+WchBLBoUSA8PcuieXMnbUcjIi3IciE0ffp0AICDgwO8vb1hYGCQY6GIiHJCcHAkevbchytXwqW2pk0dUKECr5JPVFipPUaoX79+OZGDiCjHCCGwfn0Qxo71Q0JCKgBAV1cHc+Y0w/jxbtDR4RlhRIVVlgqhYsWKISQkBJaWlrCwsPjkaaRv377VWDgioi/1+nU8Bg78A4cOBUttFSoUx/btXVCrlq0WkxFRXpClQmjp0qUwNTWV/p/X0yCi/CA4OBJNmmxBRESc1DZ0aG0sWtQKRka6WkxGRHlFlgqh/x4O8/HxyaksREQa5eRkAXt7M0RExMHS0ggbN3ZA+/YVtB2LiPIQtS+oeOXKFdy8eVN6fPDgQXh5eWHy5MlITk7WaDgioi+hqyuHr29ndO5cCTdvDmURRETpqF0IDR48GCEhIQCA0NBQeHt7w8jICLt378b333+v8YBERFmhVAqsWHEBV6+Gq7SXK1cce/d2g42NSSZzElFhpnYhFBISgho1agAAdu/eDXd3d2zfvh2bN2/G3r17NZ2PiOizwsNj0aaNL0aPPo6ePffh/fsUbUcionxC7UJICCHdgf7EiRNo06YNAMDe3h6RkZGaTUdE9BkHD95DtWpr4ef3EABw714kjh27r+VURJRfqH0dodq1a2P27Nlo0aIFTp06hTVr1gAAHj16hBIlSmg8IBFRRuLjkzF+/J9Yty5IarO1NcHmzV5o1cpZi8mIKD9RuxBatmwZevXqhQMHDmDKlCkoW7YsAGDPnj1wc3PTeEAioo8FBb1Az577EBLyRmrz8qqIDRvaw9LSSIvJiCi/UbsQqlatmspZY2kWLlwIuVyukVBERBlRKJRYuPAcfvzxJFJTPxyiNzLSxbJlHhg4sBavcUZEalO7EEoTFBSEu3fvAgAqV66MWrVqaSwUEVFG7t2LVCmCXFxssX17F5QvX1zLyYgov1K7EHr16hW8vb1x6tQpFC1aFADw7t07NG3aFDt27ICVlZWmMxIRAQC++soas2Y1xeTJ/pg4sSFmzGgCPT32RBNR9ql91tjIkSMRFxeH27dv4+3bt3j79i1u3bqFmJgYjBo1KicyElEhFRubJPX+pPnuOzdcvDgIc+c2ZxFERF9M7ULo+PHjWL16NSpVqiS1Va5cGatWrcKxY8c0Go6ICq/AwKeoUWMdZs/+R6VdLtdB7doltZSKiAoatQshpVIJXd30NyvU1dWVri9ERJRdqalKzJwZgEaNNiE0NAqzZv2Dc+eeajsWERVQahdCzZo1w+jRo/HixQup7fnz5xg7diyaN2+u0XBEVLiEhkahceNNmDHjFBQKAQCoV68UbG15ewwiyhlqF0K//PILYmJi4ODgAGdnZzg7O8PR0RExMTFYuXJlTmQkogJOCIGtW6+jRo21CAx8BgCQy2WYObMJTp3ygaOjhXYDElGBpfZZY/b29rhy5Qr8/f2l0+crVaqEFi1aaDwcERV8UVEJGDr0CHbuvC21OTlZwNe3M+rVK6XFZERUGKhVCO3cuROHDh1CcnIymjdvjpEjR+ZULiIqBIKDI9Gy5TY8fRojtfn41MCKFZ4wNdXXYjIiKiyyXAitWbMGw4cPR7ly5WBoaIh9+/bh4cOHWLhwYU7mI6ICrEyZoiha1ABPn8bAwsIA69a1Q9euX2k7FhEVIlkeI/TLL79g+vTpCA4OxrVr17BlyxasXr06J7MRUQFnYFAE27d3QZs25XDjxlAWQUSU67JcCIWGhqJfv37S4549eyI1NRXh4eE5EoyIChYhBNavD8KdO69V2qtUscaRIz1RqpSZlpIRUWGW5UIoKSkJxsbG/86oowM9PT0kJCTkSDAiKjhev46Hl9dODB58GD177kVSUqq2IxERAVBzsPSPP/4IIyMj6XFycjLmzJkDc3NzqW3JkiWaS0dE+Z6f3wP4+BxEREQcAOD69Zc4fDgEXbpU1nIyIiI1CqHGjRsjODhYpc3NzQ2hoaHSY5lMprlkRJSvJSamYuLEE1i+/ILUZmlphI0bO6B9+wpaTEZE9K8sF0IBAQE5GIOICpKbN1+iZ899uHXrldTm4eGMzZu9YGPDq0QTUd6h9gUViYgyo1QKrFx5AT/8cAJJSQoAgL6+HAsWtMSIEa7Q0WGvMRHlLSyEiEhjbt58iXHj/oRS+eE+YVWrWmP79i6oUsVay8mIiDKm9r3GiIgyU726DSZPbggAGDu2Hi5eHMQiiIjyNPYIEVG2vX+fAgODIiqHvKZNc0erVs5o1KiMFpMREWUNe4SIKFuCgl6gZs11WLz4nEq7rq6cRRAR5RvZKoROnz6N3r17o379+nj+/DkAYNu2bThz5oxGwxFR3qNQKDF//hnUq/cbQkLeYMqUv3HlCq8wT0T5k9qF0N69e+Hh4QFDQ0NcvXoVSUlJAIDo6GjMnTtX4wGJKO94+jQazZtvxcSJ/khNVQIAqlUrARMTPS0nIyLKHrULodmzZ2Pt2rXYsGEDdHV1pfYGDRrgypUrGg1HRHnHrl23Ua3aWpw69RgAIJMBkyY1xLlzA1C+fHEtpyMiyh61B0sHBwejcePG6drNzc3x7t07TWQiojwkJiYJo0Ydw5Yt16U2e3szbNvWCe7uDtoLRkSkAWoXQjY2Nnjw4AEcHBxU2s+cOQMnJydN5SKiPCA4OBJt2mxHaGiU1Obt/RXWrm2HokUNtJiMiEgz1D40NmjQIIwePRoXLlyATCbDixcv4OvriwkTJmDo0KE5kZGItKRUKTMUKfLhY8LUVA9bt3rhf//rwiKIiAoMtQuhiRMnomfPnmjevDni4uLQuHFjDBw4EIMHD8bIkSOzFWLVqlVwcHCAgYEB6tati4sXL2Zpvh07dkAmk8HLyytb6yWiTzM21sP27Z3RpIkDrl8fgj59qvPmykRUoMiEECI7MyYnJ+PBgweIi4tD5cqVYWKSvRsp7ty5E3379sXatWtRt25dLFu2DLt370ZwcDCsrTO/Im1YWBgaNmwIJycnFCtWDAcOHMjS+mJiYmBubo7opbYwU4QDJnbA4GfZyk5UkAghsG3bDTRoYA9n52LpnmMBRETaJH1/R0fDzMxMY8vN9gUV9fT0ULlyZbi6uma7CAKAJUuWYNCgQejfvz8qV66MtWvXwsjICBs3bsx0HoVCgV69emHmzJkcl0SkAVFRCejefS/69TuAXr32ISVFofI8iyAiKqjUHizdtGnTT34o/v3331leVnJyMoKCgjBp0iSpTUdHBy1atEBgYGCm8/3000+wtrbGgAEDcPr06U+uIykpSbrWEfChoiSifwUEhKFPn/149uzDe+PChec4fDgEnTpV0nIyIqKcp3YhVKNGDZXHKSkpuHbtGm7duoV+/fqptazIyEgoFAqUKFFCpb1EiRK4d+9ehvOcOXMGv/32G65du5aldcybNw8zZ85UKxdRYZCcrMC0aSexYMFZpB0gt7AwwPr17VkEEVGhoXYhtHTp0gzbZ8yYgbi4uC8O9CmxsbHo06cPNmzYAEtLyyzNM2nSJIwbN056HBMTA3t7+5yKSJQvBAdHomfPfSq3xmja1AFbt3ZCqVKaO/ZORJTXaezu871794arqysWLVqU5XksLS0hl8vx8uVLlfaXL1/CxsYm3fQPHz5EWFgY2rdvL7UplR8u81+kSBEEBwfD2dlZZR59fX3o6+ur81KICiwhBNavD8LYsX5ISEgFAOjq6mDOnGYYP95N5S7yRESFgcYKocDAQBgYqHdtET09Pbi4uMDf3186BV6pVMLf3x8jRoxIN33FihVx8+ZNlbapU6ciNjYWy5cvZ08P0WdcvRqBIUOOSI8rVCiO7du7oFYtWy2mIiLSHrULoc6dO6s8FkIgPDwcly9fxo8//qh2gHHjxqFfv36oXbs2XF1dsWzZMsTHx6N///4AgL59+8LOzg7z5s2DgYEBqlSpojJ/0aJFASBdOxGlV6uWLcaNq4clS85j6NDaWLSoFYyMdD8/IxFRAaV2IWRubq7yWEdHBxUqVMBPP/2EVq1aqR3A29sbr1+/xrRp0xAREYEaNWrg+PHj0gDqJ0+eQEcn22f5ExVqSUmp0NOTq5zpOXduc3h6lkXLls6fmJOIqHBQ64KKCoUCZ8+eRdWqVWFhYZGTuXIML6hIhcXNmy/Rs+c+DB1aG8OG1dF2HCKiL5InLqgol8vRqlUr3mWeKA9TKgWWLz+POnU24NatVxg//k/cufNa27GIiPIktQ+NValSBaGhoXB0dMyJPET0BcLDY9G//0H4+T2U2sqVK/aJOYiICje1B9/Mnj0bEyZMwOHDhxEeHo6YmBiVf0SkHQcP3kO1amtViqCxY+vh4sVBqFzZSovJiIjyriz3CP30008YP3482rRpAwDo0KGDygDMtJsyKhSKzBZBRDkgPj4Z48f/iXXrgqQ2W1sTbN7shVatOCCaiOhTslwIzZw5E0OGDMHJkydzMg8RqSEk5A3at/8fQkLeSG1eXhWxYUN7WFoaaTEZEVH+kOVCKO3kMnd39xwLQ0TqKVHCGMnJH3phjYx0sXy5JwYMqMm7xRMRZZFaY4T44UqUt5ibG+D33zuhbl07XL06GAMH1uL7lIhIDWqdNVa+fPnPfsi+ffv2iwIRUeZ2776NevVKwd7+3wubNmhQGoGBA1gAERFlg1qF0MyZM9NdWZqIcl5MTBJGjTqGLVuuo0kTB5w40Qdy+b8duiyCiIiyR61CqHv37rC2ts6pLESUgcDAp+jdez9CQ6MAAAEBYTh8OAQdO1bUcjIiovwvy2OE+IuTKHelpioxc2YAGjXaJBVBpqZ62LrVCx06VNByOiKigkHts8aIKOeFhkahd+99CAz89z54bm72+P33TnB0zJ/3+SMiyouyXAgplcqczEFE+PCDY9u2Gxgx4ihiY5MBAHK5DNOmuWPy5EYoUkTti8ETEdEnqH2vMSLKOZcvv0C/fgekx05OFvD17Yx69UppLxQRUQHGn5dEeUidOnYYPNgFAODjUwPXrg1mEURElIPYI0SkRSkpChQpoqNyMsLixa3Qpk05DogmIsoF7BEi0pLg4EjUq/cbtmy5rtJubKzHIoiIKJewECLKZUIIrFt3GTVrrsOVK+EYOfIYHjzgFdmJiLSBh8aIctHr1/EYOPAPHDoULLXZ2ZkiISFFi6mIiAovFkJEucTP7wF8fA4iIiJOahsyxAWLF3vAyEhXi8mIiAovFkJEOSwxMRWTJp3AsmUXpDZLSyNs3NgB7dtzLBARkTaxECLKQQ8evEXnzjtx8+Yrqc3Tsyw2beoIGxsTLSYjIiKAhRBRjrKwMMCbNwkAAH19ORYubIkRI1x57z4iojyCZ40R5aDixY2weXNHVK9eApcvf4uRI+uyCCIiykPYI0SkQX/8EYw6dexUDnu1bOmMoCBHyOX83UFElNfwk5lIA+LjkzFkyGF06LAD33xzEEIIledZBBER5U38dCb6QkFBL1Cr1nqsWxcEADh27AEOHw7RcioiIsoKFkJE2aRQKDF//hnUq/cbQkLeAACMjHSxYUN7tGtXXsvpiIgoKzhGiCgbnj6NRp8++3Hq1GOpzcXFFtu3d0H58sW1mIyIiNTBQohITTt33sKQIUfw7l0iAEAmAyZObIgZM5pAT0+u5XRERKQOFkJEajh//hm6d98rPba3N8O2bZ3g7u6gvVBERJRtHCNEpIZ69UqhT59qAABv769w/foQFkFERPkYe4SIPkGpFNDRUb0A4i+/tEHbtuXQrdtXvDgiEVE+xx4hokyEhkahYcON2LXrtkq7mZk+vL2rsAgiIioA2CNE9BEhBLZtu4ERI44iNjYZd+8eRv36pWBvb67taEREpGHsESL6j6ioBHTvvhf9+h1AbGwyAKBYMUPpxqlERFSwsEeI6P8FBIShT5/9ePYsRmrz8amBFSs8YWqqr8VkRESUU1gIUaGXnKzAtGknsWDBWaTdIqxoUQOsX98OXbt+pd1wRESUo1gIUaEWGhqFrl1348qVcKmtSRMHbN3qxTFBRESFAMcIUaFmaFgET55EAwB0dXWwYEEL+Pv3ZRFERFRIsBCiQs3W1hS//dYBFSta4vz5gfjuuwbprhtEREQFFw+NUaFy4kQoata0QfHiRlJbhw4V0Lp1Wejq8j5hRESFDXuEqFBITEzF2LHH0bLlNgwefBgibVT0/2MRRERUOLEQogLv5s2XcHXdgGXLLgAA9u69i+PHH2g5FRER5QUshKjAUioFli8/jzp1NuDmzVcAAH19OVas8ISnZ1ktpyMioryAY4SoQAoPj0X//gfh5/dQaqta1Rrbt3dBlSrWWkxGRER5CQshKnAOHQrGgAGHEBn5XmobO7Ye5s5tDgMD/skTEdG/+K1ABcrZs0/QseMO6bGNjQm2bPFCq1bOWkxFRER5FccIUYHi5maPTp0qAgA6dqyAmzeHsggiIqJMsUeI8jUhBGSyfy+AKJPJsGFDe3ToUAH9+lVXeY6IiOhj7BGifOvp02g0a7YVhw+HqLQXL24EH58aLIKIiOiz2CNE+dKuXbcxePBhvHuXiNu3X+HGjaGwsTHRdiwiIspn2CNE+UpMTBJ8fA7A23sP3r1LBAAYGBTBixexWk5GRET5EXuEKN8IDHyKXr324dGjd1Kbt/dXWLOmLSwsDLUXjIiI8i0WQpTnpaYqMXv2P5g9+x8oFB/uEWZqqodVq9qgd+9qHAtERETZxkKI8rSwsHfo2XMvAgOfSW1ubvb4/fdOcHS00GIyIiIqCDhGiPI0HR0Z7tx5DQCQy2WYObMJTp3yYRFEREQawUKI8rTSpc2xdm07ODlZ4MyZbzBtmjuKFOGfLRERaQa/UShPOX36MWJiklTaunevgtu3h6FevVJaSkVERAVVniiEVq1aBQcHBxgYGKBu3bq4ePFiptNu2LABjRo1goWFBSwsLNCiRYtPTk/5Q3KyAhMnnoC7+2aMHHks3fO8WSoREeUErRdCO3fuxLhx4zB9+nRcuXIF1atXh4eHB169epXh9AEBAejRowdOnjyJwMBA2Nvbo1WrVnj+/HkuJydNCQ6ORP36v2H+/LMQAti69Tr+/POhtmMREVEhIBNCCG0GqFu3LurUqYNffvkFAKBUKmFvb4+RI0di4sSJn51foVDAwsICv/zyC/r27fvZ6WNiYmBubo7opbYwU4QDJnbA4GefnY80TwiB9euDMHasHxISUgEAuro6mDOnGcaPd4OODk+LJyKiD6Tv7+homJmZaWy5Wj3ekJycjKCgIEyaNElq09HRQYsWLRAYGJilZbx//x4pKSkoVqxYhs8nJSUhKenfMScxMTFfFpo04vXreAwc+AcOHQqW2ipUKI7t27ugVi1bLSYjIqLCRKuHxiIjI6FQKFCiRAmV9hIlSiAiIiJLy/jhhx9QsmRJtGjRIsPn582bB3Nzc+mfvb39F+emL+Pn9wDVqq1VKYKGDq2NK1cGswgiIqJcpfUxQl/i559/xo4dO7B//34YGBhkOM2kSZMQHR0t/Xv69Gkup6T/On36MTw9fREREQcAsLQ0wqFD3bF6dVsYGelqOR0RERU2Wj00ZmlpCblcjpcvX6q0v3z5EjY2Np+cd9GiRfj5559x4sQJVKtWLdPp9PX1oa+vr5G89OUaNiwNT8+yOH78ATw9y2LTpo68azwREWmNVnuE9PT04OLiAn9/f6lNqVTC398f9evXz3S+BQsWYNasWTh+/Dhq166dG1FJQ2QyGTZt6ojVq9vg6NGeLIKIiEirtH5obNy4cdiwYQO2bNmCu3fvYujQoYiPj0f//v0BAH379lUZTD1//nz8+OOP2LhxIxwcHBAREYGIiAjExcVp6yVQJiIi4tC27Xb4+4eqtNvYmGDo0Dq8WSoREWmd1q9S5+3tjdevX2PatGmIiIhAjRo1cPz4cWkA9ZMnT6Cj82+9tmbNGiQnJ+Prr79WWc706dMxY8aM3IxOn3DoUDAGDDiEyMj3uH49AtevD0Hx4kbajkVERKRC69cRym28jlDOio9Pxvjxf2LduiCpzdbWBH/80QMuLiW1mIyIiPKzAnkdISpYgoJeoFevfQgOfiO1eXlVxIYN7WFpyd4gIiLKe1gI0RdTKJRYtOgcpk49idRUJQDAyEgXy5d7YsCAmhwLREREeRYLIfoiz57FoE+f/QgICJPaXFxssX17F5QvX1x7wYiIiLJA62eNUf6WkJCCS5c+3PBWJgMmTWqIc+cGsAgiIqJ8gYUQfZFy5YpjxYrWsLc3w8mT/TB3bnPo6cm1HYuIiChLWAiRWi5efI7371NU2vr3r4E7d4bD3d1BO6GIiIiyiYUQZUlqqhIzZwbAze03TJjwp8pzMpkMJiZ6WkpGRESUfSyE6LNCQ6PQuPEmzJhxCgqFwJo1l3Hy5CNtxyIiIvpiPGuMMiWEwLZtNzBixFHExiYDAORyGaZNc0ejRmW0nI6IiOjLsRCiDEVFJWDo0CPYufO21ObkZAFf386oV6+UFpMRERFpDgshSufUqTD06bMfT5/GSG0+PjWwYoUnTE31tZiMiIhIs1gIkYpTp8LQtOkWpN2BzsLCAOvWtUPXrl9pNxgREVEO4GBpUtGwYWk0bvxh/E/Tpg64cWMoiyAiIiqw2CNEKuRyHWzb1gm7d9/BmDH1oKPD+4QREVHBxR6hQuz163h06bILZ88+UWm3tzfHuHH1WQQREVGBxx6hQsrP7wF8fA4iIiIOV66E4/r1ITAz40BoIiIqXNgjVMgkJqZizJjj8PT0RUREHAAgLi4ZISFvtJyMiIgo97FHqBC5efMlevbch1u3Xkltnp5lsWlTR9jYmGgxGRERkXawECoElEqBlSsv4IcfTiApSQEA0NeXY+HClhgxwhUyGccCERFR4cRCqIALD49F//4H4ef3UGqrWtUa27d3QZUq1lpMRkREpH0cI1TAvX2bgICAMOnx2LH1cPHiIBZBREREYCFU4H31lTUWLmwJGxsT+Pn1xpIlHjAwYEcgERERwEKowLl+PQJJSakqbSNGuOLOnWFo1cpZS6mIiIjyJhZCBYRCocT8+WdQu/YGTJnyt8pzMpkMFhaGWkpGRESUd7EQKgCePo1G8+ZbMXGiP1JTlVi8OBBnzjz5/IxERESFHAeL5HO7dt3G4MGH8e5dIgBAJgMmTmwIV1c7LScjIiLK+1gI5VMxMUkYNeoYtmy5LrXZ25th27ZOcHd30F4wIiKifISFUD4UGPgUvXvvR2holNTm7f0V1qxpy7FAREREamAhlM8EBIShRYutUCgEAMDUVA+rVrVB797VeIVoIiIiNXGwdD7ToIE9XFxKAgDc3Oxx/foQ9OlTnUUQERFRNrBHKJ/R1ZXD17czdu68hR9+aIgiRVjLEhERZRcLoTwsKioBI0Ycw7hx9aReIAAoW7YYpkxprMVkRPmDEAKpqalQKBTajkJEWaCrqwu5XJ6r62QhlEcFBIShT5/9ePYsBkFBL3DlymAYGelqOxZRvpGcnIzw8HC8f/9e21GIKItkMhlKlSoFExOTXFsnC6E8JjlZgWnTTmLBgrMQH8ZD49WreNy+/Qp16vDaQERZoVQq8ejRI8jlcpQsWRJ6enocR0eUxwkh8Pr1azx79gzlypXLtZ4hFkJ5SHBwJHr23IcrV8KltqZNHbB1ayeUKmWmxWRE+UtycjKUSiXs7e1hZGSk7ThElEVWVlYICwtDSkoKC6HCRAiB9euDMHasHxISPtwwVVdXB3PmNMP48W7Q0eEvWaLs0NHhyQRE+Yk2em5ZCGnZ69fxGDjwDxw6FCy1VahQHNu3d0GtWrZaTEZERFTwsRDSsqdPY3D06H3p8dChtbFoUSsOjCYiIsoF7DfWslq1bDF7dlNYWhrh0KHuWL26LYsgIqJsCA4Oho2NDWJjY7UdhTIQGRkJa2trPHv2TNtRVLAQymX37kUiJUX1miYTJrjh9u1haN++gpZSEVFe4ePjA5lMBplMBl1dXTg6OuL7779HYmJiumkPHz4Md3d3mJqawsjICHXq1MHmzZszXO7evXvRpEkTmJubw8TEBNWqVcNPP/2Et2/f5vAryj2TJk3CyJEjYWpqqu0oOWbVqlVwcHCAgYEB6tati4sXL35y+pSUFPz0009wdnaGgYEBqlevjuPHj6tMExsbizFjxqBMmTIwNDSEm5sbLl26lG5Zd+/eRYcOHWBubg5jY2PUqVMHT548UZkmMDAQzZo1g7GxMczMzNC4cWMkJCQAACwtLdG3b19Mnz79C7eCZrEQyiVKpcDy5edRo8ZazJ79j8pzcrkOrK2NtZSMiPIaT09PhIeHIzQ0FEuXLsW6devSfXmsXLkSHTt2RIMGDXDhwgXcuHED3bt3x5AhQzBhwgSVaadMmQJvb2/UqVMHx44dw61bt7B48WJcv34d27Zty7XXlZycnGPLfvLkCQ4fPgwfH58vWk5OZvxSO3fuxLhx4zB9+nRcuXIF1atXh4eHB169epXpPFOnTsW6deuwcuVK3LlzB0OGDEGnTp1w9epVaZqBAwfir7/+wrZt23Dz5k20atUKLVq0wPPnz6VpHj58iIYNG6JixYoICAjAjRs38OOPP8LAwECaJjAwEJ6enmjVqhUuXryIS5cuYcSIESonLfTv3x++vr55qwAXhUx0dLQAIKKX2gqxCEKstcvxdb54ESM8PLYJYIYAZggdnZniwoVnOb5eosIqISFB3LlzRyQkJGg7itr69esnOnbsqNLWuXNnUbNmTenxkydPhK6urhg3bly6+VesWCEAiPPnzwshhLhw4YIAIJYtW5bh+qKiojLN8vTpU9G9e3dhYWEhjIyMhIuLi7TcjHKOHj1auLu7S4/d3d3F8OHDxejRo0Xx4sVFkyZNRI8ePUS3bt1U5ktOThbFixcXW7ZsEUIIoVAoxNy5c4WDg4MwMDAQ1apVE7t37840pxBCLFy4UNSuXVulLTIyUnTv3l2ULFlSGBoaiipVqojt27erTJNRRiGEuHnzpvD09BTGxsbC2tpa9O7dW7x+/Vqa79ixY6JBgwbC3NxcFCtWTLRt21Y8ePDgkxm/lKurqxg+fLj0WKFQiJIlS4p58+ZlOo+tra345ZdfVNo6d+4sevXqJYQQ4v3790Iul4vDhw+rTFOrVi0xZcoU6bG3t7fo3bv3J/PVrVtXTJ069bOvw9HRUfz6668ZPvep9670/R0d/dl1qIODpXPYwYP3MHDgH4iM/PfqtqNGuaJatRJaTEVUSP1eG4iPyP31GtsAvS9na9Zbt27h3LlzKFOmjNS2Z88epKSkpOv5AYDBgwdj8uTJ+N///oe6devC19cXJiYmGDZsWIbLL1q0aIbtcXFxcHd3h52dHQ4dOgQbGxtcuXIFSqVSrfxbtmzB0KFDcfbsWQDAgwcP0LVrV8TFxUlXD/bz88P79+/RqVMnAMC8efPw+++/Y+3atShXrhz++ecf9O7dG1ZWVnB3d89wPadPn0bt2rVV2hITE+Hi4oIffvgBZmZmOHLkCPr06QNnZ2e4urpmmvHdu3do1qwZBg4ciKVLlyIhIQE//PADunXrhr///hsAEB8fj3HjxqFatWqIi4vDtGnT0KlTJ1y7di3TyzbMnTsXc+fO/eT2unPnDkqXLp2uPTk5GUFBQZg0aZLUpqOjgxYtWiAwMDDT5SUlJan02gCAoaEhzpw5AwDSLWg+NY1SqcSRI0fw/fffw8PDA1evXoWjoyMmTZoELy8vAMCrV69w4cIF9OrVC25ubnj48CEqVqyIOXPmoGHDhirLdnV1xenTpzFgwIBPbovcwkIoh8THJ2P8+D+xbl2Q1GZjY4ItW7zQqpWzFpMRFWLxEUDc889Pp2WHDx+GiYkJUlNTkZSUBB0dHfzyyy/S8yEhITA3N4etbfpLbOjp6cHJyQkhISEAgPv378PJyQm6uuqdhLF9+3a8fv0aly5dQrFixQAAZcuWVfu1lCtXDgsWLJAeOzs7w9jYGPv370efPn2kdXXo0AGmpqZISkrC3LlzceLECdSvXx8A4OTkhDNnzmDdunWZFkKPHz9OVwjZ2dmpFIsjR46En58fdu3apVIIfZxx9uzZqFmzpkrRsnHjRtjb2yMkJATly5dHly5dVNa1ceNGWFlZ4c6dO6hSpUqGGYcMGYJu3bp9cnuVLFkyw/bIyEgoFAqUKKH6I7pEiRK4d+9epsvz8PDAkiVL0LhxYzg7O8Pf3x/79u2T7r9namqK+vXrY9asWahUqRJKlCiB//3vfwgMDJT296tXrxAXF4eff/4Zs2fPxvz583H8+HF07twZJ0+ehLu7O0JDQwEAM2bMwKJFi1CjRg1s3boVzZs3x61bt1CuXDmV1/jfQ3PaxkIoBwQFvUDPnvsQEvJGauvYsQJ+/bUDLC15lVsirTG2yRfrbdq0KdasWYP4+HgsXboURYoUSffFm1Ui7V49arp27Rpq1qwpFUHZ5eLiovK4SJEi6NatG3x9fdGnTx/Ex8fj4MGD2LFjB4APPUbv379Hy5YtVeZLTk5GzZo1M11PQkJCul4NhUKBuXPnYteuXXj+/DmSk5ORlJSU7mrjH2e8fv06Tp48meH9rh4+fIjy5cvj/v37mDZtGi5cuIDIyEipp+zJkyeZFkLFihX74u2pruXLl2PQoEGoWLEiZDIZnJ2d0b9/f2zcuFGaZtu2bfjmm29gZ2cHuVyOWrVqoUePHggK+vBDPu21dezYEWPHjgUA1KhRA+fOncPatWvh7u4uTTN48GD0798fAFCzZk34+/tj48aNmDdvnrQ+Q0PDPHUPQBZCGvb334/g4fE7UlM//FEYGeli2TIPDBxYi/c6ItK2bB6eym3GxsbSr/GNGzeievXq+O2336RDCeXLl0d0dDRevHiRrgchOTkZDx8+RNOmTaVpz5w5g5SUFLV6hQwNDT/5vI6OTroiKyUlJcPX8rFevXrB3d0dr169wl9//QVDQ0N4enoC+HBIDgCOHDkCOzvV+yvq6+tnmsfS0hJRUVEqbQsXLsTy5cuxbNkyVK1aFcbGxhgzZky6AdEfZ4yLi0P79u0xf/78dOtJ64Vr3749ypQpgw0bNqBkyZJQKpWoUqXKJwdbf8mhMUtLS8jlcrx8+VKl/eXLl7CxybzQtrKywoEDB5CYmIg3b96gZMmSmDhxIpycnKRpnJ2dcerUKcTHxyMmJga2trbw9vaWprG0tESRIkVQuXJllWVXqlRJOnyWtl0ymubjM8vevn0LKyurT26H3MSzxjSsQQN7VK78YQe7uNji6tXBGDTIhUUQEWWLjo4OJk+ejKlTp0qnIXfp0gW6urpYvHhxuunXrl2L+Ph49OjRAwDQs2dPxMXFYfXq1Rku/927dxm2V6tWDdeuXcv07B4rKyuEh4ertF27di1Lr8nNzQ329vbYuXMnfH190bVrV6lIq1y5MvT19fHkyROULVtW5Z+9vX2my6xZsybu3Lmj0nb27Fl07NgRvXv3RvXq1VUOGX5KrVq1cPv2bTg4OKTLYGxsjDdv3iA4OBhTp05F8+bNUalSpXRFWEaGDBmCa9euffJfZofG9PT04OLiAn9/f6lNqVTC399fOoT4KQYGBrCzs0Nqair27t2Ljh07ppvG2NgYtra2iIqKgp+fnzSNnp4e6tSpg+DgYJXpQ0JCpLFrDg4OKFmy5CenSXPr1q1P9u7lOo0Ovc4HcuOssVu3XoopU/xFUlKqxpdNRJ9X0M4aS0lJEXZ2dmLhwoVS29KlS4WOjo6YPHmyuHv3rnjw4IFYvHix0NfXF+PHj1eZ//vvvxdyuVx899134ty5cyIsLEycOHFCfP3115meTZaUlCTKly8vGjVqJM6cOSMePnwo9uzZI86dOyeEEOL48eNCJpOJLVu2iJCQEDFt2jRhZmaW7qyx0aNHZ7j8KVOmiMqVK4siRYqI06dPp3uuePHiYvPmzeLBgwciKChIrFixQmzevDnT7Xbo0CFhbW0tUlP//dwdO3assLe3F2fPnhV37twRAwcOFGZmZirbN6OMz58/F1ZWVuLrr78WFy9eFA8ePBDHjx8XPj4+IjU1VSgUClG8eHHRu3dvcf/+feHv7y/q1KkjAIj9+/dnmvFL7dixQ+jr64vNmzeLO3fuiG+//VYULVpURERESNP06dNHTJw4UXp8/vx5sXfvXvHw4UPxzz//iGbNmglHR0eVswWPHz8ujh07JkJDQ8Wff/4pqlevLurWrSuSk5Olafbt2yd0dXXF+vXrxf3798XKlSuFXC5X2XdLly4VZmZmYvfu3eL+/fti6tSpwsDAQOVsuvj4eGFoaCj++eefDF+jNs4aYyH0BYVQdHSiGDjwoLh166UGExLRlypohZAQQsybN09YWVmJuLg4qe3gwYOiUaNGwtjYWBgYGAgXFxexcePGDJe7c+dO0bhxY2FqaiqMjY1FtWrVxE8//fTJ0+fDwsJEly5dhJmZmTAyMhK1a9cWFy5ckJ6fNm2aKFGihDA3Nxdjx44VI0aMyHIhdOfOHQFAlClTRiiVSpXnlEqlWLZsmahQoYLQ1dUVVlZWwsPDQ5w6dSrTrCkpKaJkyZLi+PHjUtubN29Ex44dhYmJibC2thZTp04Vffv2/WwhJIQQISEholOnTqJo0aLC0NBQVKxYUYwZM0bK+tdff4lKlSoJfX19Ua1aNREQEJDjhZAQQqxcuVKULl1a6OnpCVdXV+lyBv99Pf369ZMeBwQESDmLFy8u+vTpI54/f64yz86dO4WTk5PQ09MTNjY2Yvjw4eLdu3fp1v3bb7+JsmXLCgMDA1G9enVx4MCBdNPMmzdPlCpVShgZGYn69eunK3K3b98uKlSokOnr00YhJBMimyPp8qmYmBiYm5sjeqktzBThgIkdMFj9y30HBj5F7977ERoahWrVSuDixYHQ1+eQK6K8IDExEY8ePYKjo2O6AbRUcK1atQqHDh2Cn5+ftqNQJurVq4dRo0ahZ8+eGT7/qfeu9P0dHQ0zMzONZeIYITWlpioxc2YAGjXahNDQD8eEHz2Kwo0bLz8zJxER5aTBgwejcePGvNdYHhUZGYnOnTtL49fyCnZhqCE0NAq9e+9DYOC/PUhubvb4/fdOcHS00GIyIiIqUqQIpkyZou0YlAlLS0t8//332o6RDguhLBBCYNu2Gxgx4ihiYz+cGimXyzBtmjsmT26EIkXYsUZERJQfsRD6jKioBAwdegQ7d96W2pycLODr2xn16pXSYjIiIiL6UiyEPuPu3Ujs3v3vtSl8fGpgxQpPmJpmfmEvIsobCtm5IET5njbeszym8xlubvaYMqURihY1wK5dX2PTpo4sgojyuLSL8+Wly/gT0eelXZlbLpfn2jrZI/SRR4+iULq0OeTyf2vEH39sjMGDXWBnp7nT9Ygo58jlchQtWhSvXr0CABgZGfHq7kR5nFKpxOvXr2FkZIQiRXKvPGEh9P+EEFi/Pghjx/ph+nR3/PBDQ+k5XV05iyCifCbt/ktpxRAR5X06OjooXbp0rv5wYSEE4PXreAwc+AcOHfpwj5SpU0+iVStn1Kxpq+VkRJRdMpkMtra2sLa2zvBmoESU9+jp6UFHJ3dH7eSJQmjVqlVYuHAhIiIiUL16daxcuRKurq6ZTr979278+OOPCAsLQ7ly5TB//ny0adMmW+v2u10KPjPWIiIiTmobOLAmKlSwzNbyiChvkcvluTregIjyF60Plt65cyfGjRuH6dOn48qVK6hevTo8PDwy7c4+d+4cevTogQEDBuDq1avw8vKCl5cXbt26pdZ6E1PkGHPQE54rW0tFkKWlEQ4d6o41a9rByEj3i18bERER5W1av9dY3bp1UadOHfzyyy8APgyWsre3x8iRIzFx4sR003t7eyM+Ph6HDx+W2urVq4caNWpg7dq1n11f2r1KKtkMxt2Ifw99eXqWxaZNHWFjY6KBV0VERESaVCDvNZacnIygoCC0aNFCatPR0UGLFi0QGBiY4TyBgYEq0wOAh4dHptNn5m7Eh1ti6BdJxYoVnjh6tCeLICIiokJGq2OEIiMjoVAoUKJECZX2EiVK4N69exnOExERkeH0ERERGU6flJSEpKQk6XF0dHTaM6hc4jV+63cOlft9x5v0ERER5WExMTEANH/RxTwxWDonzZs3DzNnzszgmaW48xKovwDAAvPcjkVERETZ8ObNG5iba+57W6uFkKWlJeRyOV6+fKnS/vLlS+kaIB+zsbFRa/pJkyZh3Lhx0uN3796hTJkyePLkiUY3JKkvJiYG9vb2ePr0qUaP91L2cH/kHdwXeQf3Rd4RHR2N0qVLo1ixYhpdrlYLIT09Pbi4uMDf3x9eXl4APgyW9vf3x4gRIzKcp379+vD398eYMWOktr/++gv169fPcHp9fX3o66e/JYa5uTn/qPMIMzMz7os8hPsj7+C+yDu4L/IOTV9nSOuHxsaNG4d+/fqhdu3acHV1xbJlyxAfH4/+/fsDAPr27Qs7OzvMmzcPADB69Gi4u7tj8eLFaNu2LXbs2IHLly9j/fr12nwZRERElA9pvRDy9vbG69evMW3aNERERKBGjRo4fvy4NCD6yZMnKtWfm5sbtm/fjqlTp2Ly5MkoV64cDhw4gCpVqmjrJRAREVE+pfVCCABGjBiR6aGwgICAdG1du3ZF165ds7UufX19TJ8+PcPDZZS7uC/yFu6PvIP7Iu/gvsg7cmpfaP2CikRERETaovVbbBARERFpCwshIiIiKrRYCBEREVGhxUKIiIiICq0CWQitWrUKDg4OMDAwQN26dXHx4sVPTr97925UrFgRBgYGqFq1Ko4ePZpLSQs+dfbFhg0b0KhRI1hYWMDCwgItWrT47L4j9aj73kizY8cOyGQy6cKn9OXU3Rfv3r3D8OHDYWtrC319fZQvX56fVRqi7r5YtmwZKlSoAENDQ9jb22Ps2LFITEzMpbQF1z///IP27dujZMmSkMlkOHDgwGfnCQgIQK1ataCvr4+yZcti8+bN6q9YFDA7duwQenp6YuPGjeL27dti0KBBomjRouLly5cZTn/27Fkhl8vFggULxJ07d8TUqVOFrq6uuHnzZi4nL3jU3Rc9e/YUq1atElevXhV3794VPj4+wtzcXDx79iyXkxdM6u6PNI8ePRJ2dnaiUaNGomPHjrkTtoBTd18kJSWJ2rVrizZt2ogzZ86IR48eiYCAAHHt2rVcTl7wqLsvfH19hb6+vvD19RWPHj0Sfn5+wtbWVowdOzaXkxc8R48eFVOmTBH79u0TAMT+/fs/OX1oaKgwMjIS48aNE3fu3BErV64UcrlcHD9+XK31FrhCyNXVVQwfPlx6rFAoRMmSJcW8efMynL5bt26ibdu2Km1169YVgwcPztGchYG6++JjqampwtTUVGzZsiWnIhYq2dkfqampws3NTfz666+iX79+LIQ0RN19sWbNGuHk5CSSk5NzK2Khoe6+GD58uGjWrJlK27hx40SDBg1yNGdhk5VC6PvvvxdfffWVSpu3t7fw8PBQa10F6tBYcnIygoKC0KJFC6lNR0cHLVq0QGBgYIbzBAYGqkwPAB4eHplOT1mTnX3xsffv3yMlJUXjN9grjLK7P3766SdYW1tjwIABuRGzUMjOvjh06BDq16+P4cOHo0SJEqhSpQrmzp0LhUKRW7ELpOzsCzc3NwQFBUmHz0JDQ3H06FG0adMmVzLTvzT1/Z0nriytKZGRkVAoFNLtOdKUKFEC9+7dy3CeiIiIDKePiIjIsZyFQXb2xcd++OEHlCxZMt0fOqkvO/vjzJkz+O2333Dt2rVcSFh4ZGdfhIaG4u+//0avXr1w9OhRPHjwAMOGDUNKSgqmT5+eG7ELpOzsi549eyIyMhINGzaEEAKpqakYMmQIJk+enBuR6T8y+/6OiYlBQkICDA0Ns7ScAtUjRAXHzz//jB07dmD//v0wMDDQdpxCJzY2Fn369MGGDRtgaWmp7TiFnlKphLW1NdavXw8XFxd4e3tjypQpWLt2rbajFToBAQGYO3cuVq9ejStXrmDfvn04cuQIZs2ape1olE0FqkfI0tIScrkcL1++VGl/+fIlbGxsMpzHxsZGrekpa7KzL9IsWrQIP//8M06cOIFq1arlZMxCQ9398fDhQ4SFhaF9+/ZSm1KpBAAUKVIEwcHBcHZ2ztnQBVR23hu2trbQ1dWFXC6X2ipVqoSIiAgkJydDT08vRzMXVNnZFz/++CP69OmDgQMHAgCqVq2K+Ph4fPvtt5gyZYrKTcIpZ2X2/W1mZpbl3iCggPUI6enpwcXFBf7+/lKbUqmEv78/6tevn+E89evXV5keAP76669Mp6esyc6+AIAFCxZg1qxZOH78OGrXrp0bUQsFdfdHxYoVcfPmTVy7dk3616FDBzRt2hTXrl2Dvb19bsYvULLz3mjQoAEePHggFaMAEBISAltbWxZBXyA7++L9+/fpip20AlXw1p25SmPf3+qN4877duzYIfT19cXmzZvFnTt3xLfffiuKFi0qIiIihBBC9OnTR0ycOFGa/uzZs6JIkSJi0aJF4u7du2L69Ok8fV5D1N0XP//8s9DT0xN79uwR4eHh0r/Y2FhtvYQCRd398TGeNaY56u6LJ0+eCFNTUzFixAgRHBwsDh8+LKytrcXs2bO19RIKDHX3xfTp04Wpqan43//+J0JDQ8Wff/4pnJ2dRbdu3bT1EgqM2NhYcfXqVXH16lUBQCxZskRcvXpVPH78WAghxMSJE0WfPn2k6dNOn//uu+/E3bt3xapVq3j6fJqVK1eK0qVLCz09PeHq6irOnz8vPefu7i769eunMv2uXbtE+fLlhZ6envjqq6/EkSNHcjlxwaXOvihTpowAkO7f9OnTcz94AaXue+O/WAhplrr74ty5c6Ju3bpCX19fODk5iTlz5ojU1NRcTl0wqbMvUlJSxIwZM4Szs7MwMDAQ9vb2YtiwYSIqKir3gxcwJ0+ezPA7IG379+vXT7i7u6ebp0aNGkJPT084OTmJTZs2qb1emRDsyyMiIqLCqUCNESIiIiJSBwshIiIiKrRYCBEREVGhxUKIiIiICi0WQkRERFRosRAiIiKiQouFEBERERVaLISISMXmzZtRtGhRbcfINplMhgMHDnxyGh8fH3h5eeVKHiLK21gIERVAPj4+kMlk6f49ePBA29GwefNmKY+Ojg5KlSqF/v3749WrVxpZfnh4OFq3bg0ACAsLg0wmw7Vr11SmWb58OTZv3qyR9WVmxowZ0uuUy+Wwt7fHt99+i7dv36q1HBZtRDmrQN19noj+5enpiU2bNqm0WVlZaSmNKjMzMwQHB0OpVOL69evo378/Xrx4AT8/vy9edmZ3Df8vc3PzL15PVnz11Vc4ceIEFAoF7t69i2+++QbR0dHYuXNnrqyfiD6PPUJEBZS+vj5sbGxU/snlcixZsgRVq1aFsbEx7O3tMWzYMMTFxWW6nOvXr6Np06YwNTWFmZkZXFxccPnyZen5M2fOoFGjRjA0NIS9vT1GjRqF+Pj4T2aTyWSwsbFByZIl0bp1a4waNQonTpxAQkIClEolfvrpJ5QqVQr6+vqoUaMGjh8/Ls2bnJyMESNGwNbWFgYGBihTpgzmzZunsuy0Q2OOjo4AgJo1a0Imk6FJkyYAVHtZ1q9fj5IlS6rc2R0AOnbsiG+++UZ6fPDgQdSqVQsGBgZwcnLCzJkzkZqa+snXWaRIEdjY2MDOzg4tWrRA165d8ddff0nPKxQKDBgwAI6OjjA0NESFChWwfPly6fkZM2Zgy5YtOHjwoNS7FBAQAAB4+vQpunXrhqJFi6JYsWLo2LEjwsLCPpmHiNJjIURUyOjo6GDFihW4ffs2tmzZgr///hvff/99ptP36tULpUqVwqVLlxAUFISJEydCV1cXAPDw4UN4enqiS5cuuHHjBnbu3IkzZ85gxIgRamUyNDSEUqlEamoqli9fjsWLF2PRokW4ceMGPDw80KFDB9y/fx8AsGLFChw6dAi7du1CcHAwfH194eDgkOFyL168CAA4ceIEwsPDsW/fvnTTdO3aFW/evMHJkyeltrdv3+L48ePo1asXAOD06dPo27cvRo8ejTt37mDdunXYvHkz5syZk+XXGBYWBj8/P+jp6UltSqUSpUqVwu7du3Hnzh1MmzYNkydPxq5duwAAEyZMQLdu3eDp6Ynw8HCEh4fDzc0NKSkp8PDwgKmpKU6fPo2zZ8/CxMQEnp6eSE5OznImIgIK5N3niQq7fv36CblcLoyNjaV/X3/9dYbT7t69WxQvXlx6vGnTJmFubi49NjU1FZs3b85w3gEDBohvv/1Wpe306dNCR0dHJCQkZDjPx8sPCQkR5cuXF7Vr1xZCCFGyZEkxZ84clXnq1Kkjhg0bJoQQYuTIkaJZs2ZCqVRmuHwAYv/+/UIIIR49eiQAiKtXr6pM069fP9GxY0fpcceOHcU333wjPV63bp0oWbKkUCgUQgghmjdvLubOnauyjG3btglbW9sMMwghxPTp04WOjo4wNjYWBgYG0p20lyxZkuk8QggxfPhw0aVLl0yzpq27QoUKKtsgKSlJGBoaCj8/v08un4hUcYwQUQHVtGlTrFmzRnpsbGwM4EPvyLx583Dv3j3ExMQgNTUViYmJeP/+PYyMjNItZ9y4cRg4cCC2bdsmHd5xdnYG8OGw2Y0bN+Dr6ytNL4SAUqnEo0ePUKlSpQyzRUdHw8TEBEqlEomJiWjYsCF+/fVXxMTE4MWLF2jQoIHK9A0aNMD169cBfDis1bJlS1SoUAGenp5o164dWrVq9UXbqlevXhg0aBBWr14NfX19+Pr6onv37tDR0ZFe59mzZ1V6gBQKxSe3GwBUqFABhw4dQmJiIn7//Xdcu3YNI0eOVJlm1apV2LhxI548eYKEhAQkJyejRo0an8x7/fp1PHjwAKampirtiYmJePjwYTa2AFHhxUKIqIAyNjZG2bJlVdrCwsLQrl07DB06FHPmzEGxYsVw5swZDBgwAMnJyRl+oc+YMQM9e/bEkSNHcOzYMUyfPh07duxAp06dEBcXh8GDB2PUqFHp5itdunSm2UxNTXHlyhXo6OjA1tYWhoaGAICYmJjPvq5atWrh0aNHOHbsGE6cOIFu3bqhRYsW2LNnz2fnzUz79u0hhMCRI0dQp04dnD59GkuXLpWej4uLw8yZM9G5c+d08xoYGGS6XD09PWkf/Pzzz2jbti1mzpyJWbNmAQB27NiBCRMmYPHixahfvz5MTU2xcOFCXLhw4ZN54+Li4OLiolKApskrA+KJ8gsWQkSFSFBQEJRKJRYvXiz1dqSNR/mU8uXLo3z58hg7dix69OiBTZs2oVOnTqhVqxbu3LmTruD6HB0dnQznMTMzQ8mSJXH27Fm4u7tL7WfPnoWrq6vKdN7e3vD29sbXX38NT09PvH37FsWKFVNZXtp4HIVC8ck8BgYG6Ny5M3x9ffHgwQNUqFABtWrVkp6vVasWgoOD1X6dH5s6dSqaNWuGoUOHSq/Tzc0Nw4YNk6b5uEdHT08vXf5atWph586dsLa2hpmZ2RdlIirsOFiaqBApW7YsUlJSsHLlSoSGhmLbtm1Yu3ZtptMnJCRgxIgRCAgIwOPHj3H27FlcunRJOuT1ww8/4Ny5cxgxYgSuXbuG+/fv4+DBg2oPlv6v7777DvPnz8fOnTsRHByMiRMn4tq1axg9ejQAYMmSJfjf//6He/fuISQkBLt374aNjU2GF4G0traGoaEhjh8/jpcvXyI6OjrT9fbq1QtHjhzBxo0bpUHSaaZNm4atW7di5syZuH37Nu7evYsdO3Zg6tSpar22+vXro1q1apg7dy4AoFy5crh8+TL8/PwQEhKCH3/8EZcuXVKZx8HBATdu3EBwcDAiIyORkpKCXr16wdLSEh07dsTp06fx6NEjBAQEYNSoUXj27JlamYgKPW0PUiIizctogG2aJUuWCFtbW2FoaCg8PDzE1q1bBQARFRUlhFAdzJyUlCS6d+8u7O3thZ6enihZsqQYMWKEykDoixcvipYtWwoTExNhbGwsqlWrlm6w8399PFj6YwqFQsyYMUPY2dkJXV1dUb16dXHs2DHp+fXr14saNWoIY2NjYWZmJpo3by6uXLkiPY//DJYWQogNGzYIe3t7oaOjI9zd3TPdPgqFQtja2goA4uHDh+lyHT9+XLi5uQlDQ0NhZmYmXF1dxfr16zN9HdOnTxfVq1dP1/6///1P6OvriydPnojExETh4+MjzM3NRdGiRcXQoUPFxIkTVeZ79eqVtH0BiJMnTwohhAgPDxd9+/YVlpaWQl9fXzg5OYlBgwaJ6OjoTDMRUXoyIYTQbilGREREpB08NEZERESFFgshIiIiKrRYCBEREVGhxUKIiIiICi0WQkRERFRosRAiIiKiQouFEBERERVaLISIiIio0GIhRERERIUWCyEiIiIqtFgIERERUaHFQoiIiIgKrf8DM/Rb02Y6V9cAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"cell_type":"code","source":[],"metadata":{"id":"ZXLAF3wCMuBh"},"execution_count":null,"outputs":[]}]}