{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-\n",
    "# Python version: 3.6\n",
    "\n",
    "\n",
    "import os\n",
    "import copy\n",
    "import time\n",
    "import pickle\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "from options import args_parser\n",
    "from update import LocalUpdate, test_inference\n",
    "from models import MLP, CNNMnist, CNNFashion_Mnist, CNNCifar\n",
    "from utils import get_dataset, average_weights, exp_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Namespace(dataset='mnist', epochs=5, frac=0.1, gpu=None, iid=1, local_bs=10, local_ep=5, lr=0.01, model='mlp', num_classes=10, num_clusters=2, num_users=100, optimizer='sgd', unequal=0, verbose=1)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import argparse\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "# parser.add_argument('--name', '-n', default='foo', help='foo')\n",
    "parser.add_argument('--model', type=str, default='mlp', help='model name(mlp or cnn)')\n",
    "parser.add_argument('--dataset', type=str, default='mnist', help=\"name of dataset(mnist or cifar)\")\n",
    "parser.add_argument('--epochs', type=int, default=5, help=\"number of rounds of training(10)\")\n",
    "parser.add_argument('--num_users', type=int, default=100, help=\"number of users: K\")\n",
    "parser.add_argument('--num_classes', type=int, default=10, help=\"number of classes\")\n",
    "parser.add_argument('--gpu', default=None, help=\"To use cuda, set to a specific GPU ID. Default set to use CPU.\")\n",
    "parser.add_argument('--frac', type=float, default=0.1, help='the fraction of clients: C')\n",
    "parser.add_argument('--local_ep', type=int, default=5, help=\"the number of local epochs: E\")\n",
    "parser.add_argument('--local_bs', type=int, default=10, help=\"local batch size: B\")\n",
    "parser.add_argument('--lr', type=float, default=0.01, help='learning rate')\n",
    "parser.add_argument('--optimizer', type=str, default='sgd', help=\"type of optimizer\")\n",
    "parser.add_argument('--verbose', type=int, default=1, help='verbose')\n",
    "parser.add_argument('--iid', type=int, default=1, help='Default set to IID. Set to 0 for non-IID.')\n",
    "parser.add_argument('--unequal', type=int, default=0, help='whether to use unequal data splits for  \\\n",
    "                        non-i.i.d setting (use 0 for equal splits)')\n",
    "# parser.add_argument('--seed', type=int, default=1, help='random seed')\n",
    "\n",
    "parser.add_argument('--num_clusters', type=int, default=2, help='verbose')\n",
    "\n",
    "\n",
    "# args = parser.parse_args([])\n",
    "args, _ = parser.parse_known_args()\n",
    "\n",
    "args\n",
    "# --dataset=mnist --gpu=0 --iid=0 --epochs=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start time:  1571372605.758036\n"
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "start_time = time.time()\n",
    "print(\"start time: \", start_time)\n",
    "\n",
    "# define paths\n",
    "path_project = os.path.abspath('..')\n",
    "logger = SummaryWriter('../logs')\n",
    "\n",
    "# args = args_parser()\n",
    "# exp_details(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if args.gpu:\n",
    "    torch.cuda.set_device(args.gpu)\n",
    "device = 'cuda' if args.gpu else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataset and user groups\n",
    "train_dataset, test_dataset, user_groups = get_dataset(args)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "keys = user_groups.keys()\n",
    "keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "l = np.arange(5,6)\n",
    "l\n",
    "{k:user_groups[k] for k in l if k in user_groups}"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "np.arange(cluster_size, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Each cluster size:  50.0\n",
      "Size of cluster 1:  50\n",
      "Size of cluster 2:  50\n"
     ]
    }
   ],
   "source": [
    "# Splitting into clusters. FL groups\n",
    "cluster_size = args.num_users / args.num_clusters\n",
    "print(\"Each cluster size: \", cluster_size)\n",
    "\n",
    "# Cluster 1\n",
    "A1 = np.arange(cluster_size, dtype=int)\n",
    "user_groupsA = {k:user_groups[k] for k in A1 if k in user_groups}\n",
    "print(\"Size of cluster 1: \", len(user_groupsA))\n",
    "# Cluster 2\n",
    "B1 = np.arange(cluster_size, cluster_size+cluster_size, dtype=int)\n",
    "user_groupsB = {k:user_groups[k] for k in B1 if k in user_groups}\n",
    "print(\"Size of cluster 2: \", len(user_groupsB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49])\n",
      "dict_keys([50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99])\n"
     ]
    }
   ],
   "source": [
    "# Check that clusters are all different\n",
    "print(user_groupsA.keys())\n",
    "print(user_groupsB.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BUILD MODEL\n",
    "def build_model(args, train_dataset):\n",
    "    if args.model == 'cnn':\n",
    "        # Convolutional neural netork\n",
    "        if args.dataset == 'mnist':\n",
    "            global_model = CNNMnist(args=args)\n",
    "        elif args.dataset == 'fmnist':\n",
    "            global_model = CNNFashion_Mnist(args=args)\n",
    "        elif args.dataset == 'cifar':\n",
    "            global_model = CNNCifar(args=args)\n",
    "\n",
    "    elif args.model == 'mlp':\n",
    "        # Multi-layer preceptron\n",
    "        img_size = train_dataset[0][0].shape\n",
    "        len_in = 1\n",
    "        for x in img_size:\n",
    "            len_in *= x\n",
    "            global_model = MLP(dim_in=len_in, dim_hidden=200,\n",
    "                               dim_out=args.num_classes)\n",
    "    else:\n",
    "        exit('Error: unrecognized model')\n",
    "        \n",
    "    return global_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199210\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Linear-1                  [-1, 200]         157,000\n",
      "           Dropout-2                  [-1, 200]               0\n",
      "              ReLU-3                  [-1, 200]               0\n",
      "            Linear-4                  [-1, 200]          40,200\n",
      "           Dropout-5                  [-1, 200]               0\n",
      "              ReLU-6                  [-1, 200]               0\n",
      "            Linear-7                   [-1, 10]           2,010\n",
      "           Softmax-8                   [-1, 10]               0\n",
      "================================================================\n",
      "Total params: 199,210\n",
      "Trainable params: 199,210\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.01\n",
      "Params size (MB): 0.76\n",
      "Estimated Total Size (MB): 0.77\n",
      "----------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<generator object Module.parameters at 0x7fd39fcdd570>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MODEL PARAM SUMMARY\n",
    "global_model = build_model(args, train_dataset)\n",
    "pytorch_total_params = sum(p.numel() for p in global_model.parameters())\n",
    "print(pytorch_total_params)\n",
    "\n",
    "from torchsummary import summary\n",
    "\n",
    "summary(global_model, (1, 28, 28))\n",
    "global_model.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP(\n",
      "  (layer_input): Linear(in_features=784, out_features=200, bias=True)\n",
      "  (relu): ReLU()\n",
      "  (dropout): Dropout(p=0.5, inplace=False)\n",
      "  (layer_hidden1): Linear(in_features=200, out_features=200, bias=True)\n",
      "  (layer_hidden2): Linear(in_features=200, out_features=10, bias=True)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Set the model to train and send it to device.\n",
    "global_model.to(device)\n",
    "global_model.train()\n",
    "print(global_model)\n",
    "\n",
    "# copy weights\n",
    "global_weights = global_model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the cluster models to train and send it to device.\n",
    "cluster_modelA = build_model(args, train_dataset)\n",
    "cluster_modelA.to(device)\n",
    "cluster_modelA.train()\n",
    "# copy weights\n",
    "cluster_modelA_weights = cluster_modelA.state_dict()\n",
    "\n",
    "# Set the cluster models to train and send it to device.\n",
    "cluster_modelB = build_model(args, train_dataset)\n",
    "cluster_modelB.to(device)\n",
    "cluster_modelB.train()\n",
    "# copy weights\n",
    "cluster_modelB_weights = cluster_modelA.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "print(np.random.choice(range(args.num_users), m, replace=False))\n",
    "np.random.choice(A1, m, replace=False)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "len(A1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the training function\n",
    "def fl_train(args, train_dataset, cluster_global_model, cluster, usergrp, epochs):\n",
    "#     print(args)\n",
    "    \n",
    "#     # Set the model to train and send it to device.\n",
    "#     cluster_global_model = build_model(args, train_dataset)\n",
    "#     cluster_global_model.to(device)\n",
    "#     cluster_global_model.train()\n",
    "\n",
    "#     # copy weights\n",
    "#     cluster_global_weights = cluster_global_model.state_dict()\n",
    "    \n",
    "    cluster_train_loss, cluster_train_accuracy = [], []\n",
    "    cluster_val_acc_list, cluster_net_list = [], []\n",
    "    cluster_cv_loss, cluster_cv_acc = [], []\n",
    "    print_every = 2\n",
    "    cluster_val_loss_pre, counter = 0, 0\n",
    "\n",
    "    for epoch in tqdm(range(epochs)):\n",
    "        cluster_local_weights, cluster_local_losses = [], []\n",
    "        print(f'\\n | Cluster Training Round : {epoch+1} |\\n')\n",
    "\n",
    "        cluster_global_model.train()\n",
    "        m = max(int(args.frac * len(cluster)), 1)\n",
    "        idxs_users = np.random.choice(cluster, m, replace=False)\n",
    "\n",
    "\n",
    "        for idx in idxs_users:\n",
    "            cluster_local_model = LocalUpdate(args=args, dataset=train_dataset, idxs=usergrp[idx], logger=logger)\n",
    "            cluster_w, cluster_loss = cluster_local_model.update_weights(model=copy.deepcopy(cluster_global_model), global_round=epoch)\n",
    "            cluster_local_weights.append(copy.deepcopy(cluster_w))\n",
    "            cluster_local_losses.append(copy.deepcopy(cluster_loss))\n",
    "\n",
    "        # averaging global weights\n",
    "        cluster_global_weights = average_weights(cluster_local_weights)\n",
    "\n",
    "        # update global weights\n",
    "        cluster_global_model.load_state_dict(cluster_global_weights)\n",
    "\n",
    "        cluster_loss_avg = sum(cluster_local_losses) / len(cluster_local_losses)\n",
    "        cluster_train_loss.append(cluster_loss_avg)\n",
    "\n",
    "#         # Calculate avg training accuracy over all users at every epoch\n",
    "#         cluster_list_acc, cluster_list_loss = [], []\n",
    "#         cluster_global_model.eval()\n",
    "        \n",
    "#         for c in range(len(cluster)):\n",
    "#             cluster_local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "#                                       idxs=usergrp[idx], logger=logger)\n",
    "#             cluster_acc, cluster_loss = cluster_local_model.inference(model=cluster_global_model)\n",
    "#             cluster_list_acc.append(cluster_acc)\n",
    "#             cluster_list_loss.append(cluster_loss)\n",
    "#         cluster_train_accuracy.append(sum(cluster_list_acc)/len(cluster_list_acc))\n",
    "    \n",
    "#     return cluster_global_model, cluster_train_loss #cluster_global_weights, cluster_loss_avg, cluster_train_accuracy \n",
    "    return cluster_global_weights, cluster_loss_avg, idxs_users\n",
    "    \n",
    "# A_model, A_weights, A_losses, A_trainacc = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)    \n",
    "# A_model, A_trainloss = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)\n",
    "# A_weights, A_losses = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Defining the evaluation function\n",
    "def fl_eval(train_dataset, global_model, cluster, usergrp, idxs_users):\n",
    "    cluster_list_acc, cluster_list_loss = [], []\n",
    "#     for c in range(len(cluster)):\n",
    "    for c in idxs_users:\n",
    "        cluster_local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "                                  idxs=usergrp[c], logger=logger)\n",
    "        acc, loss = cluster_local_model.inference(model=global_model)\n",
    "        cluster_list_acc.append(acc)\n",
    "        cluster_list_loss.append(loss)\n",
    "#     train_accuracy.append(sum(list_acc)/len(list_acc))\n",
    "\n",
    "    return cluster_list_acc, cluster_list_loss\n",
    "\n",
    "\n",
    "clusterA_list_acc, clusterA_list_loss = fl_eval(train_dataset, global_model, A1, user_groupsA, A_idxs_users)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A/home/wesleyjtann/federated_learning/Hierarchical-Federated-Learning-Quantization/src/update.py:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  return torch.tensor(image), torch.tensor(label)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Global Training Round : 1 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.096095\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.095368\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.095915\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.109742\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.111495\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.109021\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.094421\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.113914\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.097614\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.123185\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.096879\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.102096\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.083931\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.092149\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.096873\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.119807\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.110413\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.124875\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.113997\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.131701\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.115308\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.102201\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.133931\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.106337\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.122132\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.097808\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.098855\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.091453\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.101234\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.098302\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.100230\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.105763\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.106588\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.103117\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.106096\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.098537\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.097494\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.106579\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.109100\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.103675\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.112879\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.096471\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.107295\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.093205\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.080166\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.124953\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.130094\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.150509\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.093827\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.145682\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.099201\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.095056\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.101225\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.101631\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.110149\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.101510\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.090375\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.119668\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.097214\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.099370\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.107300\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.092744\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.090303\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.120303\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.101846\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.100883\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.102154\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.122853\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.104328\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.117501\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.102373\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.135977\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.119019\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.118398\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.117525\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.107699\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.095685\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.101372\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.092226\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.094981\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.104082\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.095300\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.095665\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.102902\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.106785\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.107369\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.094827\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.096985\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.110529\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.106278\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.108805\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.105078\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.086826\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.112614\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.102624\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.118228\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.098668\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.098846\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.130087\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.131778\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.095145\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.088335\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.093521\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.108089\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.095850\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.098186\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.103609\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.100243\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.099031\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.100346\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.107958\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.092704\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.101075\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.126616\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.111339\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.107630\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.116371\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.119717\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.129075\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.111362\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.106454\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.145459\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.136471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:05<00:05,  5.58s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.101967\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.125911\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.155612\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.099404\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.153660\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.127929\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.178086\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.141891\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.204750\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.178672\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.084030\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.161728\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.146982\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.188140\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.129334\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.144131\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.168870\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.292867\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.141971\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.132810\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.231259\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.096127\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.165778\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.211542\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.191558\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.207631\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.488176\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.114919\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.114328\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.108475\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.088163\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.155555\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.177846\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.248255\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.119158\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.243889\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.087960\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.088216\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.243846\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.255599\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.262805\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.158613\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.186171\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.262168\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.261845\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.264202\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.177061\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.226561\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.098491\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.257226\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.295321\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.158107\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.106710\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.104876\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.162724\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.103784\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.159022\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.089204\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.100227\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.204172\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.341967\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.303416\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.247451\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.143295\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.184025\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.161260\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.098856\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.256855\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.193474\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.298257\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.104523\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.145326\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.076796\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.319182\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.202429\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.258697\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.197052\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.094940\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.138090\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.094413\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.139131\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.197492\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.140272\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.110093\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.106380\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.182229\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.101921\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.097781\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.142803\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.072664\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.317042\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.292629\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.208952\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.211098\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.158060\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.207826\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.164574\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.146439\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.180247\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.084975\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.210029\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.434521\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.125362\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.139188\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.155077\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.150075\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.121644\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.118458\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.243267\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.152990\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.148345\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.084285\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.170403\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.134010\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.250703\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.124010\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.184921\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.099116\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.234140\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.172611\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.116087\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.111634\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.154933\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.281246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:10<00:00,  5.47s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.146177\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.254555\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.138017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 20%|██        | 1/5 [00:12<00:50, 12.63s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 1 global rounds:\n",
      "Training Loss : -0.17483100766316056\n",
      "Train Accuracy: 30.93% \n",
      "\n",
      "\n",
      " | Global Training Round : 2 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.147012\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.151953\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.196379\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.268140\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.257758\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.155039\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.270904\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.112824\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.241484\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.398757\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.352232\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.231949\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.276862\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.407187\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.256152\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.318576\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.362436\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.201942\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.378422\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.344411\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.264936\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.211596\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.385533\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.290106\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.055753\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.184245\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.151943\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.262523\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.274713\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.214811\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.335985\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.202036\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.631819\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.086931\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.273725\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.087245\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.179901\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.116714\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.387977\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.287337\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.472073\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.221048\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.231207\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.185367\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.245236\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.139833\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.390377\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.435819\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.358525\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.590469\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.382280\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.238906\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.124378\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.141865\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.073220\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.219803\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.386530\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.202705\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.094958\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.219991\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.214379\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.560911\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.343752\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.335736\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.224424\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.275565\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.233832\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.554924\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.176391\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.410493\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.274540\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.369605\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.214231\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.390689\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.291993\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.282011\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.338312\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.318113\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.109577\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.160115\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.318306\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.268941\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.208965\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.320702\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.286847\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.325984\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.153084\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.200359\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.331510\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.402132\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.445971\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.565642\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.506001\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.526667\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.386329\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.351485\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.403102\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.325300\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.262271\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.586014\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.159343\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.270694\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.176912\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.112349\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.208131\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.251215\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.386624\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.424917\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.411346\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.315811\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.181182\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.247699\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.506719\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.289343\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.370864\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.142640\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.380042\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.475689\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.589940\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.212418\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.363603\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.236667\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.354838\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:05<00:05,  5.39s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.534581\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.306094\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.378661\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.451120\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.275458\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.480436\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.169095\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.415228\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.361633\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.445511\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.498865\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.559752\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.453941\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.335437\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.574583\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.467110\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.746251\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.382823\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.287879\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.308035\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.563082\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.609374\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.507209\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.481464\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.176975\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.669144\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.522204\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.556989\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.400701\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.408805\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.321986\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.361342\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.201390\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.556509\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.410715\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.404199\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.588455\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.257368\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.432931\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.527741\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.343912\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.154908\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.533743\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.416844\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.418882\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.285753\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.575114\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.627226\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.631195\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.508772\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.621256\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.483897\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.299254\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.492781\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.384285\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.274046\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.170624\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.413859\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.418284\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.383436\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.344793\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.188355\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.572744\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.583237\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.382815\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.224879\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.311810\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.181098\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.343759\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.321365\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.485198\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.552106\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.567756\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.539102\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.534470\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.096636\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.614180\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.546948\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.418978\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.415581\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.170400\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.399018\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.473471\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.362093\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.500761\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.126003\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.510441\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.412300\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.241455\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.289909\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.544191\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.385696\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.465175\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.436777\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.326614\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.447086\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.512236\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.612227\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.507626\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.291849\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.274161\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.303308\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.382542\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.654399\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.463600\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.276605\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.325230\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.275456\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.487285\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.632627\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.588213\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.620330\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.372796\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.225821\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.575631\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.517626\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.310245\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.479611\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.377333\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.377413\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.715675\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.274157\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.693468\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.396864\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.507021\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.552704\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.465008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:10<00:00,  5.42s/it]\u001b[A\n",
      " 40%|████      | 2/5 [00:25<00:37, 12.60s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 2 global rounds:\n",
      "Training Loss : -0.3025186641203861\n",
      "Train Accuracy: 55.08% \n",
      "\n",
      "\n",
      " | Global Training Round : 3 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.382190\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.337071\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.333849\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.732037\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.661827\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.694731\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.570238\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.500775\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.527049\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.426739\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.786150\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.435274\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.614407\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.844284\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.517255\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.489667\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.357635\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.877113\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.644903\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.598231\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.718170\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.598217\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.498498\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.503880\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.519707\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.615176\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.335301\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.358890\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.597220\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.501680\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.167765\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.616069\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.302544\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.604098\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.702931\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.271459\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.361511\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.414386\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.582991\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.340298\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.311506\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.219565\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.771429\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.524952\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.488072\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.431920\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.507375\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.391642\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.489417\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.802306\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.328261\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.579090\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.459747\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.556777\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.583777\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.191930\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.385754\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.570967\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.505290\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.524262\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.603189\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.236810\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.398158\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.560080\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.308340\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.504601\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.379427\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.494617\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.510846\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.793145\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.585192\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.591614\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.519058\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.677466\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.522817\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.375437\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.631814\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.538769\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.479891\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.297407\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.769687\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.516295\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.428303\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.343928\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.834268\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.741967\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.775872\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.614213\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.418897\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.526731\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.575544\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.470707\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.735783\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.551880\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.516515\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.355525\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.568248\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.447501\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.749046\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.565979\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.625662\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.499787\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.532869\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.629767\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.614183\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.315098\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.294742\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.344216\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.534135\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.665259\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.504861\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.423758\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.437004\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.317892\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.604978\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.472225\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.900715\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.705698\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.864496\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.629222\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.482812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:05<00:05,  5.49s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.594445\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.687847\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.497201\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.340818\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.296139\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.534700\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.403491\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.574731\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.541201\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.352601\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.479368\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.391288\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.558416\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.502347\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.522978\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.690339\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.515909\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.641743\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.477063\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.507657\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.659633\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.751299\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.830758\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.690800\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.316413\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.377509\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.525574\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.658730\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.360525\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.633803\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.509590\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.306462\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.475140\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.494389\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.699487\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.434541\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.459729\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.446719\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.710813\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.688430\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.604689\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.415053\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.562233\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.772246\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.467443\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.353841\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.702360\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.707843\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.460808\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.575044\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.400823\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.579424\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.737458\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.598252\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.421292\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.551269\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.429859\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.499133\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.536433\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.767575\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.386939\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.517406\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.316310\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.518404\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.592547\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.610266\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.468919\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.805366\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.438837\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.753281\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.440607\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.430766\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.366727\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.515995\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.532614\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.828007\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.269444\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.613616\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.218701\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.530569\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.435531\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.574634\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.496577\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.462475\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.497340\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.352518\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.323660\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.677399\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.570067\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.625768\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.640603\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.664436\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.588812\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.367029\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.465609\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.388677\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.579498\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.437423\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.691720\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.784120\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.751288\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.514704\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.649028\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.657138\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.754107\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.557056\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.708715\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.691517\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.525753\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.409343\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.727356\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.514201\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.772657\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.615738\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.273152\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.683704\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.713723\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.657038\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.477929\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.663635\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.584963\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.703948\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.578498\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.912379\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.109035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:10<00:00,  5.39s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.630306\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.637145\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.511008\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.747102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 3/5 [00:37<00:25, 12.55s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 3 global rounds:\n",
      "Training Loss : -0.3874857748900023\n",
      "Train Accuracy: 64.83% \n",
      "\n",
      "\n",
      " | Global Training Round : 4 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.473995\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.733553\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.389231\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.465494\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.665651\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.461149\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.494436\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.756409\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.655071\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.688980\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.835785\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.492230\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.742140\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.650615\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.587071\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.537682\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.286922\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.480232\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.648535\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.854610\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.404578\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.537792\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.521363\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.788580\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.641968\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.724606\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.470085\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.740464\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.590903\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.634097\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.530813\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.362734\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.693709\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.865317\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.703388\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.585303\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.475991\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.763689\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.781686\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.674224\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.586856\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.711699\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.733776\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.573750\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.720216\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.772430\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.717749\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.674134\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.647888\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.962862\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.441577\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.812229\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.628097\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.544328\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.696194\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.722534\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.296514\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.630717\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.749730\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.405097\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.790288\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.558185\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.471254\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.516400\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.798978\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.483703\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.724426\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.873541\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.694313\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.666447\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.474521\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.620387\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.631224\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.457162\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.631433\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.479127\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.688169\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.641691\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.601620\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.574787\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.451159\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.674334\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.691832\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.466317\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.714814\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.856728\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.579782\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.682220\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.659093\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.592512\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.752308\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.556459\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.420131\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.410158\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.411700\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.597515\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.767982\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.598843\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.507706\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.604164\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.505186\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.529309\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.572153\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.438224\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.390990\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.732911\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.468978\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.693247\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.723309\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.656098\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.647870\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.530930\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.605062\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.304390\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.469434\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.711168\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.508186\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.434315\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.781461\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.666854\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.365875\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.574523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:05<00:05,  5.41s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.807305\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.586733\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.581675\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.492947\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.702653\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.750913\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.683387\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.752954\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.624389\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.780971\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.491977\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.479179\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.696900\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.651111\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.691325\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.598011\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.202039\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.688681\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.658913\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.402134\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.659995\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.397953\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.789494\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.594406\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.661938\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.489881\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.757910\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.433368\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.520774\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.789829\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.585502\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.547791\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.597995\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.557830\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.691243\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.911887\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.550465\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.662039\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.780288\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.411133\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.395766\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.553792\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.679488\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.776725\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.590402\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.501772\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.685816\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.773386\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.648762\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.751028\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.653487\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.772401\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.658831\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.848921\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.571958\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.497255\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.799912\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.749628\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.598397\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.659235\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.522133\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.692386\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.445788\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.650571\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.472958\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.694787\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.714502\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.588379\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.778994\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.686361\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.684346\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.472247\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.760335\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.684979\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.597670\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.752742\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.535454\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.783408\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.690450\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.476702\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.786281\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.599510\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.582394\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.849420\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.550723\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.583531\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.674006\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.901241\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.658950\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.458987\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.201226\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.562879\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.792588\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.668474\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.746352\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.856147\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.597705\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.303116\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.684652\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.698302\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.667001\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.609173\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.598438\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.437529\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.736047\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.676372\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.599883\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.781610\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.729146\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.648522\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.671831\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.536826\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.485127\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.580113\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.493260\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.445048\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.502596\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.556662\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.399793\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.803221\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.594910\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.659397\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.349196\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.775998\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.737228\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.718495\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.700629\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.783090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:10<00:00,  5.39s/it]\u001b[A\n",
      " 80%|████████  | 4/5 [00:50<00:12, 12.54s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 4 global rounds:\n",
      "Training Loss : -0.4488468113774434\n",
      "Train Accuracy: 66.00% \n",
      "\n",
      "\n",
      " | Global Training Round : 5 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.654433\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.570089\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.471738\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.741984\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.772682\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.758413\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.394502\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.628423\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.619697\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.616758\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.815901\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.597817\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.540510\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.728601\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.538965\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.979338\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.653688\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.316276\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.573031\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.774775\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.756269\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.774570\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.747217\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.798047\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.761440\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.793643\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.740420\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.495352\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.555491\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.674601\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.838863\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.593237\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.389862\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.744407\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.403373\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.478563\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.552960\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.658861\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.700710\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.582410\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.469447\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.488823\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.487638\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.720999\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.570851\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.695791\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.604215\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.582271\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.782660\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.596385\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.848837\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.702563\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.706512\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.517521\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.379177\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.409861\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.836726\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.382950\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.598349\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.700855\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.782534\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.504291\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.444872\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.662069\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.877113\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.793150\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.603831\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.740062\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.522959\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.687431\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.798923\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.562289\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.597234\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.563007\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.450922\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.832831\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.463228\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.498174\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.666230\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.670132\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.803706\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.539768\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.772972\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.632427\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.582659\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.504063\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.697014\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.539610\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.851684\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.567401\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.602079\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.941393\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.561609\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.741965\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.782658\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.721389\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.945707\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.601999\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.626185\n",
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.651494\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.503399\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.876808\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.522715\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.786569\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.722187\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.589975\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.757958\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.689130\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.344781\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.862890\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.753734\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.281356\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.704868\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.635673\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.687052\n",
      "| Global Round : 0 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.863956\n",
      "| Global Round : 0 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.575312\n",
      "| Global Round : 0 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.594297\n",
      "| Global Round : 0 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.492740\n",
      "| Global Round : 0 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.700501\n",
      "| Global Round : 0 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.662344\n",
      "| Global Round : 0 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.698127\n",
      "| Global Round : 0 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.494202\n",
      "| Global Round : 0 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.668482\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:05<00:05,  5.28s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.503171\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.734452\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.662678\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.376946\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.544139\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.648688\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.599423\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.799900\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.841077\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.583445\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.800250\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.582256\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.597629\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.657716\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.871777\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.470123\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.700126\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.764462\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.699420\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.862442\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.870723\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.446301\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.494814\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.995686\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.546619\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.733128\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.679124\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.681020\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.789482\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.562929\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.685354\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.834416\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.838260\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.434738\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.723302\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.694331\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.595231\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.594022\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.866780\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.687278\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.403198\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.602284\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.936711\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.663553\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.600109\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.568741\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.792317\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.588638\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.589747\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.853557\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.748849\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.695540\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.659012\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.988451\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.561921\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.569085\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.691870\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.695683\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.674579\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.750031\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.330671\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.546988\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.786570\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.518602\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.696935\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.581845\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.642018\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.525951\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.496258\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.679976\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.592772\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.701805\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.639035\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.770643\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.760555\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.694125\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.710259\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.853638\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.671387\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.502748\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.893779\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.758642\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.649458\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.494304\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.771362\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.702016\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.791686\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.783492\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.551651\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.587681\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.838623\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.812043\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.692372\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.410306\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.605987\n",
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.601351\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.797517\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.774008\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.697817\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.726114\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.669929\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.784978\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.851640\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.865459\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.748572\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.705689\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.491100\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.489134\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.632597\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.498213\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.699092\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.787990\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.184031\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.868899\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.475759\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.301777\n",
      "| Global Round : 1 | Local Epoch : 3 | [0/480 (0%)]\tLoss: -0.682815\n",
      "| Global Round : 1 | Local Epoch : 3 | [100/480 (21%)]\tLoss: -0.463499\n",
      "| Global Round : 1 | Local Epoch : 3 | [200/480 (42%)]\tLoss: -0.402896\n",
      "| Global Round : 1 | Local Epoch : 3 | [300/480 (62%)]\tLoss: -0.684832\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 3 | [400/480 (83%)]\tLoss: -0.386102\n",
      "| Global Round : 1 | Local Epoch : 4 | [0/480 (0%)]\tLoss: -0.932833\n",
      "| Global Round : 1 | Local Epoch : 4 | [100/480 (21%)]\tLoss: -0.399744\n",
      "| Global Round : 1 | Local Epoch : 4 | [200/480 (42%)]\tLoss: -0.742792\n",
      "| Global Round : 1 | Local Epoch : 4 | [300/480 (62%)]\tLoss: -0.671507\n",
      "| Global Round : 1 | Local Epoch : 4 | [400/480 (83%)]\tLoss: -0.705341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:10<00:00,  5.28s/it]\u001b[A\n",
      "100%|██████████| 5/5 [01:02<00:00, 12.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 5 global rounds:\n",
      "Training Loss : -0.49031288038939247\n",
      "Train Accuracy: 66.37% \n",
      "\n",
      "\n",
      " Total Run Time: 69.0038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_loss, train_accuracy = [], []\n",
    "val_acc_list, net_list = [], []\n",
    "cv_loss, cv_acc = [], []\n",
    "print_every = 1\n",
    "val_loss_pre, counter = 0, 0\n",
    "\n",
    "for epoch in tqdm(range(args.epochs)):\n",
    "    local_weights, local_losses, local_accuracies= [], [], []\n",
    "    print(f'\\n | Global Training Round : {epoch+1} |\\n')\n",
    "    \n",
    "    # ============== TRAIN ==============\n",
    "    global_model.train()\n",
    "#     m = max(int(args.frac * args.num_users), 1)\n",
    "#     idxs_users = np.random.choice(range(args.num_users), m, replace=False)\n",
    "\n",
    "#     for idx in idxs_users:\n",
    "#         local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "#                                   idxs=user_groups[idx], logger=logger)       \n",
    "#         w, loss = local_model.update_weights(\n",
    "#             model=copy.deepcopy(global_model), global_round=epoch)\n",
    "#         local_weights.append(copy.deepcopy(w))\n",
    "#         local_losses.append(copy.deepcopy(loss))\n",
    "    \n",
    "    # Cluster A\n",
    "#     A_model, A_weights, A_losses, A_trainacc = fl_train(args, train_dataset, global_model, A1, user_groupsA, 1)\n",
    "    A_weights, A_losses, A_idxs_users = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)\n",
    "    local_weights.append(copy.deepcopy(A_weights))\n",
    "    local_losses.append(copy.deepcopy(A_losses))\n",
    "#     local_accuracies.append(copy.deepcopy(A_trainacc))\n",
    "    \"\"\"\n",
    "    # Cluster B\n",
    "#     B_model, B_weights, B_losses, B_trainacc = fl_train(args, train_dataset, global_model, B1, user_groupsB, 1)\n",
    "    B_weights, B_losses = fl_train(args, train_dataset, cluster_modelB, B1, user_groupsB, 2)\n",
    "    local_weights.append(copy.deepcopy(B_weights))\n",
    "    local_losses.append(copy.deepcopy(B_losses))\n",
    "#     local_accuracies.append(copy.deepcopy(B_trainacc))\n",
    "    \"\"\"\n",
    "    \n",
    "    # averaging global weights\n",
    "    global_weights = average_weights(local_weights)\n",
    "\n",
    "    # update global weights\n",
    "    global_model.load_state_dict(global_weights)\n",
    "\n",
    "    loss_avg = sum(local_losses) / len(local_losses)\n",
    "    train_loss.append(loss_avg)\n",
    "    \n",
    "    # ============== EVAL ============== \n",
    "    # Calculate avg training accuracy over all users at every epoch\n",
    "    list_acc, list_loss = [], []\n",
    "    global_model.eval()\n",
    "    for c in range(args.num_users):\n",
    "        local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "                                  idxs=user_groups[c], logger=logger)\n",
    "        acc, loss = local_model.inference(model=global_model)\n",
    "        list_acc.append(acc)\n",
    "        list_loss.append(loss)\n",
    "    train_accuracy.append(sum(list_acc)/len(list_acc))\n",
    "    \n",
    "    # print global training loss after every 'i' rounds\n",
    "    if (epoch+1) % print_every == 0:\n",
    "        print(f' \\nAvg Training Stats after {epoch+1} global rounds:')\n",
    "        print(f'Training Loss : {np.mean(np.array(train_loss))}')\n",
    "        print('Train Accuracy: {:.2f}% \\n'.format(100*train_accuracy[-1]))\n",
    "#         print('Train Accuracy: {:.2f}% \\n'.format(100*train_accuracy[-1][0]))\n",
    "\n",
    "print('\\n Total Run Time: {0:0.4f}'.format(time.time()-start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      " Results after 5 global rounds of training:\n",
      "|---- Avg Train Accuracy: 66.37%\n",
      "|---- Test Accuracy: 67.26%\n"
     ]
    }
   ],
   "source": [
    "# Test inference after completion of training\n",
    "test_acc, test_loss = test_inference(args, global_model, test_dataset)\n",
    "\n",
    "print(f' \\n Results after {args.epochs} global rounds of training:')\n",
    "print(\"|---- Avg Train Accuracy: {:.2f}%\".format(100*train_accuracy[-1]))\n",
    "print(\"|---- Test Accuracy: {:.2f}%\".format(100*test_acc))\n",
    "\n",
    "# # Saving the objects train_loss and train_accuracy:\n",
    "# file_name = '../save/objects/{}_{}_{}_C[{}]_iid[{}]_E[{}]_B[{}].pkl'.\\\n",
    "#     format(args.dataset, args.model, args.epochs, args.frac, args.iid,\n",
    "#            args.local_ep, args.local_bs)\n",
    "\n",
    "# with open(file_name, 'wb') as f:\n",
    "#     pickle.dump([train_loss, train_accuracy], f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wesleyjtann/miniconda3/envs/fl_pytorch/lib/python3.7/site-packages/ipykernel_launcher.py:4: UserWarning: matplotlib.pyplot as already been imported, this call will have no effect.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<function matplotlib.pyplot.show(*args, **kw)>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XecFPX9x/HXm2ZDFAQRRUANUWMB9cSa2LAkFjQxii1o9GePXWOPGjUaazTGHoFgN7FEsWKLGsVDQcWGYg2K2LBhQT6/P75zshx7x95xu3Pl/Xw85nG7O9+d+ezc7n52vjPz+SoiMDMzK0W7vAMwM7OWw0nDzMxK5qRhZmYlc9IwM7OSOWmYmVnJnDTMzKxkThqtmKT2kr6Q1Kcp21rrIGljSRPLsNzlJX3R1MvNm6TBkt7MO468OWk0I9mXds00S9KMgvu7NXR5EfF9RHSOiLebsm1DSTpd0vCmXm4lSVpX0j2Spkv6WNJTkn6Td1zzIyIejohV5nc5kt6VtHHBcidHROf5Xa41T04azUj2pd05+8C9DWxb8Ni1tdtL6lD5KNseSRsCDwBjgOWBJYCDgV/kGVdb5/d/Ppw0WpDsF/uNkq6X9Dmwu6T1JD0p6VNJ70m6SFLHrH0HSSGpX3Z/VDb/bkmfS/qvpOUa2jab/3NJr2a/vC+W9LikPRvxmlaR9EgW//OSti6Yt42kl7L1vyvp8OzxJSWNzp7zsaRH61j2VZLOqvXYXZIOyW4fL2mKpM8kvVz4a7mWc4GrI+KciPgokqcjYmjBcveX9JqkjyTdJqlXre16gKTXs9fyB0n9s//bZ9n/s+Z/NljSm5KOkzQti2/bbFtMyl7vMQXrHSXplIL7c3ShZNvtiGzbTs/WtUAdbftmsU+T9KGkv2SP95f0UPbaPpT0D0mLZfOuB5YG7s72iI+Q9CNJUbDc3pLuzGKfJOm3BfNOz2IalW2bFyStWcf/s2ZbHijpNeDl7PENJVVnr2+spHVqvf6NC+7/sNdbE6ek32Ttpkk6tqDtwtlr/USpG2+tWvGU+v5pXSLCUzOcgDeBwbUeOx34FtiWlPAXAtYG1gE6kH4FvwocnLXvAATQL7s/CvgQqAI6AjcCoxrRdkngc2BINu8I4Dtgzzpey+nA8CKPdwLeAI7JljMY+AL4UTZ/GrB+drsbsGZ2+xzgr9lzOgEb1bHeTbPtqOz+EsAMoCewCvAWsFQ2bzlg+SLLWBSYBfy0nv/VFsAHwEBgQeBvwIO1tuu/smWtnv0P7wf6AV1JX367Ze0HAzOBE7LXd0C27FFA5+z5XwN9Cv5PpxTEMhh4s+D+u8CTwFLZ638V2Kd22yzOF0gJchHSe2uDbN6Pgc2ybb0k8Dhwbq11bFxw/0dAFNx/HLg42zZrZu+rjQreGzOALYH22f/2sTq2c822vCfbbgsB3YHpwC7Z/N2Bj4CudcT2w3uxJk7gsoLYvgH6Z/PPBR7O1tUXeLFge5X0/mmNk/c0Wp7HIuLfETErImZE+sX7VETMjIjJwBXARvU8/5aIqI6I74BrSV90DW27DTA+Im7P5l1A+iJoqA1IX0TnRMR3EfEAcDdQ8wv+O+AnkhaNiI8j4pmCx5cmfXF+GxGP1LH8h0lfvOtl93cC/hMRU0lfzAsCq0jqEBFvZNuvtm6AgPfqeR27AVdFxPiI+Bo4FthIUu+CNmdHxOcR8RzwEnBPRLwZEZ8A9wJrFLT9Gjgr27Y3AD2ACyLii+z5r5CSR6kujIj3I+Ij4E6K/8/XI30B/z4ivszeW48DRMSrETEm29YfkP7f9b3HfpDtnQ4Cjo2Ir7P/4TXAHgXNHomIeyPie+AfdcRX6MyI+CQiZpB+QE2MiOuzz8AoYDKwdf2LmMMpBbFNBAZkj+8EnJ6t6y3SD5Uapb5/Wh0njZbnncI7klbKulzel/QZcBrpw1+X9wtuf0X69drQtksXxhHpp9a7JcRe29LA29nza7wFLJPd3gHYDnhb0sMF3Q5nZe3GZF0+RxdbeETMIu0h7ZI9tCsp+RERrwBHkrbXB1kXyVJFFvMx6ddor3m8jrcK1vsZ8EnB6wCYWnB7RpH7hf+HD7Mv0Jp5xZ7fkAPNpfzPlyX9iv6+9gxJS0m6SdL/svfYcOp/jxVamvR6vix4rPB/XCy+ReaxzMLPwBzbvo7l1ysi6to+vWqtq/B/XOr7p9Vx0mh5apclvpzUrfCjiOgCnEz6ZVxO7wE//IqWJBrwIS0wBVg2e36NPsD/ALI9qO1IXSJ3kn51ExGfRcThEdEP2B74vaS6fvleD+yU/eJdE7i1ZkZEjIqIDUhdC+2BP9V+ckR8DowFfjWP19G35o6kRUldGv+r5zlN5Utg4YL7jf3iegfoK6l9kXlnk7ptVsveY3sy53usvlLZU4DukgoTwQ//40YqXN8c277I8udn+7xPSqaFy50dRAnvn9bISaPlW5TUp/ulpJWB/SqwzjuBNbMDtB2AQ0ldKPVpL2nBgmkB4AnSbv6RkjpK2pR0RtJNkhaStKukLlk3zefA9wDZelfIks307PG5fiEDRMTTWZsrgNHZXgCSVpa0SRbHjGwqugzgaGCf7CBvt+z5a0i6Lpt/PbC3pNWz5f2J1A3WmL2vhhoPbC2pq9LB90MauZz/ko4FnJkdAF5I0gbZvEVJX77TJS0LHFXruVNJx9PmEhFvANXZcheQNBDYi2yPrwncSeoi2jk7UL4r6VjF6Gz+eGBoNm8Q8MsGLPsm4HhJiytdv3RwzYwGvn9aFSeNlu9IYBjpS/VyUndMWWXHBHYGzid90awAPEv6NVqX3Zn94ZoBvBIR35D6pIeQjolcBOwaEa9mzxkGvJV1iezN7H7wFYEHSQfNHwf+EhGP1bPu60kHfa8reGwB4M/Zet8n7RmcWMfr/U/2/C2BNyV9DFxK9sUUEfeQuiluJe2F9SEd56iE4aRjJG+RDhDf0JiFRMRM0rGqlUl7HW8DO2az/0A6LjEduAP4Z62nnwmcqnQ222FFFr8z0J+0nW8Bjo+IhxoTZ5G4p5G6MH9Pei8eDmwTER9nTU4AVgI+BU5izvfAvPyB9P98k3SsbWTBvJLfP61NzVklZo2WdWlMAXbMvmDNrJXynoY1iqStJC2W7Z6fROpmGptzWGZWZk4a1lgbkk5t/BDYCtg+624ys1bM3VNmZlYy72mYmVnJWl3Br+7du0e/fv3yDsPMrEUZN27chxExr1PnW1/S6NevH9XV1XmHYWbWokiqfWV9Ue6eMjOzkjlpmJlZyZw0zMysZE4aZmZWMicNMzMrmZOGmZmVzEnDzMxK5qRR45tv4Pe/h7dKOlXZzKxNctKo8d57cOmlsOuuMHNm3tGYmTVLTho1+vWDyy6DJ56A007LOxozs2bJSaPQrrvCsGFwxhnwyCN5R2Nm1uw4adR28cWwwgqw++7w0Ud5R2Nm1qw4adS26KJw/fUwdSrssw94vBEzsx84aRSz1lrwpz/Bbbel4xxmZgY4adTt8MNhyy3hiCPghRfyjsbMrFnIJWlI6ibpfkmTsr9di7QZKOm/kiZKek7SzhUNsl07GDECunSBoUNhxoyKrt7MrDnKa0/jWGBMRPQHxmT3a/sK+E1ErAJsBVwoafEKxgg9e8LIkTBxIhx5ZEVXbWbWHOWVNIYAI7LbI4DtazeIiFcjYlJ2ewrwATDPoQib3JZbpoRx6aVw660VX72ZWXOSV9LoGRHvAWR/l6yvsaRBQCfg9QrENrczz0wHx/feG955J5cQzMyag7IlDUkPSHqhyDSkgcvpBfwD2CsiZtXRZl9J1ZKqp02b1hThz6lTp3Qa7rffpus3vv++6ddhZtYClC1pRMTgiFi1yHQ7MDVLBjVJ4YNiy5DUBbgLODEinqxnXVdERFVEVPXoUaYerP794W9/g0cfTXseZmZtUF7dU3cAw7Lbw4DbazeQ1Am4FRgZETdXMLa67bFHKjVy6qnw+ON5R2NmVnF5JY2zgM0lTQI2z+4jqUrSVVmbnYCfAXtKGp9NA/MJNyOlA+J9+qTk8emnuYZjZlZpilZWJqOqqiqqq6vLu5KnnoINN4QddoAbb0zJxMysBZM0LiKq5tXOV4Q3xjrrwOmnw803w9VX5x2NmVnFOGk01tFHw2abwaGHwksv5R2NmVlFOGk0Vrt26WrxhReGXXaBr7/OOyIzs7Jz0pgfSy8Nw4fDhAlpfHEzs1bOSWN+bb116qK66CK48868ozEzKysnjaZw9tkwYADstRdMmZJ3NGZmZeOk0RQWWABuuAG++ipdAOgyI2bWSjlpNJWVVkpdVA8+COeck3c0ZmZl4aTRlH77W9hpJzjppHQBoJlZK+Ok0ZQkuPxyWGaZdBru9Ol5R2Rm1qScNJra4ovDddfB22/DAQdAKyvTYmZtm5NGOay/PpxyShqDY+TIvKMxM2syThrlctxxsNFGcNBB8OqreUdjZtYknDTKpX17GDUqnY67yy5p1D8zsxbOSaOcevdOVXCfeQaOPz7vaMzM5puTRrltvz0ceCCcdx7cc0/e0ZiZzRcnjUo491xYdVUYNgymTs07GjOzRnPSqISFFkpnUn32WUocs2blHZGZWaM4aVTKqqvCBRfAvfemv2ZmLZCTRiXtt18aV/y442DcuLyjMTNrMCeNSpLgqqugZ08YOhQ+/zzviMzMGsRJo9K6dUvXb0yeDL/7Xd7RmJk1iJNGHjbaCE48EUaMSHWqzMxaCCeNvJx0EmywAey/f9rrMDNrAZw08tKhA1x7LbRrl8qMfPdd3hGZmc2Tk0ae+vaFK6+EsWPh5JPzjsbMbJ6cNPL261/D//0fnH02jBmTdzRmZvVy0mgOLrwwjTG+xx4wbVre0ZiZ1clJozlYeOFUZuSjj2CvvTzan5k1W04azcWAAamw4V13wcUX5x2NmVlRThrNycEHwzbbwNFHw/jxeUdjZjYXJ43mRIJrroEllkin4X75Zd4RmZnNIZekIambpPslTcr+di3Spq+kcZLGS5ooaf88Yq247t1TmZFXXoHDDss7GjOzOeS1p3EsMCYi+gNjsvu1vQesHxEDgXWAYyUtXcEY87PppnDssam44c035x2NmdkP8koaQ4AR2e0RwPa1G0TEtxHxTXZ3AdpaV9qpp8I666RrON56K+9ozMyA/L6Ie0bEewDZ3yWLNZK0rKTngHeAsyNiSgVjzFfHjqmY4axZsOuuMHNm3hGZmZUvaUh6QNILRaYhpS4jIt6JiNWBHwHDJPWsY137SqqWVD2tNV0ct/zycPnl8MQTcNppeUdjZoYihwvJJL0CbBwR70nqBTwcESvO4znXAHdFxC31tauqqorq6uomjLYZ2GuvVEb9oYdSWXUzsyYmaVxEVM2rXV7dU3cAw7Lbw4DbazeQ1FvSQtntrsAGwCsVi7A5ufhi+NGPYPfd01XjZmY5yStpnAVsLmkSsHl2H0lVkq7K2qwMPCVpAvAIcG5EPJ9LtHnr3BluuAGmToV99nGZETPLTS7dU+XUKrunapx/Phx5JPztb3DAAXlHY2atSHPvnrLGOOww2GorOOIIeOGFvKMxszbISaMladcOhg+HxRaDoUNhxoy8IzKzNsZJo6Xp2RNGjoSJE1NXlZlZBTlptERbbAFHHQWXXgq33pp3NGbWhjhptFRnnAFrrQV77w3vvJN3NGbWRjhptFSdOqXR/r77Ll2/8f33eUdkZm2Ak0ZL1r8/XHIJPPoonHlm3tGYWRvgpNHS7bEH7LYbnHIKPP543tGYWSvnpNHSSeliv379UjXcTz/NOyIza8WcNFqDLl3S8Y0pU2DffV1mxMzKxkmjtRg0CE4/PY30d/XVeUdjZq2Uk0ZrcvTRMHgwHHIIvPRS3tGYWSvUoKShZJFyBWPzqV27dLX4IovALrvA11/nHZGZtTLzTBqSRkrqImlhYCLwhqQjyh+aNUqvXmnApgkT4Pe/zzsaM2tlStnTWC0iPgO2B+4DegN7ljMom0+/+AUceihcdBHceWfe0ZhZK1JK0ugkqQMwBLgtIr4FZpU3LJtvZ58NAwemoWKnTMk7GjNrJUpJGlcBbwNdgUck9QG+KGtUNv8WWCCdhvvVV+kCQJcZMbMmMM+kEREXRMTSEbFFpGH+3gE2LX9oNt9WWimNL/7gg3DOOXlHY2atQCkHwg+W1CW7fTnwFPDTcgdmTWSvvWCnneDEE+Gpp/KOxsxauFK6p/aNiM8kbQEsAxwA/Lm8YVmTkeDyy6F373Qa7vTpeUdkZi1YKUmjpibFz4FrImJcic+z5mLxxeG66+Dtt+GAA1xmxMwarZQv/wmSRgPbAndL6szsRGItxfrrp0q411+fLgA0M2uEUpLGXsApwKCI+ApYENi7nEFZmRx3HGy0ERx0ELz6at7RmFkLVMrZU98D3YFjJJ0FrB0Rz5Y9Mmt67dvDqFHpdNxddoFvv807IjNrYUo5e+oM4BhgcjYdLen0cgdmZdK7N/z97/DMM3D88XlHY2YtTCndU9sCgyPiioi4AtgC2K68YVlZDRkCBx4I550H99yTdzRm1oKUehbUonXctpbq3HNh1VVh2DCYOjXvaMyshSglafwZeEbSVZKuBqqBs8sblpXdQgvBDTfAZ5+lxDHL5cTMbN5KORA+CtgQGJ1NP4uIa8sdmFXAKqvAhRfCvffCBRfkHY2ZtQAd6pohafVaD72W/V1C0hIR8Vz5wrKK2XdfuO++dDruxhvDWmvlHZGZNWN1Jg3gknrmBfCzJo7F8iDBlVfCgAEwdGg6q2pRH7Yys+LqTBoR4aKEbUW3bnDttbDJJvC738Hw4XlHZGbNlGtIWfKzn8FJJ6WhYq/1ISszKy6XpCGpm6T7JU3K/natp20XSf+T9NdKxtgmnXgibLBBKmo4eXLe0ZhZM5TXnsaxwJiI6A+Mye7X5Y/AIxWJqq3r0CHtZbRvn8qMfPdd3hGZWTNTShmR1YtMfSXNT8IZAozIbo8Atq9j3WsBPYH75mNd1hB9+6YD42PHwskn5x2NmTUzpXzxXw2MA0YC/yBd3HcrMEnSZo1cb8+IeA8g+7tk7QZZUjoPOHpeC5O0r6RqSdXTpk1rZEj2gx13TKfinn02PPBA3tGYWTNSStKYBKwVEQMjYgCwFjAe2JL0pV6UpAckvVBkGlJibAcCoyPinXk1zOpiVUVEVY8ePUpcvNXrggvSGON77AFOxGaWqe86jRorF17IFxHPS1ozIl6TVOeTImJwXfMkTZXUKyLek9QL+KBIs/WAn0o6EOgMdJL0RUTUd/zDmsrCC6cBm9ZZJ40z/u9/p2s6zKxNK2VP43VJF0vaIJsuAl6TtAAws5HrvQMYlt0eBtxeu0FE7BYRfSKiH3AUMNIJo8IGDEiFDe+6Cy6+OO9ozKwZKCVp/AZ4l3SG03HAFNIX/Uygscc0zgI2lzQJ2Dy7j6QqSVc1cplWDgcdBNtuC0cfDePH5x2NmeVMEa1ruO+qqqqorq7OO4zW5cMP015Hly5QXQ2LLJJ3RGbWxCSNi4iqebUr5ZTbdSXdLelFSa/WTE0TprUI3bvDP/4Br7wChx2WdzRmlqNSDoRfQxrudRzwfXnDsWZr001TJdwzz4TNN4eddso7IjPLQSnHND6LiH9HxJSImFozlT0ya35OOQXWXTddw/Hmm3lHY2Y5KCVpPCjpT5LWLrwqvOyRWfPTsSNcdx1EwG67wczGnjxnZi1VKd1TG9b6Cx5Po+1abjm47DLYdVc47bQ0mVmbMc+k4XE1bC677AL33w+nnw5rrgnbFy0dZmatUH3Dve4SEddLOqTY/Ii4qHxhWbN30UUwYQLssAPst1+6CLBz57yjMrMyq++YRs0YFz3qmKwt69wZnngCjjkGrrgiXcfx2GN5R2VmZeaL+2z+PfYYDBsGb7yRrhw/7TRYYIG8ozKzBij14r55HtOQ1B34LdCvsH1E7Ds/AVorsuGGqavqqKPgz3+G0aPTxYADB+YdmZk1sVJOub2dNBDSY6RR9moms9k6d05nVY0eDR99BIMGpQsBfVquWatSyim3i0TEkWWPxFqHn/8cXnghFTo84YRUUn3kSOjfP+/IzKwJlLKncbekLcoeibUe3bqlsTiuvz7VqxowAC65JF0UaGYtWilJY3/gHklfSPpY0ieSPi53YNYKDB2a9jo23hgOPhi23BLefTfvqMxsPpSSNLoDHYHFSKfadsen3Fqpll46DeJ0+eXpFN1VV4Vrr/Veh1kLVWfSkFTTCb1KHZNZaaRU5HDChJQ0dt8dfv3rNE6HmbUo9R0IPxbYG7ikyDzXnrKGW2EFeOQROO88OOmkdH3HlVemkQHNrEXwxX2Wj+efhz32SHsfe+8N55+fRgY0s1w02ch92cJWkvRLSbvWTPMforVpq60GY8fC8cfDNdfA6qunvRAza9ZKGe71ROAK4DLg58CFwI5ljsvagk6d4IwzUjdVp06wySZw5JHw9dd5R2ZmdShlT2NnYBPgvYjYAxhAaRcFmpVmvfXg2WfhwANTN9Vaa8G4cXlHZWZFlJI0ZkTE98BMSYsC7wPLlzcsa3MWWQT++le4916YPj0NK3vaafDdd3lHZmYFSkkaz0paHPg7UA2MBZ4pa1TWdm2xRTpIvvPO8Ic/wAYbwMsv5x2VmWXqTRqSBJwSEZ9GxCXA1sB+EfGbikRnbVPXrjBqFNx8M0yeDGuskQZ9mjUr78jM2rx6k0ak83HvLLj/WkR4L8MqY8cdUxmSwYPh0EPT37ffzjsqszatlO6psZLWLHskZsUstRTccQdcfTU8/XQ6VXfECJchMctJfWVEas6Q2pCUOF6R9IykZyV5b8MqR4Lf/haeey4N7LTnnmls8g8+yDsyszanvlNnxwJrAttXKBaz+i23HDz0EFx4YboocNVVUyHEHXbIOzKzNqO+7ikBRMTrxaYKxWc2p3bt4Igj4JlnYNll4Ze/TOOTT5+ed2RmbUJ9exo9JB1R18yIOL8M8ZiV5ic/gSefhNNPT1eVP/RQKkey2WZ5R2bWqtW3p9Ee6AwsWsdklq+OHeHUU9M4HQsvPPssq6++yjsys1arvj2N9yLitIpFYtZYgwal7qrjj4e//CVdVT5yZHrczJrUPI9plIOkbpLulzQp+9u1jnbfSxqfTXeUKx5rBRZeOB0gHzMm7Wmsvz6cfLLLkJg1sfqSRjk7h48FxkREf2BMdr+YGRExMJu2K2M81lpsumkqQ7L77vDHP6YaVhMn5h2VWatRZ9KIiI/LuN4hwIjs9gh8Wq81pcUWg+HD4dZb4Z13UtXc886D77/POzKzFq+kQZjKoGdEvAeQ/V2yjnYLSqqW9KSkOhOLpH2zdtXTpk0rR7zWEm2/fSpDstVWcNRRaS/kjTfyjsqsRStb0pD0gKQXikxDGrCYPtnwg7sCF0paoVijiLgiIqoioqpHjx5NEr+1EksumfY4hg+H8ePTCIFXX+0yJGaNVLakERGDI2LVItPtwFRJvQCyv0XrQUTElOzvZOBhYI1yxWutmJQuAHz++XRG1T77wHbbwfvv5x2ZWYuTV/fUHcCw7PYw4PbaDSR1lbRAdrs7sAHwYsUitNanTx+4//50Wu4DD6QyJLfckndUZi1KXknjLGBzSZOAzbP7SKqSdFXWZmWgWtIE4CHgrIhw0rD5064dHHJIGl52+eXh179OZ1p98knekZm1CIpW1rdbVVUV1dXVeYdhLcHMmfCnP6VhZXv2hL//PY0caNYGSRqXHUOuV157Gmb569ABTjop1bBabDHYcks46CD48su8IzNrtpw0zNZaC8aNS9VzL700jdnx3//mHZVZs+SkYQaw4ILpAsCHHkqlRzbcMNWy+vbbvCMza1acNMwKbbRRGiFwr73S8Y5Bg9J9MwOcNMzm1qULXHVVGpv8/fdh7bXh7LNdhsQMJw2zum27bSpDsu22cOyxaS/kdQ9aaW2bk4ZZfbp3h5tvhlGjUgIZMCCNS97KTlU3K5WThtm8SLDbbilprL8+7L8//OIXMGVK3pGZVZyThlmpevdOowJecgk88kgqQ3LDDXlHZVZRThpmDSHBgQfChAmw4oqwyy4wdCh89FHekZlVhJOGWWP07w//+Q+ccQb861+w2mpw9915R2VWdk4aZo3VoUO6AHDsWFhiiXScY7/94Isv8o7MrGycNMzm18CBUF0NxxwDV16ZzrB67LG8ozIrCycNs6awwALpAsBHH033f/azlES++SbfuMyamJOGWVPacMN0kHzffeGcc6CqKg0za9ZKOGmYNbXOneGyy2D06HRW1aBBcOaZafwOsxbOScOsXH7+83RB4C9/CSecAD/9KTzzTN5Rmc0XJw2zcurWLV0AeP318MoraeyOddeFESNgxoy8ozNrMCcNs0oYOjQVO7zwQpg+HfbcE5ZZBg4/PCUTsxbCScOsUrp2hUMPhRdfTIM9bbFFKkmy0kqw6aapMKIHfbJmzknDrNIk2Hjj1G31zjvpIPnkybDTTtCnTzr+8eabeUdpVpSThlmeevaE445LXVejR6czrc46C5ZfHrbZBu6804M/WbPipGHWHLRvn862uuMOeOONtLcxblwaAGr55VONq/ffzztKMycNs2anTx/44x/h7bfhlltSccQTT4Rll4Vf/xoefNCDQFlunDTMmquOHeFXv4IHHkhnWB16aEoYm22WDp6ffz58/HHeUVob46Rh1hL8+Mdw7rnwv//ByJFpGNojj4Sll4Zhw+C///Xeh1WEk4ZZS7LggrDHHvD446nG1W9/m8bzWH99WGONVL7k88/zjtJaMScNs5Zq9dXhb39LY5Vffnk6lfeAA9Lex/77p6Ri1sScNMxaukUXTVV1n3kGnnwSdtwxlSkZOBDWWy91Z7lkiTURJw2z1kKCddaBa65Jxz4uuAA++SQd81hmGTjiCHj11byjtBbOScOsNerWDQ47DF56KZUs2XxzuPhiWHHFdPaVS5ZYIzlpmLVmNSVLbrwxlSw544x09XlNyZITT4S33so7SmtBckkakrpJul/SpOxv1zra9ZF0n6SXJL0oqV9lIzVrRZZaCo4/PiWNu+6CtddOda+WWy6VLLnrLpcssXnKa0/jWGBMRPQHxmT3ixkJnBN4XJ+QAAANZElEQVQRKwODgA8qFJ9Z69W+PfziF/Dvf89ZsmSbbVyyxOYpr6QxBBiR3R4BbF+7gaSfAB0i4n6AiPgiIr6qXIhmbUDfvrNLltx885wlS3baKR0P8UWDViCvpNEzIt4DyP4uWaTNj4FPJf1L0rOSzpHUvtjCJO0rqVpS9bRp08oYtlkr1bFjOlX3gQfg5ZfhkEPS7U03TSVLLrjAJUsMKGPSkPSApBeKTENKXEQH4KfAUcDawPLAnsUaRsQVEVEVEVU9evRokvjN2qwVV4TzzptdsmSJJdLpusssk0YcfPJJ7320YWVLGhExOCJWLTLdDkyV1Asg+1vsWMW7wLMRMTkiZgK3AWuWK14zq2WhhVLJkieegPHjU8L45z/TBYNrrJGuQnfJkjYnr+6pO4Bh2e1hwO1F2jwNdJVUs+uwKfBiBWIzs9oGDIBLL00lSy67LD22//6pZMkBB8Bzz+Ubn1VMXknjLGBzSZOAzbP7SKqSdBVARHxP6poaI+l5QMCVOcVrZpBKluy3Hzz7bKqs+6tfwfDhKamsv75LlrQBilbWN1lVVRXV1dV5h2HWdnz8cap1ddllqUxJt26pK2u//VJJd2sRJI2LiKp5tfMV4WY2f7p1g8MPT2dd1QwSddFF6YD64MFp9MHvvss7SmsiThpm1jQk2GQTuOmm2SVLJk1KQ9TWlCx5++28o7T55KRhZk2vpmTJ5Mlw551QVTW7ZMm228Lo0S5Z0kI5aZhZ+bRvD1tvPbtkyXHHwdNPp8dWWCElkqlT847SGsBJw8wqo29fOP301EV1000paZxwAvTuDTvv7JIlLYSThplVVqdO6TjHmDHp4Pnvfgf3359Klqy8Mlx4oUuWNGNOGmaWnxVXhPPPTyVLRoyArl3TmVg1JUv++c80jO3HH3svpJnwdRpm1ryMH59KlIwaBV98MfvxLl2gX7/i03LLweKL5xFtq1HqdRpOGmbWPH35JbzyCrz55tzTG2/MmVAAFlus/qSy2GIVDL7lKTVpdKhEMGZmDbbIIrDmmmmqLQI++SQlj9oJ5fXXU1n3L7+c8zmLL153UunXz0mlRE4aZtbySOlK9G7dYK215p4fkY6DFEsqkybBfffBV7XGdOvatf6k0qVLGV9Qy+GkYWatj5TGAVliiXRhYW0R8NFHs7u6CpPKK6/AvffOnVS6das/qSy6aBlfUPPhpGFmbY8E3bunqa6k8uGHxZPKSy/B3XfPXc13iSXqTyqdO5fxBVWOk4aZWW0S9OiRprXXnnt+BEybVjypTJwId90FX38953O6d68/qSyySBlfUNNx0jAzaygJllwyTYMGzT0/Aj74YM6zvWpuP/98KqvyzTdzPqdHj7oTSt++zSapOGmYmTU1CXr2TNM668w9f9asupPKhAlwxx1zJ5Ull6w/qSy8cDlf0Q+cNMzMKq1du1QJeKmlYN11554/a1Yq5FgsqTz7LNx2G3z77ZzPWXLJVJr+hhvKGrqThplZc9OuHfTqlab11pt7/qxZ8P77c1/w2KNH2UNz0jAza2natYOll07T+utXdtUVXZuZmbVoThpmZlYyJw0zMyuZk4aZmZXMScPMzErmpGFmZiVz0jAzs5I5aZiZWcla3XCvkqYBb83HIroDHzZROE3JcTWM42oYx9UwrTGuvhExz0vKW13SmF+SqksZJ7fSHFfDOK6GcVwN05bjcveUmZmVzEnDzMxK5qQxtyvyDqAOjqthHFfDOK6GabNx+ZiGmZmVzHsaZmZWMicNMzMrWZtMGpK2kvSKpNckHVtk/gKSbszmPyWpXzOJa09J0ySNz6Z9KhTX3yV9IOmFOuZL0kVZ3M9JWrOZxLWxpOkF2+vkCsW1rKSHJL0kaaKkQ4u0qfg2KzGuim8zSQtKGitpQhbXqUXaVPwzWWJcuXwms3W3l/SspDuLzCvf9oqINjUB7YHXgeWBTsAE4Ce12hwIXJbdHgrc2Ezi2hP4aw7b7GfAmsALdcz/BXA3IGBd4KlmEtfGwJ05bK9ewJrZ7UWBV4v8Lyu+zUqMq+LbLNsGnbPbHYGngHVrtcnjM1lKXLl8JrN1HwFcV+z/Vc7t1Rb3NAYBr0XE5Ij4FrgBGFKrzRBgRHb7FmAzSWoGceUiIh4FPq6nyRBgZCRPAotL6tUM4spFRLwXEc9ktz8HXgKWqdWs4tusxLgqLtsGX2R3O2ZT7TN0Kv6ZLDGuXEjqDWwNXFVHk7Jtr7aYNJYB3im4/y5zf3B+aBMRM4HpwBLNIC6AX2XdGbdIWrbMMZWq1NjzsF7WvXC3pFUqvfKsW2AN0q/UQrlus3righy2WdbVMh74ALg/IurcXhX8TJYSF+TzmbwQOAaYVcf8sm2vtpg0imXb2r8eSmnT1EpZ57+BfhGxOvAAs39J5C2P7VWKZ0j1dAYAFwO3VXLlkjoD/wQOi4jPas8u8pSKbLN5xJXLNouI7yNiINAbGCRp1VpNctleJcRV8c+kpG2ADyJiXH3NijzWJNurLSaNd4HCXwO9gSl1tZHUAViM8neDzDOuiPgoIr7J7l4JrFXmmEpVyjatuIj4rKZ7ISJGAx0lda/EuiV1JH0xXxsR/yrSJJdtNq+48txm2To/BR4Gtqo1K4/P5DzjyukzuQGwnaQ3Sd3Ym0oaVatN2bZXW0waTwP9JS0nqRPpINEdtdrcAQzLbu8IPBjZEaU846rV570dqU+6ObgD+E12RtC6wPSIeC/voCQtVdOPK2kQ6f3+UQXWK+Bq4KWIOL+OZhXfZqXElcc2k9RD0uLZ7YWAwcDLtZpV/DNZSlx5fCYj4riI6B0R/UjfEw9GxO61mpVte3VoioW0JBExU9LBwL2kM5b+HhETJZ0GVEfEHaQP1j8kvUbKzkObSVyHSNoOmJnFtWe54wKQdD3prJrukt4F/kA6KEhEXAaMJp0N9BrwFbBXM4lrR+AASTOBGcDQCiR/SL8E9wCez/rDAY4H+hTElsc2KyWuPLZZL2CEpPakJHVTRNyZ92eyxLhy+UwWU6nt5TIiZmZWsrbYPWVmZo3kpGFmZiVz0jAzs5I5aZiZWcmcNMzMrGROGpa77NqAGyS9LulFSaMl/TjvuOoiaWlJtzTyuXtKWrrg/lWSftIEMRVWW31Z0uHzu8wS1rlxsQqr1ro5aViusgvJbgUejogVIuInpGsHeuYbWd0iYkpE7NjIp+8J/JA0ImKfiHixSQJLlUwHkq7HOKEZ1SazVsRJw/K2CfBddmEZABExPiL+k10tfY6kFyQ9L2ln+OEX7iOSbpL0qqSzJO2mNPbB85JWyNoNl3Sp0hgSkyVtpDQGx0uShtesT9IXBbd3rJmXPf8iSU9kz98xe7yfsjE8lAranZut9zlJv8seP1nS01nsV2SvZUegCrg22yNYSNLDkqqy5+ySLecFSWcXxifpDKUigk9KqjehRsRHpIsGe2XP7ytpTBbfGEl9Cl7fD8mvZjtk2/dhpQJ8L0u6NkvuNWO+vCzpMeCXBc/dSLPHlHhW0qKl/futpXHSsLytCtRVeO2XwEBgAKmEwzmaXbZhAHAosBrpKucfR8QgUqno3xUsoyuwKXA4qbjcBcAqwGqSBpYQXy9gQ2Ab4Kwi8/cFlgPWyIrWXZs9/teIWDsiVgUWAraJiFuAamC3iBgYETNqFpJ1WZ2dxToQWFvS9tnsRYAnsyKCjwL/V1/AWVJYEHiuJhZSGfaa+C4q4XWvARwG/IQ0xssGkhYk1VfaFvgpsFRB+6OAg7I9nZ+Sria3VshJw5qzDYHrs0qjU4FHgLWzeU9n40N8Qxq86r7s8eeBfgXL+HdWBuN5YGpEPB8Rs4CJtdrV5baImJV1IRX7hT+YNNjNTICIqCkKt4nSiGnPkxLBvEqMr03qopuWLeta0iBTAN8CNccOxtUT986SJgKTgb9ExNfZ4+uRBusB+Adpu87L2Ih4N9tW47N1rgS8ERGTsm1aWCTvceB8SYcAi9dsD2t9nDQsbxOpuzJofYPGfFNwe1bB/VnMWVPtmyJtarcrrKWzYD3rKRaPaj2f7Bf534AdI2I10q/z2ssttpy6fFdQ/+l76q4Zd2NErEL6pX+epKXqaFezrJlk3wFZ91OngjaFr7twnUXrDkXEWcA+pL2qJyWtVPfLsZbMScPy9iCwgKQfulwkrS1pI1JXzM7ZcYMepF/eY8sQw1RJK0tqB+zQwOfeB+yvVH4aSd2YnSA+VBq7ovCg+eekoVZrewrYSFJ3pQJ5u5D2rBosIv5L2qOoGQP8CWYXrNsNeCy7/SazE/YQsmKP9XgZWK7mmFEWIwCSVsj24s4mdcE5abRSThqWq+wX9A7A5kqn3E4ETiGNLXErqV9+Aim5HBMR75chjGNJ3T8PAg0tT34V8DbwnKQJwK7Z2AtXkrrEbiOVva8xHLis5kB4zYNZWfTjgIdIr/eZiLi9cS8HSMdH9soOSB+S3X6OdPynJplcSUpUY4F1gC/rW2DW3bUvcFd2IPytgtmHZQfwJ5COZ9w9H7FbM+Yqt2ZmVjLvaZiZWcmcNMzMrGROGmZmVjInDTMzK5mThpmZlcxJw8zMSuakYWZmJft/dJ1RYa2Y47gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOTTING (optional)\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.use('Agg')\n",
    "\n",
    "# Plot Loss curve\n",
    "plt.figure()\n",
    "plt.title('Training Loss vs Communication rounds')\n",
    "plt.plot(range(len(train_loss)), train_loss, color='r')\n",
    "plt.ylabel('Training loss')\n",
    "plt.xlabel('Communication Rounds')\n",
    "# plt.savefig('../save/fed_{}_{}_{}_C[{}]_iid[{}]_E[{}]_B[{}]_loss.png'.\n",
    "#             format(args.dataset, args.model, args.epochs, args.frac,\n",
    "#                    args.iid, args.local_ep, args.local_bs))\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function matplotlib.pyplot.show(*args, **kw)>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XmYFNX1//H3hx2RiMoqi7jgAoqoA2rcMAZFRdBoouIvilH5GkUQRAU3FMUAIgiyqLjGfSfITqK4JBEYkEVAFDAqIDiAiLIPnN8fVYPF2DPdA9NTs5zX8/QzXVW3qk7VdNfpurfqlswM55xzLj/l4g7AOedc8efJwjnnXFKeLJxzziXlycI551xSniycc84l5cnCOedcUp4snHMpk9RI0s+Syqdh2T9LOrSwlxsnSY0lmaQKcceytzxZpImkaZJ+kFQ57lgKk6TnJGVLOijuWIo7SUdIekPSGkk/SponqUc6DrRFxcy+MbN9zWzH3iwn/H5cl2vZ+5rZsr2L0KWLJ4s0kNQYOB0woH2a1lHkv1QkVQMuAX4ErizidZeoX2aSDgOmA98Cx5rZfsAfgQygepyxlWUl7XNUrJiZvwr5BdwL/BsYDIyLjD8ZWAWUj4y7GJgXvi8H9AKWAmuB14EDwmmNCZLPtcA3wIfh+DfCZf4IfAg0iyz7QOBdYAMwE3gQ+Dgy/ShgKrAOWAz8Kcl2XUVw8OsGfJZrWnngzjD2n4BZQMNwWrPIelYDd4bjnwMejCyjNbA8Mvw/4A5gHrAVqBDZPz8BC4GLc8VxPbAoMv0E4DbgrVzlHgMeTbCNvYA3c40bCgwL33cCloXL/wq4Mo999SIwPsn+bA8sANYD04Cjc237beG2bwSeBuoAE8N1/xPYP9dn45rw//MDcAPQMpx/PTA8suz7gBcjwznzVwiHpwEPEHyGfwKmADXzKHsA8CywMlzvmHD8/sA4ICscPw5oEE7rB+wAtgA/58QWLvfw8P1+wN/D+b8G7gbKRf4HHwODwmV/BZyXz37+H7/+HB0dbuf68H/QPlJ+GnBdZLgTu39vLNy/X4brHwEo8j0YBKwh+JzclGt/dSKFz09xfMUeQGl8AUuAG4ETge1Anci0pUCbyPAbQK/w/S3AJ0ADoDLwBPBKOC3nS/p3oBpQNRz/F4JfqpWBR4E5kWW/Gr72AZoSHEg+DqdVC4evCb88J4Qf8Gb5bNe/gIEEB61s4ITItNuA+cCRgIDjCJJVdeA74FagSjh8UjjPcyRPFnOAhpHt/SNwEEFivYzgQFovMm0FwUFSwOHAwUC9sFyNsFwF4HvgxATbeDCwCfhNOFw+jP/kcJ9tAI4Mp9XLa38RJPBr8tmXR4QxtQEqAreHn5tKkW3/JNzX9cN4ZwPHh//r94A+uT4bj4f7+ByCA/EYoHZk/jPD8veRPFksDWOsGg73z6PseOA1guRQMbKOAwnOQvcJ/+dvECaSyDquy7VPosni78A/wnkbA18A14bTOhF8r64P/z9/JUhWymNf/4/I5yiMcwnBj5tKwO8IDt5HJoqNxMliHFADaESQ0NqG024APg/XdQDwfs7+ogCfn+L4ij2A0vYCTgs/yDm/xD4HukemPwg8E76vTnDAODgcXgScHSlbL1xWhciX9NB81l0jLLNf+CXanvPBjKw7J1lcBnyUa/4nCA9ACZbdCNgJtAiHJwNDI9MXAx0SzHcF8Gkey3yO5MniL0n295yc9YYxdcuj3ETg+vB9O2BhPsv8GLgqfN8GWBq+r0bwS/QSwuSVzzK25xxA8ph+D/B6ZLgcQaJrHdn2KyPT3wJGRYZv5pdf8TmfjfqR6WuBy3LNf0v4/j6SJ4u7I9NvBCblLht+PncSnuEk2R8tgB8iw9PII1mEn92tQNPItP8DpoXvOwFLItP2Ceetm8e6d/scEVQRryI8UwnHvQLclyg2EieL0yLDr/PLD773gBsi085h92SR0uenOL68zaLwXQ1MMbM14fDL4Tgiw38IG77/AMw2s6/DaQcD70haL2k9QfLYQfDrMse3OW8klZfUX9JSSRsIvhQANYFaBB/QbxPNG67rpJx1heu7Eqibx3b9GVhkZnPC4ZeAjpIqhsMNCX6N5pbX+FRFY0bSVZLmRGI+hmB7k63reeD/he//H/BCPut8mSDJAXQMhzGzjQRJ9gbgO0njJR2VxzLWEhxM83IQQfUK4bJ3Emxr/UiZ1ZH3mxMM75trmQUtn59Vkfeb8pi3IbDOzH7IPUHSPpKekPR1+Nn8EKiRYuN+TYJf/F9Hxn3N7vtmV3xmtil8m9/2RT9HBwHfhvs8r+Unk9f+OSjXuqL/44J8foodTxaFSFJV4E/AmZJWSVoFdAeOk3QcgJktJPgAnUfkQBT6lqDutUbkVcXMVkTKWOR9R6AD8HuCs4nGOaEQnBpnE1Rp5WiYa10f5FrXvmb21zw27yrg0Mh2DSb4Up8XWd5hCebLazwEZ1X7RIYTJapd2yvpYGA00AU40MxqAJ8RbG+ydY0Bmks6huDM4qU8ykFQZdJaUgOCNqVd/yMzm2xmbQgSwedhPIn8k+AXZF5WEiTsnG0Twf9nRZ5zFJ5U9nsqvgUOkFQjwbRbCaokTzKz3wBnhONz/leWYJ4cawjOzA6OjGvE3u2b6PpWAg0lRY9/0eXvzf75jt2/Z412CyL1z0+x48micF1EcCbQlOC0uwVBQ9pHBAfbHC8DXQm+QG9Exj8O9AsPikiqJalDPuurTnC6vpbgw/1QzgQLLm18G7gv/JV3VK4YxgFHSPqzpIrhq6Wko3OvRNIpBAfhVpHtOobdz5qeAh6Q1ESB5pIODNdTV9ItkipLqi7ppHCeOcD5kg6QVJegzSY/1Qi+9FlhXNeEceR4Cugp6cQwhsNz9qWZbQHeDGOeYWbf5LUSM8siqIp4FvjKzBaF66sjqX14VdhWgsbZvC4h7QP8VtLD4bYRxvNieHB9HbhA0tnh2dmt4TL/k2QfFIY5wBnhPRP7Ab33ZCFm9h1B9d5ISfuHn6GcpFCd4GxmvaQDCPZH1Gog4T0V4Wf3dYLvQvXwf9iD4KKBwjCdICHcHsbcGriQoH0Pgv3zh/B7czjBRSWpeh3oKqmBpP0JLpgACvz5KXY8WRSuq4FnLbgWfVXOCxgOXBm5bO8Vgvr59yLVVRBcdTMWmCLpJ4IGzpPI298JzlJWEFz580mu6V0IzjhWEVS7vELwIcXMfiKoT72c4JfWKmAAQeNpou36h5nNz7VdQ4F24cFgMMEXZQpBI97TBPWyPxHU+18YruNL4KxwuS8Acwmqz6YQNJTmKTwrewT4L8HB5liCK3Zypr9BcKXNywQNlmMIGhlzPB/Ok18VVI6XCc7Yomd+5QgO6isJruw6k6A+P1GsS4FTCM72Fkj6kaDdIBP4ycwWE1SHPUbwS/pC4EIz25ZCbHvFzKYS7Ot5BFetjduLxf2Z4Czgc4JG9JyE/yhBY/Iags/lpFzzDQUuDe9FGpZguTcTHNCXEbQhvQw8sxdx7hLu4/YEZ8VrgJEEbVSfh0WGANsIPmPPk/9ZaG6jCdrO5hJckPB2ZFrKn5/iKOdyL1cGSBpA0Ah4ddLCpZCkRgQHtbpmtiHueJwrSfzMohSTdFRYHSRJrQhOp9+JO644hPXTPYBXPVE4V3B+N2PpVp2g6ukggiqCRwiuXS9Twjri1QRVdm1jDse5EsmroZxzziXl1VDOOeeSKjXVUDVr1rTGjRvHHYZzzpUos2bNWmNmtZKVKzXJonHjxmRmZsYdhnPOlSiSvk5eyquhnHPOpcCThXPOuaQ8WTjnnEvKk4VzzrmkPFk455xLypOFc865pDxZOOecS6rU3GfhnHMlyY4dO9i6dWuer23btqU8vV69enTu3Dmt8XqycM6VamZGdnZ2SgfgwiqTyjJ27tyZPPgUnXLKKZ4snHOl144dO1i7di3ff//9bq+ffvqpUA/ihdlhasWKFalcuXLCV6VKlahcuTJVq1alRo0aeU7P77UnZcqVS3+LgicL51yhMTM2bNjwq4N/oldWVhZr1qzJ90Ce7MBZqVIl9t13Xw488MBCPfjmVaZSpUpFcmAujjxZOOfytXnzZrKyslJKAN9//z3bt29PuJwaNWpQu3ZtateuzVFHHcUZZ5xBrVq1do3LedWqVYv99tuPihUrIqmIt9blxZOFc2VMdnY2a9as2e0Xfn4H/59//jnhcqpUqUKdOnWoXbs2Bx10EC1atNjtgJ87AVSqVKmIt9QVprQmC0ltCR7MXh54ysz6JyjzJ+A+wIC5ZtYxHL8DmB8W+8bM2qczVudKKjNj/fr1Cat5Eh38165dm3A55cuX3+0gf+ihh/7qV380EVSrVs1/+ZchaUsWksoDI4A2wHJgpqSxZrYwUqYJ0Bs41cx+kFQ7sojNZtYiXfE5V5xt2rQpz3r+ROPyqvo54IADdh3gmzZtSuvWrRMmgNq1a1OjRo0yWx/vkkvnmUUrYImZLQOQ9CrQAVgYKXM9MMLMfgAws+/TGI9zsdm+fftuVT/JksDGjRsTLqdatWq7Du4NGzbkxBNPTFjvX7t2bWrWrEnFihWLeEtdaZXOZFEf+DYyvBw4KVeZIwAk/Zugquo+M5sUTqsiKRPIBvqb2ZjcK5DUGegM0KhRo8KN3rlC8Omnn3LNNdcwd+7chNMrVKiw2wG+SZMmCat8ct5Xq1atiLfAuUA6k0Wiyszc18hVAJoArYEGwEeSjjGz9UAjM1sp6VDgPUnzzWzpbgszexJ4EiAjI6PwLqR2bi/t3LmTIUOG0Lt3b2rVqkWfPn2oW7fur5JAjRo1vN7flQjpTBbLgYaR4QbAygRlPjGz7cBXkhYTJI+ZZrYSwMyWSZoGHA8sxblibuXKlXTq1ImpU6dy8cUXM3r0aA488MC4w3Jur6SzNWsm0ETSIZIqAZcDY3OVGQOcBSCpJkG11DJJ+0uqHBl/Kru3dThXLI0dO5bmzZvz8ccf88QTT/DWW295onClQtqShZllA12AycAi4HUzWyCpr6Scy2AnA2slLQTeB24zs7XA0UCmpLnh+P7Rq6icK242bdrEX//6Vzp06ECjRo2YPXs2nTt39iomV2qoMPtMiVNGRoZlZmbGHYYrg+bMmUPHjh1ZtGgRPXv25MEHH6Ry5cpxh+VcSiTNMrOMZOX8omrn9tDOnTsZPHgwJ510EuvXr2fq1Kk8/PDDnihcqeTdfTi3B7777js6derElClT6NChA0899RQ1a9aMOyzn0sbPLJwroHfffZfmzZvz0Ucf8cQTT/DOO+94onClnicL51K0adMmbrzxRtq3b0/Dhg29EduVKZ4snEvB3LlzycjIYNSoUfTs2ZP//ve/HHXUUXGH5VyR8WThXD5y7sRu1aoV69evZ8qUKd6I7cokb+B2Lg/RRuz27dvz9NNPe9uEK7P8zMK5BMaNG7erEXvUqFGMGTPGE4Ur0zxZOBexefNmbrrpJi688ELq16/PrFmzuOGGG7wR25V5niycC82bN4+MjAxGjhxJjx49mD59OkcffXTcYTlXLHiycGXezp07efTRR2nZsiXr1q1j8uTJPPLII96I7VyEN3C7Mm3VqlV06tSJyZMn0759e5566ilq1aoVd1jOFTt+ZuHKrPHjx9O8eXM+/PDDXY3YniicS8yThStzNm/eTJcuXWjXrh0HHXSQN2I7lwJPFq5MmTdvHi1btmTEiBF0797dG7GdS5EnC1cmmBlDhw6lVatWrF27lsmTJzN48GBvxHYuRd7A7Uq91atX06lTJyZNmkS7du145plnvG3CuQLyMwtXqo0fP55jjz2WadOmMWLECMaOHeuJwrk94MnClUqbN2/m5ptvpl27dtSrV4/MzExuvPFGb8R2bg+lNVlIaitpsaQlknrlUeZPkhZKWiDp5cj4qyV9Gb6uTmecrnSZP38+rVq1Yvjw4dxyyy1Mnz6dZs2axR2WcyVa2tosJJUHRgBtgOXATEljzWxhpEwToDdwqpn9IKl2OP4AoA+QARgwK5z3h3TF60o+M+Oxxx7j9ttvp0aNGkycOJG2bdvGHZZzpUI6zyxaAUvMbJmZbQNeBTrkKnM9MCInCZjZ9+H4c4GpZrYunDYV8G+9y9Pq1au54IIL6NatG23atGH+/PmeKJwrROlMFvWBbyPDy8NxUUcAR0j6t6RPJLUtwLxI6iwpU1JmVlZWIYbuSpIJEybQvHlz3n//fYYPH+6N2M6lQTqTRaKWRMs1XAFoArQGrgCeklQjxXkxsyfNLMPMMvzgUPZs2bKFrl27csEFF1CnTh0yMzO56aabvBHbuTRIZ7JYDjSMDDcAViYo8w8z225mXwGLCZJHKvO6Muyzzz6jZcuWPPbYY3Tr1o0ZM2Z4I7ZzaZTOZDETaCLpEEmVgMuBsbnKjAHOApBUk6BaahkwGThH0v6S9gfOCce5Mi6nETsjI4OsrCwmTpzIo48+SpUqVeIOzblSLW1XQ5lZtqQuBAf58sAzZrZAUl8g08zG8ktSWAjsAG4zs7UAkh4gSDgAfc1sXbpidSXD999/zzXXXMOECRM4//zzefbZZ6ldu3bcYTlXJsjsV00BJVJGRoZlZmbGHYZLk4kTJ9KpUyd+/PFHBg0a5G0TzhUSSbPMLCNZOb+D2xVrW7ZsoVu3bpx//vnUrl2bzMxMunTp4onCuSLmycIVW5999hmtWrVi2LBhdO3alZkzZ3LMMcfEHZZzZZInC1fsmBnDhw+nZcuWrF69mgkTJjB06FBvxHYuRt5FuStWvv/+e6699lrGjRvH+eefzzPPPEOdOnXiDsu5Ms/PLFyxMXnyZJo3b87UqVMZNmwY48aN80ThXDHhycLFbsuWLXTv3p22bdtSq1YtZs6cyc033+yN2M4VI14N5WK1YMECOnbsyLx587j55psZMGAAVatWjTss51wufmbhYmFmjBgxgoyMDFatWsX48eMZNmyYJwrniik/s3BFLisri7/85S+MGzeO8847j2effdbbJpwr5vzMwhWpyZMnc+yxxzJ16lSGDh3K+PHjPVE4VwJ4snBFYuvWrbsasWvWrMnMmTPp2rWrN2I7V0J4NZRLu4ULF9KxY0fmzp1Lly5dGDhwoLdNOFfC+JmFSxszY9SoUZx44omsXLmScePG8dhjj3micK4ESposJHlnPK7AsrKyuOiii7jxxhtp3bo18+bN44ILLog7LOfcHkrlzOJxSTMk3Rg+8tS5fE2ZMoXmzZszadIkHn30UcaPH0/dunXjDss5txeSJgszOw24kuAxp5mSXpbUJu2RuRJn69at9OjRg3PPPZcDDjiAmTNn0q1bN8qV89pO50q6lBq4zexLSXcDmcAw4HgFl7HcaWZvpzNAVzJEG7FvuukmHn74YW+bcK4USaXNormkIcAi4HfAhWZ2dPh+SJrjc8VctBF7xYoVvPvuuwwfPtwThXOlTCr1A8OB2cBxZnaTmc0GMLOVwN35zSipraTFkpZI6pVgeidJWZLmhK/rItN2RMaPLdhmuaKwZs2aXY3YZ555JvPnz6ddu3Zxh+WcS4NUqqHOBzab2Q4ASeWAKma2ycxeyGsmSeWBEUAbYDkwU9JYM1uYq+hrZtYlwSI2m1mLlLbCFbmpU6dy1VVXsW7dOoYMGULXrl29bcK5UiyVb/c/gWidwj7huGRaAUvMbJmZbQNeBToUPERX3Nx5552cc845HHDAAcyYMYNbbrnFE4VzpVwq3/AqZvZzzkD4fp8U5qsPfBsZXh6Oy+0SSfMkvSmpYXS9kjIlfSLpohTW54rApEmT+Nvf/sY111xDZmYmxx13XNwhOeeKQCrJYqOkE3IGJJ0IbE5hvkSd/liu4XeBxmbWnOBs5fnItEZmlgF0BB6VdNivViB1DhNKZlZWVgohub2RnZ3NrbfeymGHHcaoUaO8Edu5MiSVNotbgDckrQyH6wGXpTDfcoJ7M3I0AFZGC5jZ2sjgaGBAZNrK8O8ySdOA44GlueZ/EngSICMjI3cicoVs9OjRLFy4kLfffpvKlSvHHY5zrgglTRZmNlPSUcCRBGcLn5vZ9hSWPRNoIukQYAVwOcFZwi6S6pnZd+Fge4LLc5G0P7DJzLZKqgmcCgxMcZtcGqxfv557772XM888k4su8lpB58qaVHudPRJoClQhuCEPM/t7fjOYWbakLsBkoDzwjJktkNQXyDSzsUBXSe2BbGAd0Cmc/WjgCUk7CarK+ie4isoVoYceeoi1a9cyePBg71bcuTJIZvnX3kjqA7QmSBYTgPOAj83s0rRHVwAZGRmWmZkZdxil0rJlyzj66KPp2LEjzz77bNzhOOcKkaRZYftwvlJp4L4UOBtYZWbXAMcBXmFdhtx+++1UqFCBfv36xR2Kcy4mqSSLzWa2E8iW9Bvge+DQ9IbliouPPvqIt956izvuuIODDjoo7nCcczFJpc0iM+yafDQwC/gZmJHWqFyxsHPnTrp37079+vXp2bNn3OE452KUb7IIe5b9m5mtJ3iuxSTgN2Y2r0iic7F66aWXmDVrFn//+9/ZZ59U7sN0zpVWqTRwzzKzE4sonj3mDdyFa+PGjRx55JHUq1eP6dOne3cezpVSqTZwp1IN9YmklmY2sxDiciXEoEGDWLFiBa+88oonCudcSsniLOD/JH0NbCS4Mc/CLjpcKbRixQoGDhzIpZdeyumnnx53OM65YiCVZHFe2qNwxcpdd91FdnY2AwYMSF7YOVcmpFK/YHm8XCk0a9Ysnn/+ebp168ahh/oV0s65QCpnFuMJkoMIuvs4BFgMNEtjXC4GZkaPHj2oWbMmd911V9zhOOeKkVQ6Ejw2Ohx2V/5/aYvIxWbMmDF8+OGHjBw5kv322y/ucJxzxUjSS2cTziTNNrMTkpcsOn7p7N7ZunUrzZo1o3LlysydO5cKFVLtY9I5V5IV2qWzknpEBssBJwD+pKFSZvjw4SxdupSJEyd6onDO/UoqR4XqkffZBG0Yb6UnHBeHNWvW8MADD9C2bVvatm0bdzjOuWIolTaL+4siEBef++67j59//plHHnkk7lCcc8VU0ktnJU0NOxLMGd5f0uT0huWKyqJFi3j88cfp3LkzTZs2jTsc51wxlcp9FrXCjgQBMLMfgNrpC8kVpZ49e1KtWjXuv99PIJ1zeUslWeyQ1ChnQNLB+E15pcKUKVOYMGECd999N7Vq1Yo7HOdcMZZKA/ddwMeSPgiHzwA6py8kVxSys7Pp0aMHhxxyCF27do07HOdcMZf0zMLMJhFcLvsa8Dpwopml1GYhqa2kxZKWSOqVYHonSVmS5oSv6yLTrpb0Zfi6OvVNcql4+umnWbBgAQMHDqRyZX9KrnMuf6k8z+Ji4D0z+zEcrgG0NrMxSeYrD3wBtAGWAzOBK8xsYaRMJyDDzLrkmvcAIBPIIKjymkWQpH7Ia31+U17qNmzYwOGHH85RRx3FBx98QPCMK+dcWZTqTXmptFn0yUkUAGFjd58U5msFLDGzZWa2DXgV6JDCfADnAlPNbF2YIKYCfgNAIXnooYfIyspi8ODBniiccylJJVkkKpNKW0d94NvI8PJwXG6XSJon6U1JDQs4ryugr776iiFDhvDnP/+ZjIykPyaccw5ILVlkShos6TBJh0oaQlAtlEyin6y567zeBRqHD1L6J/B8AeZFUmdJmZIys7K8B5JU9OrVi/Lly/PQQw/FHYpzrgRJJVncDGwjaOB+A9gC3JjCfMuBhpHhBsDKaAEzW2tmW8PB0cCJqc4bzv+kmWWYWYZf+pncv//9b15//XVuu+02GjRoEHc4zrkSpMC9zkqqAlxoZm8kKVeBoIH7bGAFQQN3RzNbEClTz8y+C99fDNxhZieHDdyzCK7CAphN0MC9Lq/1eQN3/nbu3Mkpp5zC8uXL+eKLL6hWrVrcITnnioFC63U2XFh54BzgivDvxwRnGXkys2xJXYDJQHngGTNbIKkvkGlmY4GuktoTdFC4DugUzrtO0gMECQagb36JwiX3yiuvMGPGDJ577jlPFM65Asv3zELSGUBH4AJgBnAqcKiZbSqa8FLnZxZ527RpE0ceeSS1a9dm5syZlCuXSu2jc64s2OszC0nLgW+AUcBtZvaTpK+KY6Jw+Rs8eDDLly/nxRdf9EThnNsj+R053iK4XPUy4EJJ1fA+oUqclStX0r9/fy6++GLOPPPMuMNxzpVQeSYLM+sGNAYGA2cRNFbXkvQnSfsWTXhub91zzz1s27aNgQMHxh2Kc64Ey7dOwgLvmdn1BImjI3AR8L/0h+b21qeffsqzzz5L165dOfzww+MOxzlXghX40lkASVXNbHMa4tlj3sC9OzPj7LPPZt68eSxZsoQaNWokn8k5V+YU6qWzuRW3ROF+bezYsbz//vsMHz7cE4Vzbq/t0ZlFceRnFr/Ytm0bzZo1o0KFCsybN4+KFSvGHZJzrpgq9DMLSdXMbOPeheWKwsiRI1myZAnjx4/3ROGcKxRJL7qX9FtJC4FF4fBxkkamPTK3R9auXcv9999PmzZtOO+88+IOxzlXSqRyh9YQgudLrAUws7kEj1Z1xVDfvn3ZsGGDP6vCOVeoUrqd18y+zTVqRxpicXtp8eLFjBw5kuuvv55jjjkm7nCcc6VIKm0W30r6LWCSKgFdCaukXPFy2223UbVqVe6///64Q3HOlTKpnFncANxE0PXHcqBFOOyKkX/961+8++673HXXXdSpUyfucJxzpYxfOlsK7NixgxNOOIENGzawaNEiqlSpEndIzrkSotAunZU0LMHoHwmeSfGPPQnOFa5nn32WefPm8dprr3micM6lRSrVUFUIqp6+DF/NgQOAayU9msbYXAp++ukn7r77bn7729/yxz/+Me5wnHOlVCoN3IcDvzOzbABJo4ApQBtgfhpjcyno378/q1evZuzYsX6prHMubVI5s6gPRJ/DWQ04yMx2AFvTEpVLyddff80jjzzClVdeSatWreIOxzlXiqVyZjEQmCNpGiCCG/IeCh+G9M80xuaS6NWrF5J46KGH4g7FOVfKJT2zMLOngd8CY8LXaWb2lJltNLPb8ptXUltJiyUtkdQrn3KXSjJJGeFwY0mbJc0JX48XbLNKv/9BX3DmAAAW50lEQVT+97+8+uqr9OzZk0aNGsUdjnOulEu1I8EtwHcEjd2HSzrczD7MbwZJ5YERBG0by4GZksaa2cJc5aoT3Og3PdcilppZixTjK1PMjB49elC3bl3uuOOOuMNxzpUBqVw6ex3QDWgAzAFOBv4L/C7JrK2AJWa2LFzOq0AHYGGucg8QVHX1LFDkZdhrr73GJ598wtNPP82++/oTbp1z6ZdKA3c3oCXwtZmdBRwPZKUwX30g2qfU8nDcLpKOBxqa2bgE8x8i6VNJH0g6PdEKJHWWlCkpMysrlZBKvs2bN3PHHXfQokULrr766rjDcc6VEalUQ20xsy2SkFTZzD6XdGQK8yW6jnPX7eKSyhH0aNspQbnvgEZmtlbSicAYSc3MbMNuCzN7EngSgju4U4ipxBsyZAjffPMNzz33HOXLl487HOdcGZFKslguqQZB4/ZUST8AK1OZD2gYGW6Qa77qwDHAtPD+gLrAWEntzSyT8LJcM5slaSlwBFA2+/MIrVq1ir/97W906NCBs846K+5wnHNlSNJkYWYXh2/vk/Q+sB8wKYVlzwSaSDoEWAFcDnSMLPdHoGbOcHhpbk8zy5RUC1hnZjskHQo0AZaltkml1z333MOWLVsYOHBg3KE458qYfJNFWFU0z8yOATCzD1JdsJllS+oCTAbKA8+Y2QJJfQn6lRqbz+xnAH0lZRM8O+MGM1uX6rpLo7lz5/L000/TrVs3jjjiiLjDcc6VMUl7nZX0EtDbzL4pmpD2TGnuddbMaNOmDZ9++ilLlixh//33jzsk51wpUWi9zgL1gAWSZgAbc0aaWfu9iM8VwPjx4/nXv/7F0KFDPVE452KRypnFmYnGF6RKqiiU1jOL7du3c+yxx2JmfPbZZ1SsWDHukJxzpUihnVmY2QeSDgaamNk/Je1D0AbhisCoUaNYvHgxY8eO9UThnItN0pvyJF0PvAk8EY6qT3AZrUuzdevWcd9993H22WfTrl27uMNxzpVhqdzBfRNwKrABwMy+BGqnMygXeOCBB1i/fj2DBw/2Z1U452KVSrLYambbcgYkVSByJ7ZLjy+++ILhw4dz7bXX0rx587jDcc6Vcakkiw8k3QlUldQGeAN4N71hudtvv50qVarwwAMPxB2Kc86llCx6EXQcOB/4P2ACcHc6gyrr3n//ff7xj3/Qu3dv6tatG3c4zjmX0qWzFwMTzKxYP0K1tFw6u2PHDjIyMli3bh2ff/45VatWjTsk51wpluqls6mcWbQHvpD0gqQLwjYLlybPP/88c+bMoX///p4onHPFRtIzCwBJFYHzgMuA04CpZnZdmmMrkNJwZvHzzz/TpEkTGjduzH/+8x+/Aso5l3aF2d0HZrZd0kSCq6CqEjzxrlgli9JgwIABrFq1infeeccThXOuWEnlpry2kp4DlgCXAk8R9BflCtG3337LoEGDuPzyyzn55JPjDsc553aTyplFJ+BV4P+KeyN3Sda7d2/MjP79+8cdinPO/UrSMwszu9zMxuQkCkmnShqR/tDKjhkzZvDSSy/Ro0cPDj744LjDcc65X0mpzUJSC4Kn3P0J+Ap4O51BlSVmRvfu3alTpw69e/eOOxznnEsoz2Qh6QiCR6FeAawFXiO4esof/lyI3njjDf7zn/8wevRoqlevHnc4zjmXUJ6XzkraCXwEXGtmS8Jxy8zs0CKML2Ul8dLZLVu2cPTRR/Ob3/yG2bNnU7689/zunCtahXHp7CUEZxbvS5pE0Mjt13MWoqFDh/K///2PqVOneqJwzhVreTZwm9k7ZnYZcBQwDegO1JE0StI5qSw8vOx2saQlknrlU+5SSSYpIzKudzjfYknnprxFJcTq1avp168f7dq14/e//33c4TjnXL5SuRpqo5m9ZGbtgAbAHILOBfMlqTwwguDO76bAFZKaJihXHegKTI+Ma0pwVtMMaAuMDJdXavTp04fNmzczaNCguENxzrmkUukbahczW2dmT5jZ71Io3gpYYmbLwudhvEpw53duDwADgS2RcR2AV81sq5l9RXBDYKuCxFqczZ8/n9GjR/PXv/6VI488Mu5wnHMuqQIliwKqD3wbGV4ejttF0vFAQzMbV9B5w/k7S8qUlJmVlVU4UaeZmXHrrbey33770adPn7jDcc65lKQzWSRqDN916ZWkcsAQ4NaCzrtrhNmTZpZhZhm1atXa40CL0sSJE5k6dSr33nsvBx54YNzhOOdcStLZ3fhyoGFkuAGwMjJcHTgGmBZ2mlcXGCupfQrzlkjbt2/n1ltvpUmTJtx4441xh+OccylLZ7KYCTSRdAiwgqDBumPORDP7EaiZMyxpGtDTzDIlbQZeljQYOAhoAsxIY6xF4sknn+Tzzz9nzJgxVKpUKe5wnHMuZWlLFmaWLakLMBkoDzxjZgsk9QUyzWxsPvMukPQ6sBDIBm4ysx3pirUo/PDDD/Tp04fWrVvTvn37uMNxzrkCSetT78xsAsEzu6Pj7s2jbOtcw/2AfmkLroj169ePdevWMWTIEH9WhXOuxElnA7cLLVmyhGHDhnHNNdfQokWLuMNxzrkC82RRBO644w4qVarEgw8+GHcozjm3RzxZpNkHH3zA22+/Ta9evahXzx8w6JwrmTxZpNHOnTvp0aMHDRo0oEePHnGH45xzeyytDdxl3QsvvMDs2bN58cUX2WeffeIOxznn9pifWaTJxo0bufPOO2nVqhVXXHFF3OE459xe8TOLNHn44YdZuXIlr7/+OuXKeU52zpVsfhRLg+XLlzNw4ED+9Kc/ceqpp8YdjnPO7TVPFmlw5513smPHDvr37x93KM45Vyg8WRSyzMxMXnjhBbp3784hhxwSdzjOOVcoPFkUIjOje/fu1KpVizvvvDPucJxzrtB4A3chevvtt/n44495/PHH+c1vfhN3OM45V2hk9qtnCpVIGRkZlpmZGdv6t27dStOmTdlnn3349NNPqVDB87BzrviTNMvMMpKV8yNaIRk2bBjLli1j8uTJniicc6WOt1kUgqysLB588EHOP/98zjnnnLjDcc65QufJohD06dOHjRs3MmjQoLhDcc65tPBksZcWLFjAE088wQ033MDRRx8ddzjOOZcWniz2Us+ePalevTr33Xdf3KE451zaeEvsXpg0aRKTJk1i0KBB1KxZM+5wnHMubdJ6ZiGpraTFkpZI6pVg+g2S5kuaI+ljSU3D8Y0lbQ7Hz5H0eDrj3BPZ2dnceuutHHbYYXTp0iXucJxzLq3SdmYhqTwwAmgDLAdmShprZgsjxV42s8fD8u2BwUDbcNpSMyu2D6wePXo0Cxcu5K233qJy5cpxh+Occ2mVzjOLVsASM1tmZtuAV4EO0QJmtiEyWA0oEXcI/vjjj9x7772cccYZXHzxxXGH45xzaZfOZFEf+DYyvDwctxtJN0laCgwEukYmHSLpU0kfSDo90QokdZaUKSkzKyurMGPPV79+/Vi7di2DBw9GUpGt1znn4pLOZJHoKPqrMwczG2FmhwF3AHeHo78DGpnZ8UAP4GVJv+psycyeNLMMM8uoVatWIYaet2XLljF06FCuuuoqTjzxxCJZp3POxS2dyWI50DAy3ABYmU/5V4GLAMxsq5mtDd/PApYCR6QpzgK54447qFChAv369Ys7FOecKzLpTBYzgSaSDpFUCbgcGBstIKlJZPAC4MtwfK2wgRxJhwJNgGVpjDUlH330EW+++Sa333479ev/qkbNOedKrbRdDWVm2ZK6AJOB8sAzZrZAUl8g08zGAl0k/R7YDvwAXB3OfgbQV1I2sAO4wczWpSvWVOzcuZMePXpQv359evbsGWcozjlX5NJ6U56ZTQAm5Bp3b+R9tzzmewt4K52xFdRLL71EZmYmzz//PNWqVYs7HOecK1L+PIsUbNq0iSOOOIJ69eoxffp0ypXzXlKcc6WDP8+iEA0aNIgVK1bwyiuveKJwzpVJfuRLYuXKlQwYMIBLLrmE009PeLuHc86Vep4skrjrrrvIzs5mwIABcYfinHOx8WSRj9mzZ/P888/TtWtXDjvssLjDcc652HiyyIOZ0aNHDw488EDuvvvu5DM451wp5g3ceRgzZgwffPABI0eOZL/99os7HOeci5VfOpvAtm3baNq0KZUrV2bu3LlUqOA51TlXOvmls3th+PDhLF26lIkTJ3qicM45vM3iV9asWUPfvn0599xzadu2bfIZnHOuDPBkkcv999/PTz/9xCOPPBJ3KM45V2x4sohYtGgRo0aNonPnzjRr1izucJxzrtjwZBFx2223Ua1aNfr27Rt3KM45V6x4621o6tSpjB8/noEDB1JUT91zzrmSwi+dBbKzszn++OPZuHEjixYtonLlyoUcnXPOFU9+6WwBPPPMM3z22We88cYbniiccy6BMt9msWHDBu655x5OO+00LrnkkrjDcc65YqnMJ4uNGzdy6qmnMnjwYCTFHY5zzhVLZb4aql69erz99ttxh+Gcc8VaWs8sJLWVtFjSEkm9Eky/QdJ8SXMkfSypaWRa73C+xZLOTWeczjnn8pe2ZCGpPDACOA9oClwRTQahl83sWDNrAQwEBofzNgUuB5oBbYGR4fKcc87FIJ1nFq2AJWa2zMy2Aa8CHaIFzGxDZLAakHMdbwfgVTPbamZfAUvC5TnnnItBOtss6gPfRoaXAyflLiTpJqAHUAn4XWTeT3LNWz/BvJ2BzgCNGjUqlKCdc879WjrPLBJdWvSrOwDNbISZHQbcAeQ8ki7VeZ80swwzy/C7rp1zLn3SmSyWAw0jww2AlfmUfxW4aA/ndc45l0bpTBYzgSaSDpFUiaDBemy0gKQmkcELgC/D92OByyVVlnQI0ASYkcZYnXPO5SNtbRZmli2pCzAZKA88Y2YLJPUFMs1sLNBF0u+B7cAPwNXhvAskvQ4sBLKBm8xsR7pidc45l79S05GgpCzg671YRE1gTSGFU5g8roLxuArG4yqY0hjXwWaWtNG31CSLvSUpM5WeF4uax1UwHlfBeFwFU5bjKvN9QznnnEvOk4VzzrmkPFn84sm4A8iDx1UwHlfBeFwFU2bj8jYL55xzSfmZhXPOuaQ8WTjnnEuqTCWLFJ6vUVnSa+H06ZIaF5O4OknKCp/7MUfSdUUU1zOSvpf0WR7TJWlYGPc8SScUk7haS/oxsr/uLaK4Gkp6X9IiSQskdUtQpsj3WYpxFfk+k1RF0gxJc8O47k9Qpsi/kynGFct3Mlx3eUmfShqXYFr69peZlYkXwV3kS4FDCXq4nQs0zVXmRuDx8P3lwGvFJK5OwPAY9tkZwAnAZ3lMPx+YSNDx48nA9GISV2tgXAz7qx5wQvi+OvBFgv9lke+zFOMq8n0W7oN9w/cVgenAybnKxPGdTCWuWL6T4bp7AC8n+n+lc3+VpTOLpM/XCIefD9+/CZwtpf3B3KnEFQsz+xBYl0+RDsDfLfAJUENSvWIQVyzM7Dszmx2+/wlYxK+71i/yfZZiXEUu3Ac/h4MVw1fuK26K/DuZYlyxkNSAoB+9p/Iokrb9VZaSRaLna+T+wuwqY2bZwI/AgcUgLoBLwmqLNyU1TDA9DqnGHodTwmqEiZKaFfXKw9P/4wl+lUbFus/yiQti2Gdhlcoc4Htgqpnlub+K8DuZSlwQz3fyUeB2YGce09O2v8pSskjlGRkpPUejkKWyzneBxmbWHPgnv/xyiFsc+ysVswn6uzkOeAwYU5Qrl7Qv8BZwi+3+NEiIcZ8liSuWfWZmOyx4rHIDoJWkY3IViWV/pRBXkX8nJbUDvjezWfkVSzCuUPZXWUoWqTwjY1cZSRWA/Uh/dUfSuMxsrZltDQdHAyemOaZUFcvnjpjZhpxqBDObAFSUVLMo1i2pIsEB+SUzeztBkVj2WbK44txn4TrXA9OAtrkmxfGdTBpXTN/JU4H2kv5HUF39O0kv5iqTtv1VlpJF0udrhMNXh+8vBd6zsKUozrhy1Wm3J6hzLg7GAleFV/icDPxoZt/FHZSkujn1tJJaEXzO1xbBegU8DSwys8F5FCvyfZZKXHHsM0m1JNUI31cFfg98nqtYkX8nU4krju+kmfU2swZm1pjgOPGemf2/XMXStr/S+QzuYsVSe77G08ALkpYQZOPLi0lcXSW1J3i2xzqCKzHSTtIrBFfJ1JS0HOhD0NiHmT0OTCC4umcJsAm4ppjEdSnwV0nZwGbg8iJI+hD88vszMD+s7wa4E2gUiS2OfZZKXHHss3rA85LKEySn181sXNzfyRTjiuU7mUhR7S/v7sM551xSZakayjnn3B7yZOGccy4pTxbOOeeS8mThnHMuKU8WzjnnkvJk4WITXtv/qqSlkhZKmiDpiLjjyoukgyS9uYfzdpJ0UGT4KUlNCyGmaO+nn0vqvrfLTGGdrRP1eOpKN08WLhbhDWDvANPM7DAza0pw7X+deCPLm5mtNLNL93D2TsCuZGFm15nZwkIJLOhZtAXB/RR3FaO+w1wp4snCxeUsYHt4QxgAZjbHzD4K725+WNJnkuZLugx2/aL9QNLrkr6Q1F/SlQqePTBf0mFhueckjVLwDIdlks5U8AyMRZKey1mfpJ8j7y/NmRbOP0zSf8L5Lw3HN1b4DA0FHc0NCtc7T9LN4fh7Jc0MY38y3JZLgQzgpfAMoKqkaZIywnmuCJfzmaQB0fgk9VPQud8nkvJNpGa2luBmv3rh/AdL+lcY378kNYps366kl7Mfwv07TUHHeJ9LeilM6jnPXPlc0sfAHyLznqlfnunwqaTqqf37XUnjycLF5Rggrw7R/gC0AI4j6GrhYf3SvcJxQDfgWIK7ko8ws1YEXTbfHFnG/sDvgO4Enb4NAZoBx0pqkUJ89YDTgHZA/wTTOwOHAMeHncm9FI4fbmYtzewYoCrQzszeBDKBK82shZltzllIWDU1IIy1BdBS0kXh5GrAJ2Hnfh8C1+cXcJgMqgDzcmIh6A49J75hKWz38cAtQFOCZ6ycKqkKQf9HFwKnA3Uj5XsCN4VnNqcT3P3tSiFPFq44Og14Jez5czXwAdAynDYzfD7DVoKHRk0Jx88HGkeW8W7YXcV8YLWZzTezncCCXOXyMsbMdoZVRYl+0f+e4CEz2QBmltNZ21kKnlA2nyABJOvquyVBVVxWuKyXCB7uBLANyGkbmJVP3JdJWgAsA4aa2ZZw/CkED8kBeIFgvyYzw8yWh/tqTrjOo4CvzOzLcJ9GO6/7NzBYUlegRs7+cKWPJwsXlwXk3VNnfg9r2Rp5vzMyvJPd+zrbmqBM7nLRvm6q5LOeRPEo1/yEv8BHApea2bEEv8ZzLzfRcvKyPdI/0w7y7svtNTNrRvDL/hFJdfMol7OsbMLvfljNVClSJrrd0XUm7BfIzPoD1xGcRX0i6ai8N8eVZJ4sXFzeAypL2lW1IqmlpDMJqlwuC9sFahH80p6RhhhWSzpaUjng4gLOOwW4QUE30Eg6gF8SwxoFz46INob/RPBI09ymA2dKqqmg47orCM6kCszM/ktwBpHzjO3/8EtHclcCH4fv/8cviboDYSeM+fgcOCSnTSiMEQBJh4VnbQMIqto8WZRSnixcLMJfzBcDbRRcOrsAuI/g2Q7vENS7zyVIKreb2ao0hNGLoJrnPaCg3YQ/BXwDzJM0F+gYPvtgNEHV1xiC7udzPAc8ntPAnTMy7J68N/A+wfbONrN/7NnmAEH7xzVhQ3PX8P08gvadnCQymiBBzQBOAjbmt8CwWqszMD5s4P46MvmWsGF+LkF7xcS9iN0VY97rrHPOuaT8zMI551xSniycc84l5cnCOedcUp4snHPOJeXJwjnnXFKeLJxzziXlycI551xS/x9nblcMf3DJ7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot Average Accuracy vs Communication rounds\n",
    "plt.figure()\n",
    "plt.title('Average Accuracy vs Communication rounds')\n",
    "plt.plot(range(len(train_accuracy)), train_accuracy, color='k')\n",
    "plt.ylabel('Average Accuracy')\n",
    "plt.xlabel('Communication Rounds')\n",
    "# plt.savefig('../save/fed_{}_{}_{}_C[{}]_iid[{}]_E[{}]_B[{}]_acc.png'.\n",
    "#             format(args.dataset, args.model, args.epochs, args.frac,\n",
    "#                    args.iid, args.local_ep, args.local_bs))\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fl_pytorch",
   "language": "python",
   "name": "fl_pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}