{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-\n",
    "# Python version: 3.6\n",
    "\n",
    "\n",
    "import os\n",
    "import copy\n",
    "import time\n",
    "import pickle\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "from options import args_parser\n",
    "from update import LocalUpdate, test_inference\n",
    "from models import MLP, CNNMnist, CNNFashion_Mnist, CNNCifar\n",
    "from utils import get_dataset, average_weights, exp_details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Namespace(dataset='mnist', epochs=5, frac=0.1, gpu=None, iid=1, local_bs=10, local_ep=3, lr=0.01, model='mlp', num_classes=10, num_clusters=2, num_users=100, optimizer='sgd', unequal=0, verbose=1)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import argparse\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "# parser.add_argument('--name', '-n', default='foo', help='foo')\n",
    "parser.add_argument('--model', type=str, default='mlp', help='model name(mlp or cnn)')\n",
    "parser.add_argument('--dataset', type=str, default='mnist', help=\"name of dataset(mnist or cifar)\")\n",
    "parser.add_argument('--epochs', type=int, default=5, help=\"number of rounds of training(10)\")\n",
    "parser.add_argument('--num_users', type=int, default=100, help=\"number of users: K\")\n",
    "parser.add_argument('--num_classes', type=int, default=10, help=\"number of classes\")\n",
    "parser.add_argument('--gpu', default=None, help=\"To use cuda, set to a specific GPU ID. Default set to use CPU.\")\n",
    "parser.add_argument('--frac', type=float, default=0.1, help='the fraction of clients: C')\n",
    "parser.add_argument('--local_ep', type=int, default=3, help=\"the number of local epochs: E\")\n",
    "parser.add_argument('--local_bs', type=int, default=10, help=\"local batch size: B\")\n",
    "parser.add_argument('--lr', type=float, default=0.01, help='learning rate')\n",
    "parser.add_argument('--optimizer', type=str, default='sgd', help=\"type of optimizer\")\n",
    "parser.add_argument('--verbose', type=int, default=1, help='verbose')\n",
    "parser.add_argument('--iid', type=int, default=1, help='Default set to IID. Set to 0 for non-IID.')\n",
    "parser.add_argument('--unequal', type=int, default=0, help='whether to use unequal data splits for  \\\n",
    "                        non-i.i.d setting (use 0 for equal splits)')\n",
    "# parser.add_argument('--seed', type=int, default=1, help='random seed')\n",
    "\n",
    "parser.add_argument('--num_clusters', type=int, default=2, help='verbose')\n",
    "\n",
    "\n",
    "# args = parser.parse_args([])\n",
    "args, _ = parser.parse_known_args()\n",
    "\n",
    "args\n",
    "# --dataset=mnist --gpu=0 --iid=0 --epochs=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start time:  1571373590.742761\n"
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "start_time = time.time()\n",
    "print(\"start time: \", start_time)\n",
    "\n",
    "# define paths\n",
    "path_project = os.path.abspath('..')\n",
    "logger = SummaryWriter('../logs')\n",
    "\n",
    "# args = args_parser()\n",
    "# exp_details(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if args.gpu:\n",
    "    torch.cuda.set_device(args.gpu)\n",
    "device = 'cuda' if args.gpu else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataset and user groups\n",
    "train_dataset, test_dataset, user_groups = get_dataset(args)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "keys = user_groups.keys()\n",
    "keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "l = np.arange(5,6)\n",
    "l\n",
    "{k:user_groups[k] for k in l if k in user_groups}"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "np.arange(cluster_size, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Each cluster size:  50.0\n",
      "Size of cluster 1:  50\n",
      "Size of cluster 2:  50\n"
     ]
    }
   ],
   "source": [
    "# Splitting into clusters. FL groups\n",
    "cluster_size = args.num_users / args.num_clusters\n",
    "print(\"Each cluster size: \", cluster_size)\n",
    "\n",
    "# Cluster 1\n",
    "A1 = np.arange(cluster_size, dtype=int)\n",
    "user_groupsA = {k:user_groups[k] for k in A1 if k in user_groups}\n",
    "print(\"Size of cluster 1: \", len(user_groupsA))\n",
    "# Cluster 2\n",
    "B1 = np.arange(cluster_size, cluster_size+cluster_size, dtype=int)\n",
    "user_groupsB = {k:user_groups[k] for k in B1 if k in user_groups}\n",
    "print(\"Size of cluster 2: \", len(user_groupsB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49])\n",
      "dict_keys([50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99])\n"
     ]
    }
   ],
   "source": [
    "# Check that clusters are all different\n",
    "print(user_groupsA.keys())\n",
    "print(user_groupsB.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BUILD MODEL\n",
    "def build_model(args, train_dataset):\n",
    "    if args.model == 'cnn':\n",
    "        # Convolutional neural network\n",
    "        if args.dataset == 'mnist':\n",
    "            global_model = CNNMnist(args=args)\n",
    "        elif args.dataset == 'fmnist':\n",
    "            global_model = CNNFashion_Mnist(args=args)\n",
    "        elif args.dataset == 'cifar':\n",
    "            global_model = CNNCifar(args=args)\n",
    "\n",
    "    elif args.model == 'mlp':\n",
    "        # Multi-layer preceptron\n",
    "        img_size = train_dataset[0][0].shape\n",
    "        len_in = 1\n",
    "        for x in img_size:\n",
    "            len_in *= x\n",
    "            global_model = MLP(dim_in=len_in, dim_hidden=200,\n",
    "                               dim_out=args.num_classes)\n",
    "    else:\n",
    "        exit('Error: unrecognized model')\n",
    "        \n",
    "    return global_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199210\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Linear-1                  [-1, 200]         157,000\n",
      "           Dropout-2                  [-1, 200]               0\n",
      "              ReLU-3                  [-1, 200]               0\n",
      "            Linear-4                  [-1, 200]          40,200\n",
      "           Dropout-5                  [-1, 200]               0\n",
      "              ReLU-6                  [-1, 200]               0\n",
      "            Linear-7                   [-1, 10]           2,010\n",
      "           Softmax-8                   [-1, 10]               0\n",
      "================================================================\n",
      "Total params: 199,210\n",
      "Trainable params: 199,210\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.01\n",
      "Params size (MB): 0.76\n",
      "Estimated Total Size (MB): 0.77\n",
      "----------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<generator object Module.parameters at 0x7fed31502d68>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MODEL PARAM SUMMARY\n",
    "global_model = build_model(args, train_dataset)\n",
    "pytorch_total_params = sum(p.numel() for p in global_model.parameters())\n",
    "print(pytorch_total_params)\n",
    "\n",
    "from torchsummary import summary\n",
    "\n",
    "summary(global_model, (1, 28, 28))\n",
    "global_model.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP(\n",
      "  (layer_input): Linear(in_features=784, out_features=200, bias=True)\n",
      "  (relu): ReLU()\n",
      "  (dropout): Dropout(p=0.5, inplace=False)\n",
      "  (layer_hidden1): Linear(in_features=200, out_features=200, bias=True)\n",
      "  (layer_hidden2): Linear(in_features=200, out_features=10, bias=True)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Set the model to train and send it to device.\n",
    "global_model.to(device)\n",
    "global_model.train()\n",
    "print(global_model)\n",
    "\n",
    "# copy weights\n",
    "global_weights = global_model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the cluster models to train and send it to device.\n",
    "cluster_modelA = build_model(args, train_dataset)\n",
    "cluster_modelA.to(device)\n",
    "cluster_modelA.train()\n",
    "# copy weights\n",
    "cluster_modelA_weights = cluster_modelA.state_dict()\n",
    "\n",
    "# Set the cluster models to train and send it to device.\n",
    "cluster_modelB = build_model(args, train_dataset)\n",
    "cluster_modelB.to(device)\n",
    "cluster_modelB.train()\n",
    "# copy weights\n",
    "cluster_modelB_weights = cluster_modelA.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "print(np.random.choice(range(args.num_users), m, replace=False))\n",
    "np.random.choice(A1, m, replace=False)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "len(A1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the training function\n",
    "def fl_train(args, train_dataset, cluster_global_model, cluster, usergrp, epochs):\n",
    "#     print(args)\n",
    "    \n",
    "#     # Set the model to train and send it to device.\n",
    "#     cluster_global_model = build_model(args, train_dataset)\n",
    "#     cluster_global_model.to(device)\n",
    "#     cluster_global_model.train()\n",
    "\n",
    "#     # copy weights\n",
    "#     cluster_global_weights = cluster_global_model.state_dict()\n",
    "    \n",
    "    cluster_train_loss, cluster_train_accuracy = [], []\n",
    "    cluster_val_acc_list, cluster_net_list = [], []\n",
    "    cluster_cv_loss, cluster_cv_acc = [], []\n",
    "    print_every = 2\n",
    "    cluster_val_loss_pre, counter = 0, 0\n",
    "\n",
    "    for epoch in tqdm(range(epochs)):\n",
    "        cluster_local_weights, cluster_local_losses = [], []\n",
    "        print(f'\\n | Cluster Training Round : {epoch+1} |\\n')\n",
    "\n",
    "        cluster_global_model.train()\n",
    "        m = max(int(args.frac * len(cluster)), 1)\n",
    "        idxs_users = np.random.choice(cluster, m, replace=False)\n",
    "\n",
    "\n",
    "        for idx in idxs_users:\n",
    "            cluster_local_model = LocalUpdate(args=args, dataset=train_dataset, idxs=usergrp[idx], logger=logger)\n",
    "            cluster_w, cluster_loss = cluster_local_model.update_weights(model=copy.deepcopy(cluster_global_model), global_round=epoch)\n",
    "            cluster_local_weights.append(copy.deepcopy(cluster_w))\n",
    "            cluster_local_losses.append(copy.deepcopy(cluster_loss))\n",
    "\n",
    "        # averaging global weights\n",
    "        cluster_global_weights = average_weights(cluster_local_weights)\n",
    "\n",
    "        # update global weights\n",
    "        cluster_global_model.load_state_dict(cluster_global_weights)\n",
    "\n",
    "        cluster_loss_avg = sum(cluster_local_losses) / len(cluster_local_losses)\n",
    "        cluster_train_loss.append(cluster_loss_avg)\n",
    "\n",
    "#         # Calculate avg training accuracy over all users at every epoch\n",
    "#         cluster_list_acc, cluster_list_loss = [], []\n",
    "#         cluster_global_model.eval()\n",
    "        \n",
    "#         for c in range(len(cluster)):\n",
    "#             cluster_local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "#                                       idxs=usergrp[idx], logger=logger)\n",
    "#             cluster_acc, cluster_loss = cluster_local_model.inference(model=cluster_global_model)\n",
    "#             cluster_list_acc.append(cluster_acc)\n",
    "#             cluster_list_loss.append(cluster_loss)\n",
    "#         cluster_train_accuracy.append(sum(cluster_list_acc)/len(cluster_list_acc))\n",
    "    \n",
    "#     return cluster_global_model, cluster_train_loss #cluster_global_weights, cluster_loss_avg, cluster_train_accuracy \n",
    "    return cluster_global_weights, cluster_loss_avg\n",
    "    \n",
    "# A_model, A_weights, A_losses, A_trainacc = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)    \n",
    "# A_model, A_trainloss = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)\n",
    "# A_weights, A_losses = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Defining the evaluation function\n",
    "def fl_eval(train_dataset, global_model, cluster, usergrp, idxs_users):\n",
    "    cluster_list_acc, cluster_list_loss = [], []\n",
    "#     for c in range(len(cluster)):\n",
    "    for c in idxs_users:\n",
    "        cluster_local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "                                  idxs=usergrp[c], logger=logger)\n",
    "        acc, loss = cluster_local_model.inference(model=global_model)\n",
    "        cluster_list_acc.append(acc)\n",
    "        cluster_list_loss.append(loss)\n",
    "#     train_accuracy.append(sum(list_acc)/len(list_acc))\n",
    "\n",
    "    return cluster_list_acc, cluster_list_loss\n",
    "\n",
    "\n",
    "clusterA_list_acc, clusterA_list_loss = fl_eval(train_dataset, global_model, A1, user_groupsA, A_idxs_users)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A/home/wesleyjtann/federated_learning/Hierarchical-Federated-Learning-Quantization/src/update.py:23: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  return torch.tensor(image), torch.tensor(label)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Global Training Round : 1 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.100857\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.100047\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.112111\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.100246\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.104230\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.113424\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.096679\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.107666\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.103782\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.107227\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.103962\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.112365\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.094971\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.113438\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.103372\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.092448\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.101388\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.100194\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.090739\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.112528\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.096536\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.093168\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.103377\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.109103\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.108132\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.102430\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.110719\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.104929\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.110915\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.104475\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.100970\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.106110\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.097208\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.102194\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.099839\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.094566\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.099416\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.106896\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.098361\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.100935\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.099541\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.100186\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.112728\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.097048\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.100526\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.096451\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.094883\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.098996\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.094276\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.107676\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.096422\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.109544\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.096068\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.105284\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.100946\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.104671\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.106957\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.102325\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.110442\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.101427\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.100147\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.102497\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.099932\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.103132\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.111764\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.106003\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.096527\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.107534\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.113402\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.096577\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.100754\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.102313\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.116259\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.110776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.33s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.098218\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.098466\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.109916\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.106987\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.108120\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.112795\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.098149\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.109277\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.107090\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.124893\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.104270\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.112728\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.136079\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.108355\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.104513\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.140901\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.117750\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.101770\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.103844\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.111238\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.118748\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.110553\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.109676\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.124417\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.111813\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.125769\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.110462\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.117090\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.109357\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.129611\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.114711\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.107735\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.104063\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.102355\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.113977\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.108910\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.099600\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.109610\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.104909\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.108242\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.110685\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.105758\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.131436\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.117481\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.110580\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.124219\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.110107\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.116366\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.110332\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.103688\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.099626\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.108546\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.099734\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.119686\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.112350\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.123477\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.105634\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.124847\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.101999\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.120311\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.109085\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.103470\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.101760\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.109335\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.097711\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.101371\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.118686\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.098035\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.107208\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.095571\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.099719\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.107208\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.124590\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.108282\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.119184\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.121970\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.30s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.101080\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.109916\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.097413\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.100618\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.103357\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.093933\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.105164\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.103412\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.098486\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.104743\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.102533\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.095370\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.117858\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.090344\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.107520\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.101319\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.097230\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.095070\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.095487\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.090245\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.096202\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.098904\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.106770\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.108324\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.092899\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.112811\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.097211\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.114134\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.102231\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.110806\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.098989\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.102122\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.102184\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.101153\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.104900\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.103939\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.097911\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.103702\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.108284\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.110930\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.116632\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.110373\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.105898\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.101033\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.111955\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.101592\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.095632\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.100756\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.097122\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.097686\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.095605\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.098790\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.101441\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.105528\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.100789\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.106193\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.114413\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.105731\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.106645\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.109613\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.110765\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.102614\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.104060\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.111328\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.102156\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.092578\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.087450\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.102447\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.102316\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.101231\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.093832\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.095593\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.115610\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.108616\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.104478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.25s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.119640\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.113125\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.113017\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.104444\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.149040\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.123606\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.108377\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.126285\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.103101\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.104067\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.126674\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.146827\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.121886\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.104782\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.174108\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.115671\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.118109\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.114469\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.120435\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.131178\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.097231\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.119369\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.099893\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.163296\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.119078\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.117570\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.171035\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.114804\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.130180\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.184412\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.122134\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.116119\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.122086\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.111170\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.114483\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.117547\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.109733\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.104893\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.158043\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.139514\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.127952\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.106656\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.194543\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.176013\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.105296\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.102806\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.115411\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.110068\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.108464\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.108108\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.093494\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.110743\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.108629\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.157490\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.135836\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.112330\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.143692\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.103280\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.104450\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.179214\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.106193\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.104179\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.105937\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.095405\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.126998\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.140501\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.108952\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.112859\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.118407\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.113322\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.113660\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.27s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.116291\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.107890\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.096771\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.110155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 20%|██        | 1/5 [00:15<01:00, 15.11s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 1 global rounds:\n",
      "Training Loss : -0.11835016533215012\n",
      "Train Accuracy: 11.02% \n",
      "\n",
      "\n",
      " | Global Training Round : 2 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.103788\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.124520\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.114299\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.145534\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.138649\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.114966\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.122558\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.134733\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.123736\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.112224\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.130063\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.150401\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.147438\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.104001\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.173925\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.135794\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.121769\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.119789\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.124090\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.144771\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.133520\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.120075\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.117584\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.144586\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.115055\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.143094\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.163526\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.156175\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.126259\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.152429\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.118142\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.107569\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.125585\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.132071\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.123963\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.120931\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.119364\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.147026\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.112300\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.108466\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.106952\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.184861\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.149753\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.118819\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.170912\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.110156\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.107673\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.117743\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.124888\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.116966\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.105407\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.136570\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.105889\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.117598\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.139528\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.113931\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.127686\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.112301\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.166619\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.113397\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.127245\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.122186\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.125888\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.134207\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.096899\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.123457\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.101928\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.136779\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.115530\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.131274\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.123836\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.153258\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.153507\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.158199\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.135160\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.33s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.104442\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.164620\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.120836\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.193371\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.194931\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.140455\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.154382\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.236311\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.165358\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.198935\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.129477\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.154732\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.142029\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.205927\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.165086\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.119762\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.228284\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.167859\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.230968\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.115345\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.189367\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.142020\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.224089\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.144497\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.188611\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.236451\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.107698\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.212618\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.204246\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.361878\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.168504\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.174325\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.134823\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.182427\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.208014\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.180268\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.146608\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.268453\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.466237\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.244282\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.202597\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.220192\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.135630\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.185482\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.169058\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.131060\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.138799\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.186655\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.180701\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.272249\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.145666\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.283917\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.147170\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.202206\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.131924\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.321578\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.174560\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.174876\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.242217\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.267036\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.117579\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.190332\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.217312\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.177139\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.126357\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.165996\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.246081\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.334294\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.151577\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.141875\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.114454\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.329442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.27s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.228863\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.186034\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.158489\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.293675\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.348793\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.214149\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.096615\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.101243\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.181496\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.212373\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.121522\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.111174\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.164767\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.161369\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.115230\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.110964\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.114274\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.112714\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.102961\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.115056\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.127225\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.206347\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.143718\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.127799\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.160052\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.218792\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.182002\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.113474\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.165089\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.282806\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.104127\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.201896\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.150227\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.159943\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.145643\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.120267\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.111244\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.165866\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.115995\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.083800\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.188450\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.103839\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.255808\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.185105\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.230509\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.203516\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.269309\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.217008\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.114783\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.312290\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.125587\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.131423\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.303910\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.180441\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.131332\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.097710\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.105430\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.121236\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.178857\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.297934\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.113233\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.242310\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.180025\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.100728\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.186556\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.187356\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.087102\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.186211\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.106866\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.173621\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.117627\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.140819\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.084654\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.115404\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.174623\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.247649\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.221832\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.107562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.22s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.180432\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.192808\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.196250\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.249605\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.178010\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.234440\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.101431\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.222505\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.251281\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.221870\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.193507\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.136078\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.224869\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.216376\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.206193\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.160629\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.195176\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.107965\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.346714\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.159434\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.248898\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.139042\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.267175\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.255003\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.244339\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.200268\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.298107\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.256993\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.211879\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.259207\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.096782\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.093928\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.174458\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.230963\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.282766\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.294520\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.196984\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.148026\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.229902\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.171783\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.281237\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.157601\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.471003\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.328839\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.265454\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.230850\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.117734\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.129348\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.237758\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.343076\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.189356\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.212642\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.293499\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.197820\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.353138\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.342032\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.440353\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.129061\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.372662\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.291661\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.272659\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.297827\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.125304\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.270592\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.179954\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.303236\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.096905\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.221431\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.209716\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.158989\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.107809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.23s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.170461\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.130594\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.174149\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.187100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 2/5 [00:29<00:45, 15.03s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 2 global rounds:\n",
      "Training Loss : -0.1627309200535011\n",
      "Train Accuracy: 38.72% \n",
      "\n",
      "\n",
      " | Global Training Round : 3 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.110173\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.293254\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.249888\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.174756\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.469957\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.440835\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.251654\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.327775\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.251389\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.238180\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.334770\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.366944\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.424836\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.498360\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.346365\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.208946\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.271214\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.308561\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.256249\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.243124\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.230602\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.426536\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.472594\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.254922\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.408270\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.347207\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.262365\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.434080\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.489376\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.338536\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.221106\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.437558\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.185915\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.127616\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.373956\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.316467\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.236744\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.321773\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.261782\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.397607\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.393754\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.331608\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.453774\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.479490\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.185697\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.308380\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.335292\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.231110\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.154542\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.280152\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.329261\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.375186\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.244160\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.521186\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.301046\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.343309\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.469731\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.351018\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.431346\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.529050\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.317076\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.108828\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.174431\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.288193\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.315289\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.243398\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.206979\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.398974\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.547113\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.454170\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.402298\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.359457\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.394198\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.565506\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.279731\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.27s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.354158\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.281336\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.368299\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.253815\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.384870\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.463986\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.466654\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.401450\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.483701\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.384815\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.328793\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.433201\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.591396\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.456922\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.449153\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.201486\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.255838\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.288240\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.487906\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.340445\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.436615\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.274988\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.750472\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.639665\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.540113\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.592574\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.427272\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.625876\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.653298\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.450728\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.511482\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.493567\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.367453\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.501954\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.245117\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.394624\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.442759\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.556722\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.396066\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.621249\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.417349\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.412176\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.457028\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.510484\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.443289\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.390990\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.431959\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.290443\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.182814\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.552058\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.471973\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.483306\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.490743\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.413686\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.428670\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.304157\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.509121\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.379211\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.617368\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.671514\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.382106\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.439802\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.560365\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.459585\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.613676\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.497453\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.311681\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.696007\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.441738\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.324544\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.314842\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.389206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.23s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.307684\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.437409\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.295715\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.147033\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.170065\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.340817\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.398823\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.306288\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.398700\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.473822\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.356662\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.384247\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.281396\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.358673\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.300347\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.314623\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.150588\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.095913\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.171821\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.305060\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.217321\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.329922\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.253766\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.297189\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.157551\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.308294\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.254766\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.193650\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.328622\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.318608\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.400028\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.286453\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.353613\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.201227\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.156182\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.201076\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.067839\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.275100\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.253928\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.308443\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.197468\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.291795\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.322561\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.172263\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.139317\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.354528\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.424428\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.429765\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.453816\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.210938\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.384212\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.468228\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.487292\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.250353\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.323116\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.391353\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.256449\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.354861\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.066251\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.337570\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.512530\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.202601\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.329291\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.234508\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.154665\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.201318\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.252949\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.381174\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.173929\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.218625\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.322700\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.249236\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.349067\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.481397\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.261228\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.287124\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.27s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.466811\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.327603\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.391008\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.369772\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.338073\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.250659\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.535911\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.330429\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.342733\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.619712\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.427493\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.283747\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.371072\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.571000\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.315509\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.505237\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.686434\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.250029\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.401897\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.514868\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.222079\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.421259\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.463798\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.227850\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.407516\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.466984\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.495359\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.551587\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.548849\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.360561\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.365605\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.205082\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.572170\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.500133\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.612736\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.385566\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.274539\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.302508\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.385508\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.373368\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.225562\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.604700\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.331731\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.377465\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.259499\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.537015\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.511889\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.278049\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.319187\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.349303\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.306906\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.379015\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.401433\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.391555\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.473027\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.090940\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.287658\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.398341\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.536491\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.280456\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.428342\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.381447\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.238918\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.252306\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.251069\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.215958\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.346154\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.457423\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.533483\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.406434\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.270770\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.270506\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.590345\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.499051\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.228099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.30s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.220282\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.180745\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 3/5 [00:44<00:29, 14.99s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 3 global rounds:\n",
      "Training Loss : -0.24815733651544347\n",
      "Train Accuracy: 48.77% \n",
      "\n",
      "\n",
      " | Global Training Round : 4 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.396831\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.679605\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.314163\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.609537\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.474734\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.410321\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.576728\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.353473\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.583158\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.442256\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.714756\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.205173\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.318468\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.381140\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.648968\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.470746\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.373764\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.468832\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.356066\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.631554\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.411100\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.620670\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.486423\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.619792\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.617322\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.481130\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.316742\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.592379\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.344818\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.660234\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.147885\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.125242\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.642614\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.540693\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.559193\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.365132\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.608713\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.681682\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.508264\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.575768\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.266211\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.387643\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.469307\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.455515\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.210189\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.550930\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.603507\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.470551\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.423855\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.411153\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.347587\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.622014\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.416460\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.584990\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.361842\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.924628\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.731779\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.596319\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.434705\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.621225\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.462873\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.540302\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.556779\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.391307\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.534676\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.318842\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.490639\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.624162\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.380268\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.590116\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.607340\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.596344\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.506927\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.565206\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.553551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.23s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.546873\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.256462\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.606191\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.607133\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.609170\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.611921\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.651014\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.768721\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.680739\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.666212\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.322106\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.443702\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.567944\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.740900\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.587084\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.507096\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.595778\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.713122\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.478305\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.524881\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.495876\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.487573\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.667831\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.413623\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.568645\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.514620\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.833101\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.663068\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.603557\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.717201\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.591744\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.587561\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.428864\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.619895\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.568811\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.354556\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.394176\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.457431\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.735929\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.661475\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.545386\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.566526\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.667070\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.719682\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.573064\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.557232\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.557847\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.529374\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.358456\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.571984\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.624191\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.682586\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.378861\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.511189\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.642432\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.351637\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.774463\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.617756\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.596835\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.449425\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.572674\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.217058\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.630019\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.309142\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.471405\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.514086\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.482663\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.424995\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.558664\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.668133\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.674523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.26s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.256615\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.414615\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.752895\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.570405\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.314796\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.373569\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.554034\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.352541\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.498668\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.426619\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.180963\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.389360\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.396227\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.456225\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.333099\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.170675\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.559080\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.387470\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.567031\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.136612\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.229613\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.339997\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.477871\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.497743\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.554000\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.545850\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.640360\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.451328\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.553082\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.656642\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.752747\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.394696\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.595623\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.727249\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.482508\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.372480\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.547817\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.505245\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.429091\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.432414\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.523304\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.434182\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.489281\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.399716\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.567256\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.661370\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.672982\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.685436\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.365793\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.289795\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.223115\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.465351\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.329740\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.454833\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.236218\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.563280\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.502808\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.305408\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.470605\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.446473\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.386083\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.547197\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.542300\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.604120\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.404204\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.385387\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.572683\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.406690\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.411507\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.399742\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.532445\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.502171\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.392266\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.396539\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.518149\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.401224\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.364111\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.28s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.501337\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.274204\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.493367\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.911110\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.573572\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.675815\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.609680\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.531496\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.542597\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.767397\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.523088\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.429815\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.687789\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.486842\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.250008\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.757738\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.579633\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.495026\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.649739\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.489137\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.479351\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.525268\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.597484\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.482387\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.330413\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.309832\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.505367\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.530735\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.613002\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.459790\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.597838\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.668610\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.375391\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.498110\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.482563\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.660650\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.513497\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.354234\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.595942\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.357295\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.337812\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.514803\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.207137\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.458919\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.437555\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.385060\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.742543\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.402113\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.437383\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.452836\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.331803\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.434479\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.448446\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.323057\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.610819\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.656547\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.578870\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.678959\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.320215\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.526308\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.483144\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.344038\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.552376\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.356960\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.446612\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.472517\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.473468\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.316755\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.241761\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.465876\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.489462\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.599956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.25s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.662868\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.227003\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.714915\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.567730\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.642885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 80%|████████  | 4/5 [00:59<00:14, 14.97s/it]\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 4 global rounds:\n",
      "Training Loss : -0.31894223853855186\n",
      "Train Accuracy: 58.33% \n",
      "\n",
      "\n",
      " | Global Training Round : 5 |\n",
      "\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.363580\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.627605\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.529865\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.466053\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.431936\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.547320\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.521907\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.640067\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.453926\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.343627\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.474993\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.541853\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.648974\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.562843\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.592274\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.652190\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.797449\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.849488\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.584711\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.463014\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.628147\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.805698\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.843872\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.480845\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.640716\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.416251\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.710467\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.859192\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.385449\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.449698\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.555133\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.723419\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.440371\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.600868\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.639424\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.666763\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.416582\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.405500\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.602648\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.584994\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.554680\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.722724\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.402197\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.568701\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.691146\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.465106\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.437001\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.778021\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.720662\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.774471\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.460437\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.365713\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.864866\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.456378\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.685614\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.412565\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.690796\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.674432\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.713624\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.501496\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.351961\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.603511\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.536808\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.675739\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.582581\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.487722\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.299241\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.550517\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.390222\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.586772\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.788520\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.732630\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.705607\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.646352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.29s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.399440\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.691609\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.712075\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.559205\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.633299\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.677866\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.569986\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.674948\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.742058\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.798656\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.716719\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.721263\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.658766\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.799565\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.728211\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.533830\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.788041\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.466980\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.634522\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.802618\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.645760\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.874287\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.672921\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.333505\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.491914\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.795609\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.634622\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.526268\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.595420\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.623700\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.669492\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.709031\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.622454\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.761049\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.593542\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.711749\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.788277\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.664997\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.758712\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.763911\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.846070\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.675897\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.860607\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.722620\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.564069\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.372308\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.486902\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.587328\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.538385\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.545241\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.724366\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.554482\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.637443\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.691521\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.595202\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.709852\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.723938\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.623389\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.752848\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.779133\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.707021\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.556254\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.610982\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.672305\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.555384\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.483079\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.578079\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.338971\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.510505\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.640990\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.316132\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.806867\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.907704\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.852306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.22s/it]\u001b[A\n",
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.702398\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.778836\n",
      "\n",
      " | Cluster Training Round : 1 |\n",
      "\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.511301\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.583138\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.538313\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.617255\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.554005\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.610579\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.629802\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.538496\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.381903\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.533803\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.479498\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.690059\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.603853\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.425910\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.670302\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.663416\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.624876\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.519388\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.595857\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.326610\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.510289\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.514295\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.637446\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.461696\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.485843\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.386557\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.773129\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.491656\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.509070\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.722722\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.726702\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.600366\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.596561\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.486334\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.559523\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.598810\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.686987\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.598394\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.443513\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.474307\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.680113\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.505086\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.393672\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.777820\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.560947\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.494531\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.687863\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.333028\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.446344\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.576400\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.589450\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.701243\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.594791\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.676745\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.339726\n",
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.563100\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.350395\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.484228\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.588615\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.569428\n",
      "| Global Round : 0 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.622891\n",
      "| Global Round : 0 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.752291\n",
      "| Global Round : 0 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.455264\n",
      "| Global Round : 0 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.542752\n",
      "| Global Round : 0 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.493469\n",
      "| Global Round : 0 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.879509\n",
      "| Global Round : 0 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.617540\n",
      "| Global Round : 0 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.779711\n",
      "| Global Round : 0 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.808335\n",
      "| Global Round : 0 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.664017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:03<00:03,  3.16s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 0 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.603933\n",
      "| Global Round : 0 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.676986\n",
      "| Global Round : 0 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.561134\n",
      "| Global Round : 0 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.514175\n",
      "| Global Round : 0 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.479453\n",
      "\n",
      " | Cluster Training Round : 2 |\n",
      "\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.496375\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.534662\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.644580\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.310221\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.522677\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.500754\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.703199\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.492449\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.497441\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.537914\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.701324\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.828100\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.724727\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.271229\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.404799\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.409597\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.629588\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.731619\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.726212\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.736339\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.732081\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.474938\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.635087\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.548971\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.315982\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.761590\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.448176\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.530107\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.561794\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.606415\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.654767\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.643082\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.598587\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.473343\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.543252\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.429765\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.485586\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.766576\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.520245\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.451200\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.464683\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.689323\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.740777\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.210301\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.368533\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.555582\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.575229\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.585830\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.887224\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.790519\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.700906\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.664373\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.820839\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.552050\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.812297\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.614978\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.664460\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.583594\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.753344\n",
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.887813\n",
      "| Global Round : 1 | Local Epoch : 0 | [0/480 (0%)]\tLoss: -0.596942\n",
      "| Global Round : 1 | Local Epoch : 0 | [100/480 (21%)]\tLoss: -0.649852\n",
      "| Global Round : 1 | Local Epoch : 0 | [200/480 (42%)]\tLoss: -0.561844\n",
      "| Global Round : 1 | Local Epoch : 0 | [300/480 (62%)]\tLoss: -0.507072\n",
      "| Global Round : 1 | Local Epoch : 0 | [400/480 (83%)]\tLoss: -0.598828\n",
      "| Global Round : 1 | Local Epoch : 1 | [0/480 (0%)]\tLoss: -0.660993\n",
      "| Global Round : 1 | Local Epoch : 1 | [100/480 (21%)]\tLoss: -0.790286\n",
      "| Global Round : 1 | Local Epoch : 1 | [200/480 (42%)]\tLoss: -0.501520\n",
      "| Global Round : 1 | Local Epoch : 1 | [300/480 (62%)]\tLoss: -0.652477\n",
      "| Global Round : 1 | Local Epoch : 1 | [400/480 (83%)]\tLoss: -0.728103\n",
      "| Global Round : 1 | Local Epoch : 2 | [0/480 (0%)]\tLoss: -0.626058\n",
      "| Global Round : 1 | Local Epoch : 2 | [100/480 (21%)]\tLoss: -0.540041\n",
      "| Global Round : 1 | Local Epoch : 2 | [200/480 (42%)]\tLoss: -0.795200\n",
      "| Global Round : 1 | Local Epoch : 2 | [300/480 (62%)]\tLoss: -0.321233\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:06<00:00,  3.16s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Global Round : 1 | Local Epoch : 2 | [400/480 (83%)]\tLoss: -0.827032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 5/5 [01:14<00:00, 14.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "Avg Training Stats after 5 global rounds:\n",
      "Training Loss : -0.3762295932783228\n",
      "Train Accuracy: 61.85% \n",
      "\n",
      "\n",
      " Total Run Time: 83.9269\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_loss, train_accuracy = [], []\n",
    "val_acc_list, net_list = [], []\n",
    "cv_loss, cv_acc = [], []\n",
    "print_every = 1\n",
    "val_loss_pre, counter = 0, 0\n",
    "\n",
    "for epoch in tqdm(range(args.epochs)):\n",
    "    local_weights, local_losses, local_accuracies= [], [], []\n",
    "    print(f'\\n | Global Training Round : {epoch+1} |\\n')\n",
    "    \n",
    "    # ============== TRAIN ==============\n",
    "    global_model.train()\n",
    "    \n",
    "    # Cluster A\n",
    "    A_weights, A_losses = fl_train(args, train_dataset, cluster_modelA, A1, user_groupsA, 2)\n",
    "    local_weights.append(copy.deepcopy(A_weights))\n",
    "    local_losses.append(copy.deepcopy(A_losses))\n",
    "    \n",
    "    # Cluster B\n",
    "    B_weights, B_losses = fl_train(args, train_dataset, cluster_modelB, B1, user_groupsB, 2)\n",
    "    local_weights.append(copy.deepcopy(B_weights))\n",
    "    local_losses.append(copy.deepcopy(B_losses))\n",
    "    \n",
    "    \n",
    "    # averaging global weights\n",
    "    global_weights = average_weights(local_weights)\n",
    "\n",
    "    # update global weights\n",
    "    global_model.load_state_dict(global_weights)\n",
    "\n",
    "    loss_avg = sum(local_losses) / len(local_losses)\n",
    "    train_loss.append(loss_avg)\n",
    "    \n",
    "    # ============== EVAL ============== \n",
    "    # Calculate avg training accuracy over all users at every epoch\n",
    "    list_acc, list_loss = [], []\n",
    "    global_model.eval()\n",
    "    for c in range(args.num_users):\n",
    "        local_model = LocalUpdate(args=args, dataset=train_dataset,\n",
    "                                  idxs=user_groups[c], logger=logger)\n",
    "        acc, loss = local_model.inference(model=global_model)\n",
    "        list_acc.append(acc)\n",
    "        list_loss.append(loss)\n",
    "    train_accuracy.append(sum(list_acc)/len(list_acc))\n",
    "    \n",
    "    # print global training loss after every 'i' rounds\n",
    "    if (epoch+1) % print_every == 0:\n",
    "        print(f' \\nAvg Training Stats after {epoch+1} global rounds:')\n",
    "        print(f'Training Loss : {np.mean(np.array(train_loss))}')\n",
    "        print('Train Accuracy: {:.2f}% \\n'.format(100*train_accuracy[-1]))\n",
    "#         print('Train Accuracy: {:.2f}% \\n'.format(100*train_accuracy[-1][0]))\n",
    "\n",
    "print('\\n Total Run Time: {0:0.4f}'.format(time.time()-start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      " Results after 5 global rounds of training:\n",
      "|---- Avg Train Accuracy: 61.85%\n",
      "|---- Test Accuracy: 61.96%\n"
     ]
    }
   ],
   "source": [
    "# Test inference after completion of training\n",
    "test_acc, test_loss = test_inference(args, global_model, test_dataset)\n",
    "\n",
    "print(f' \\n Results after {args.epochs} global rounds of training:')\n",
    "print(\"|---- Avg Train Accuracy: {:.2f}%\".format(100*train_accuracy[-1]))\n",
    "print(\"|---- Test Accuracy: {:.2f}%\".format(100*test_acc))\n",
    "\n",
    "# # Saving the objects train_loss and train_accuracy:\n",
    "# file_name = '../save/objects/{}_{}_{}_C[{}]_iid[{}]_E[{}]_B[{}].pkl'.\\\n",
    "#     format(args.dataset, args.model, args.epochs, args.frac, args.iid,\n",
    "#            args.local_ep, args.local_bs)\n",
    "\n",
    "# with open(file_name, 'wb') as f:\n",
    "#     pickle.dump([train_loss, train_accuracy], f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/wesleyjtann/miniconda3/envs/fl_pytorch/lib/python3.7/site-packages/ipykernel_launcher.py:4: UserWarning: matplotlib.pyplot as already been imported, this call will have no effect.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<function matplotlib.pyplot.show(*args, **kw)>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3XecVNX5x/HPl06CGg2oIAIWRKyoq7E3sCtYiA0jtpBoLImxYDT2KFii0WiUkERUNJaoYMECKjH+bGsDO1ZQULFrrMDz++Pc1XGzu8yyO3O3fN+v17ym3DP3PnOnPHPPueccRQRmZmbFaJN3AGZm1nw4aZiZWdGcNMzMrGhOGmZmVjQnDTMzK5qThpmZFc1JowWT1FbSZ5J6NWZZaxkkbSnp2RKsd0VJnzX2evMmaZCk1/OOI29OGk1I9qNddVkg6YuC+8Pqu76ImB8RXSJiZmOWrS9JZ0q6orHXW06SNpR0p6SPJX0g6RFJ++cdV0NExP0RsXpD1yPpTUlbFqz31Yjo0tD1WtPkpNGEZD/aXbIv3Exgl4LHxlcvL6ld+aNsfSRtCkwGpgArAj8GDgd2zDOu1s6f/3w4aTQj2T/26yRdK+lTYD9JG0l6WNJHkuZIukhS+6x8O0khqU92/+ps+SRJn0p6SNIK9S2bLd9B0kvZP++LJT0o6YBFeE2rS5qaxT9d0k4Fy3aW9Hy2/Tcl/SZ7fGlJd2TP+UDSv2tZ91hJo6o9drukI7Pbv5M0W9Inkl4o/LdczXnA3yLi3Ih4P5LHImLvgvX+UtLLkt6XdIuk7tX266GSXsleyymS+mbv2yfZ+1n1ng2S9LqkEyTNzeLbJdsXM7LXe1zBdq+WdGrB/e9VoWT77ehs336cbatjLWV7Z7HPlfSepD9lj/eVdF/22t6TdJWkJbJl1wI9gEnZEfHRklaWFAXr7Snptiz2GZIOKlh2ZhbT1dm+eUbSurW8n1X78jBJLwMvZI9vKqkye32PSvpJtde/ZcH9b496q+KUtH9Wbq6kkQVlf5C91g+VqvHWqxZPsZ+fliUifGmCF+B1YFC1x84EvgZ2ISX8zsD6wE+AdqR/wS8Bh2fl2wEB9MnuXw28B1QA7YHrgKsXoezSwKfAkGzZ0cA3wAG1vJYzgStqeLwD8BpwXLaeQcBnwMrZ8rnAxtntpYB1s9vnAn/OntMB2KKW7W6d7Udl938MfAEsA6wOvAEsmy1bAVixhnUsBiwANqvjvdoWeBcYAHQCLgXurbZfb8rWtVb2Ht4D9AGWJP34DcvKDwLmASdmr+/QbN1XA12y538J9Cp4n04tiGUQ8HrB/TeBh4Fls9f/EnBI9bJZnM+QEuQPSZ+tTbJlqwADs329NPAgcF61bWxZcH9lIAruPwhcnO2bdbPP1RYFn40vgO2Attl7+59a9nPVvrwz22+dga7Ax8A+2fL9gPeBJWuJ7dvPYlWcwGUFsX0F9M2Wnwfcn22rN/Bcwf4q6vPTEi8+0mh+/hMRt0bEgoj4ItI/3kciYl5EvAqMAbao4/k3RkRlRHwDjCf90NW37M7AUxExIVt2AemHoL42If0QnRsR30TEZGASUPUP/htgNUmLRcQHEfFEweM9SD+cX0fE1FrWfz/ph3ej7P6ewAMR8Q7ph7kTsLqkdhHxWrb/qlsKEDCnjtcxDBgbEU9FxJfASGALST0LyoyOiE8jYhrwPHBnRLweER8CdwHrFJT9EhiV7dt/At2ACyLis+z5L5KSR7EujIi3I+J94DZqfs83Iv0AHx8R/80+Ww8CRMRLETEl29fvkt7vuj5j38qOTjcARkbEl9l7+A/gZwXFpkbEXRExH7iqlvgKnRURH0bEF6Q/UM9GxLXZd+Bq4FVgp7pX8T2nFsT2LLB29viewJnZtt4g/VGpUuznp8Vx0mh+ZhXekbRqVuXytqRPgNNJX/7avF1w+3PSv9f6lu1RGEekv1pvFhF7dT2Amdnzq7wBLJfd3g0YDMyUdH9BtcOorNyUrMrn2JpWHhELSEdI+2QP7UtKfkTEi8BvSfvr3ayKZNkaVvMB6d9o94W8jjcKtvsJ8GHB6wB4p+D2FzXcL3wf3st+QKuW1fT8+jQ0F/OeL0/6Fz2/+gJJy0q6XtJb2WfsCur+jBXqQXo9/y14rPA9rim+Hy5knYXfge/t+1rWX6eIqG3/dK+2rcL3uNjPT4vjpNH8VB+W+HJStcLKEbE4cDLpn3EpzQG+/RctSdTjS1pgNrB89vwqvYC3ALIjqMGkKpHbSP+6iYhPIuI3EdEH2BU4XlJt/3yvBfbM/vGuC9xctSAiro6ITUhVC22Bs6s/OSI+BR4F9ljI6+hddUfSYqQqjbfqeE5j+S/wg4L7i/rDNQvoLaltDctGk6pt1sw+Ywfw/c9YXUNlzwa6SipMBN++x4uocHvf2/c1rL8h++dtUjItXO93QRTx+WmJnDSav8VIdbr/ldQf+EUZtnkbsG7WQNsOOIpUhVKXtpI6FVw6Av9HOsz/raT2krYmnZF0vaTOkvaVtHhWTfMpMB8g2+5KWbL5OHv8f/4hA0TEY1mZMcAd2VEAkvpL2iqL44vsUuM6gGOBQ7JG3qWy568j6Zps+bXAwZLWytZ3NqkabFGOvurrKWAnSUsqNb4fuYjreYjUFnBW1gDcWdIm2bLFSD++H0taHjim2nPfIbWn/Y+IeA2ozNbbUdIA4ECyI75GcBupimivrKF8X1JbxR3Z8qeAvbNlGwC712Pd1wO/k/Qjpf5Lh1ctqOfnp0Vx0mj+fgsMJ/2oXk6qjimprE1gL+CPpB+alYAnSf9Ga7Mf3325vgBejIivSHXSQ0htIhcB+0bES9lzhgNvZFUiB/NdPXg/4F5So/mDwJ8i4j91bPtaUqPvNQWPdQTOybb7NunI4KRaXu8D2fO3A16X9AHwF7Ifpoi4k1RNcTPpKKwXqZ2jHK4gtZG8QWog/ueirCQi5pHaqvqTjjpmAkOzxaeQ2iU+BiYC/6r29LOA05TOZvt1DavfC+hL2s83Ar+LiPsWJc4a4p5LqsI8nvRZ/A2wc0R8kBU5EVgV+Aj4Pd//DCzMKaT383VSW9uVBcuK/vy0NFVnlZgtsqxKYzYwNPuBNbMWykcatkgkbS9piezw/PekaqZHcw7LzErMScMW1aakUxvfA7YHds2qm8ysBXP1lJmZFc1HGmZmVrRcBvzKTlu8jjSMwuvAnlnP2Orl7gQ2JPWC3rmYdXft2jX69OnTaLGambUGjz/++HsRsbBT5/NJGqRhFqZExKhsgLCRpFPmqjuX1DGn6L4Hffr0obKysnGiNDNrJSRV71lfo7yqp4YA47Lb40i9ev9HREwh9T8wM7MmIK+ksUxEzAHIrpduyMokjciGRq6cO3duowRoZmb/q2TVU5ImU/M4Lyc29rYiYgxpmAgqKip8OpiZWYmULGlExKDalkl6R1L3iJiTjZfzbqniMDOzxpNX9dRE0rhCZNcTcorDzMzqIa+kMQrYRtIMYJvsPpIqJI2tKiTpAeAGYGA2HeN2uURrZmZATqfcZjOIDazh8UrgkIL7m5UzLjMzq5t7hFeJgGOPhSeeWHhZM7NWykmjyiuvwJgxsN56sN12MHVqSiRmZvYtJ40qK68MM2fC2WfDU0/BllvCJpvArbc6eZiZZZw0Ci2xBIwcCa+/DpdcArNnw+DBsNZacM01MG9e3hGameXKSaMmnTvDYYfBjBlw5ZWwYAEMGwb9+sFll8GXX+YdoZlZLpw06tK+PfzsZzB9OtxyC3TtCoceCiusAOeeC596WCwza12cNIrRpg0MGQIPPwxTpsAaa8Bxx0GvXvD734PHuzKzVsJJoz4k2HpruOceePTRdPvMM6F3bzjqKJg1K+8IzcxKykljUa2/PvzrX/Dcc7DnnnDppbDiinDQQfDii3lHZ2ZWEk4aDdW/P1xxBbz8cmrv+Oc/02NDh8Ljj+cdnZlZo3LSaCy9e8NFF6XTdU84ASZPhoqK1FHw/vvd18PMWgQnjca29NLwhz/AG2/AqFHw9NOw1Vaw8cYwcWI6fdfMrJly0iiVJZaA44+H115L7R1vv53OwFp7bRg/3h0FzaxZctIotc6dU1vHjBlw1VWpmmq//WCVVeAvf3FHQTNrVpw0yqVdu5Qspk2DCRNSNdZhh0GfPnDOOfDJJ3lHaGa2UE4a5damTRrP6qGH4N5707hWxx+fGtJPOskdBc2sSXPSyIuUGsjvvhseewwGDoSzzkrJ48gj04i7ZmZNjJNGU1BRATfemDoK7rVXautYaSU48EB44YW8ozMz+5aTRlOy6qrwj3+kCaEOOwyuuw5WWw322AMqK/OOzszMSaNJ6tUL/vSn1NfjxBPTIInrrw/bbgv33eeOgmaWGyeNpqxbNzjjjNS+MXp0OvNq661ho43SGVjuKGhmZeak0Rwsvngaiv3111N7x7vvwq67pjOvrr7aHQXNrGycNJqTTp3gl7+El15KyUJKk0T17Zt6nX/xRd4RmlkL56TRHLVrl6afffrpNJ5V9+7wq1+lGQVHj3ZHQTMrGSeN5qxNG9hlF3jwwTSS7oABMHJkakg/8cRUjWVm1oicNFoCCbbYAu68M52au802cPbZaYiSI45IZ2GZmTUCJ42WZr314IYb4PnnYe+94bLLYOWV4YAD0mNmZg3gpNFS9esHf/87vPpqau+44QZYfXXYffc0bImZ2SJw0mjpll8eLrwwVVGddFLqHLjBBqkK69573VHQzOrFSaO16NoVTj89JY9zzoFnnkmDJG64IdxyizsKmllRnDRam8UXh2OPTTMKXnYZvPce7LYbrLlmmiTqm2/yjtDMmrBckoakpSTdI2lGdr1kDWUGSHpI0rOSpknaK49YW6xOneAXv4AXX4RrroG2bWH//VNHwUsucUdBM6tRXkcaI4EpEdEXmJLdr+5zYP+IWB3YHrhQ0o/KGGPr0K4d7LNP6ih4662w3HJw+OHpdN1Ro+Djj/OO0MyakLySxhBgXHZ7HLBr9QIR8VJEzMhuzwbeBbqVLcLWRoKdd4b//AemToV11oETTkgdBX/3O3cUNDMgv6SxTETMAciul66rsKQNgA7AK7UsHyGpUlLlXE+X2jASbL556ij4+OOw3XbpiKN373QE4o6CZq1ayZKGpMmSnqnhMqSe6+kOXAUcGBE1nuITEWMioiIiKrp188FIo1l3Xbj++jR74LBhMGZM6ig4fHiaZdDMWp2SJY2IGBQRa9RwmQC8kyWDqqRQY92HpMWB24GTIuLhUsVqC7HKKjB2bOooePjhaWra1VdPZ129+mre0ZlZGeVVPTURGJ7dHg5MqF5AUgfgZuDKiLihjLFZbXr2hAsuSFVUJ5+cOgoOHAhvv513ZGZWJnkljVHANpJmANtk95FUIWlsVmZPYHPgAElPZZcB+YRr39O1K5x2GkyeDHPnwk47waef5h2VmZWBooUNI1FRURGVlZV5h9F63HEHDB6chiWZOBHat887IjNbBJIej4iKhZVzj3BrmB13hMsvT2db/eIXHsvKrIVrl3cA1gIcfDDMmpWqrHr1glNPzTsiMysRJw1rHKec8l3i6NkTDjkk74jMrAScNKxxSGkAxNmz4Ze/hB49UtWVmbUobtOwxtO+fZrsae214ac/TVPPmlmL4qRhjatLF7j9dlh66XQqrjv/mbUoThrW+JZdNp1NNW8ebL99mrPDzFoEJw0rjX790lDrs2bBLrvA55/nHZGZNQInDSudjTeG8ePhkUfSgIfz5+cdkZk1kJOGldbuu8Of/pTmIT/ySHf+M2vmfMqtld4RR6RqqnPPTZ3/jj8+74jMbBE5aVh5jBqVEsfIkanz37BheUdkZovAScPKo00buOKKNIz6gQemM6wGDsw7KjOrJ7dpWPl07Ag335zOrNp9d5g2Le+IzKyenDSsvH70ozSc+mKLpWFGZs3KOyIzqwcnDSu/5ZeHSZPSxE077AAffZR3RGZWJCcNy8eaa6aqqpdegl13ha++yjsiMyuCk4blZ+utU+P41KlwwAGwYEHeEZnZQvjsKcvXvvvCm2+mvhs9e6a+HGbWZDlpWP6OPRZmzoTzzkvtHUcemXdEZlYLJw3Ln5SGGnnrLfj1r2G55WCPPfKOysxq4DYNaxratoVrroENN0y9xR98MO+IzKwGThrWdHTuDBMnQu/eaTj1F17IOyIzq8ZJw5qWrl1TH4727dMETm+/nXdEZlbAScOanhVXTFPGvvde6jX+6ad5R2RmGScNa5oqKuCGG9L4VD/9KXzzTd4RmRlOGtaU7bADXH453HUXjBjhCZzMmgCfcmtN28EHp0ENTzstTeB02ml5R2TWqjlpWNN3yikpcZx+euo1/vOf5x2RWavlpGFNnwSXXQazZ8Ohh0KPHrDTTnlHZdYq1atNQ8kPSxWMWa3at08N42uvDXvuCY89lndEZq3SQpOGpCslLS7pB8CzwGuSji59aGbVdOmSTsVdeul0pPHKK3lHZNbqFHOksWZEfALsCtwN9AQOaMhGJS0l6R5JM7LrJWso01vS45KekvSspF82ZJvWQiy7LNx5J8yfn86umjs374jMWpVikkYHSe2AIcAtEfE10NCJD0YCUyKiLzAlu1/dHGDjiBgA/AQYKalHA7drLUG/fnDrralxfPBg+PzzvCMyazWKSRpjgZnAksBUSb2Azxq43SHAuOz2ONJRzPdExNcRUTWdW8ciY7XWYuON0wCHjzyS5uSYPz/viMxahYX+EEfEBRHRIyK2jYgAZgFbN3C7y0TEnGz9c4ClayokaXlJ07Jtjo6I2bWUGyGpUlLlXFdXtB677QYXXQQTJsARR7jzn1kZFNMQfrikxbPblwOPAJsV8bzJkp6p4TKk2OAiYlZErAWsDAyXtEwt5cZEREVEVHTr1q3Y1VtLcPjhcNxx8Je/wOjReUdj1uIV009jRET8WdK2wHLAocAYYL26nhQRg2pbJukdSd0jYo6k7sC7C1nXbEnPkpLVjUXEbK3J2Wen9o0TTkid//bbL++IzFqsYtoJqo75dwD+ERGPF/m8ukwEhme3hwMTqheQ1FNS5+z2ksAmwIsN3K61RG3awD/+AVttBQcdBFOm5B2RWYtVzI//05LuAHYBJknqwneJZFGNAraRNAPYJruPpApJY7My/YFHJD0NTAXOi4jpDdyutVQdO8JNN6Uzq3bbDZ5+Ou+IzFokxUIaDyW1JVVFvRwRH0jqCiwfEU+WI8D6qqioiMrKyrzDsLy8+WaaMjYCHnooDXJoZgsl6fGIqFhYuWLOnpoPdAWOkzQKWL+pJgwzevZMM/999lnq/Pfhh3lHZNaiFHP21B+A44BXs8uxks4sdWBmi2zNNeGWW2DGjFRV9dVXC3+OmRWlmDaNXYBB2WmtY4BtgcGlDcusgbbaCsaNg6lTYfhwWNDQQQzMDIofGn0x4MOC22ZN3z77pDaO445L1VbnnZd3RGbNXjFJ4xzgCUlTAAFbAieXMiizRnPMMTBzJpx/Piy/PBx1VN4RmTVrC00aEXG1pPtIgwYKODki3ip5ZGaNQYILL4S33oLf/CYdceyxR95RmTVbtSYNSWtVe+jl7PrHkn4cEdNKF5ZZI2rbFsaPh0GDYNgwWGYZ2HTTvKMya5bqOtK4pI5lAWzeyLGYlU7nzjBxYhodd/BgePBB6N8/76jMmp1ak0ZELHRQQrNm5cc/ThM4bbRR6sPx0EPQvXveUZk1K56jwlqXFVZIU8a+916aMvbTT/OOyKxZcdKw1me99eCGG2DaNBg6FL75Ju+IzJoNJw1rnXbYAcaMgbvvhp//3BM4mRVpoafc1nAWFcDHwKyIcDdba74OOijNw3HqqWlgw9NPzzsisyavmM59fwMGAM+S+mn0B54BlpA0IiI8eYE1XyefnBLHGWekPhwjRuQdkVmTVkz11AxgvYgYEBFrk4ZJfwrYDji/lMGZlZyUpordYQc49FC47ba8IzJr0opJGv0LO/JlEyGtGxEv1/Ecs+ajfXu4/npYZx3Yay947LG8IzJrsopJGq9IuljSJtnlIuBlSR2BeSWOz6w8unRJp+Ius0w6FfeVV/KOyKxJKiZp7A+8CYwETgBmk+b1ngcMLF1oZmW2zDKp89+CBbD99jB3bt4RmTU5xczc93lEjI6IXSJi54gYFRH/jYj5EfFxOYI0K5tVVoFbb01Dqu+yC3z+ed4RmTUpxczct6GkSZKek/RS1aUcwZnlYqON4Npr4dFH05wc81wLa1almOqpfwCXAoOAzQouZi3XrrvCxRenQQ6PPNKd/8wyxfTT+CQibi15JGZNza9+lfpwjB6dOv+NHJl3RGa5KyZp3CvpbOAm4KuqBz2fhrUKZ52V2jdOOAGWWw5+9rO8IzLLVTFJY9Nq1+D5NKy1aNMG/v53mDMnDTvSvXuazMmslSpmule3X1jr1qED3HQTbLYZ7L47PPAArL123lGZ5aKu6V73iYhrJR1Z0/KIuKh0YZk1MUssAXfckc6s2nHHNIFTr155R2VWdnWdPbVkdt2tlotZ69KzJ0yaBP/9bxqr6sMP847IrOzqmu710uz69+ULx6yJW2MNuOUW2G67dFru3XdDx455R2VWNsXMp9EVOAjoU1g+IjyGtLVOW24J48aljn/77586ArbxfGbWOhRz9tQE4GHgP8D80oZj1kzsvXc6FffYY2H55eG88/KOyKwsikkaP4yI35Y8ErPm5re/hZkz4fzzU+I46qi8IzIruWKOqSdJ2rYxNyppKUn3SJqRXS9ZR9nFJb0l6c+NGYNZg0lwwQXpNNzf/AZuvDHviMxKrpik8UvgTkmfSfpA0oeSPmjgdkcCUyKiLzAlu1+bM4CpDdyeWWm0bQtXXw0bbwz77Zf6cJi1YMUkja5Ae2AJ0qm2XWn4KbdDgHHZ7XHArjUVkrQesAxwdwO3Z1Y6nTvDhAnQpw8MGQLPP593RGYlU2vSkNQ3u7l6LZeGWCYi5gBk10vXsP02pDnIj23gtsxK78c/Tn04OnRIfTjmzMk7IrOSqKshfCRwMHBJDcsWOvaUpMnAsjUsOrHI2A4D7oiIWZLqLChpBDACoJd76VpeVlgh9RrffPPUa/zf/4bFFss7KrNGpchhngBJLwJbRsQcSd2B+yOiX7Uy40nzdiwAugAdgEsjos7xqSsqKqKysrJEkZsV4c47YeedYeBAuO02aN8+74jMFkrS4xFRsbByRfVIkrSqpN0l7Vt1aWB8E0nzjJNdT6heICKGRUSviOgDHANcubCEYdYkbL89/PWvqbf4z3/uCZysRSmmR/hJwLbAqsBdwHakjn7XNGC7o4DrJR0MzAR+mm2rAvhlRBzSgHWb5e/AA9METqeckvpwnHFG3hGZNYqFVk9Jmg4MAJ6IiLWz6qTLI2JwOQKsL1dPWZMRASNGwNixcPnl6bZZE1Vs9VQxPcK/iIj5kuZJWgx4G1ixwRGatXQS/OUvMHs2HHoo9OiR2jrMmrFi2jSelPQj4O9AJfAo8ERJozJrKdq1g+uug3XXhb32gvvvzzsiswapM2konet6akR8FBGXADsBv4iI/csSnVlL0KVLOouqRw/Yais4+GCYOzfvqMwWSZ1JI1KDx20F91+OCB9lmNXXMsvAE0+kUXGvvBJWWQUuvRTme+Boa16KqZ56VNK6JY/ErKVbbDE45xyYNi1VV/3qV7D++mnqWLNmoq5hRKoayTclJY4XJT0h6UlJPtowW1T9+8Pkyamt491302CHBx2Ubps1cXUdaTyaXe8K9AN2JPWnGJpdm9mikmDPPeGFF+D44+Gqq6BfP/jzn2HevLyjM6tVXUlDABHxSk2XMsVn1rJ16QKjRsH06VBRAUccka4ffDDvyMxqVFc/jW6Sjq5tYUT8sQTxmLVOq66ahh3517/ShE6bbgrDh8Po0akR3ayJqOtIoy1poMDFarmYWWOSYOjQNB/HyJFwzTWpyurii11lZU1GrcOISHoiIprdWVMeRsRajBdfTNVV99wDa60Fl1ySjkDMSqAxRrmtexILMyutfv3grrvS3OMffgibbZaqrN55J+/IrBWrK2kMLFsUZlYzCfbYI1VZnXACXHtt6hh40UWusrJc1Jo0IuKDcgZiZnX44Q/hrLPgmWdgww3hqKNgvfXggQfyjsxamaImYTKzJmKVVdLMgP/6F3z0UZpa9mc/g7ffzjsyayWcNMyaGwl23z1VWZ14Ilx/fWr/uPBCV1lZyTlpmDVXP/gBnHlmqrLaaKPUv2OddeDf/847MmvBnDTMmru+fWHSJLjpJvjkE9hiC9hvP5gzJ+/IrAVy0jBrCSTYbbdUZXXSSXDDDanK6oIL4Jtv8o7OWhAnDbOW5Ac/gDPOSFVWm24KRx+dqqymTs07MmshnDTMWqK+feH22+GWW+Czz2DLLWHYsDRfuVkDOGmYtVQSDBkCzz0Hv/99Ok23Xz84/3xXWdkic9Iwa+l+8AM4/fRUZbX55nDMMTBgANx/f96RWTPkpGHWWqy8Mtx2G0yYAJ9/DlttBfvsA2+9lXdk1ow4aZi1JhIMHpyqrE4+GW6+Oc3lcd55rrKyojhpmLVGnTvDaafBs8+mRvJjj4W114Z77807MmvinDTMWrOVVoJbb4WJE+HLL2HgQNh7b1dZWa2cNMwMdtklHXWcempq8+jXD845B77+Ou/IrIlx0jCzpHNnOOWUlDy23hqOPz5VWU2Zkndk1oQ4aZjZ9624Yqquuu22dKQxaBDsuSe8+WbekVkT4KRhZjXbaad01HHaaandY9VVYfRoV1m1ck4aZla7Tp3SqbnPPZeOOEaOhLXWgsmT847McpJL0pC0lKR7JM3Irpespdx8SU9ll4nljtPMMiuskMaxuv32NNHTNtvAT38Ks2blHZmVWV5HGiOBKRHRF5iS3a/JFxExILsMLl94ZlajHXdMw5GccUZq81h1VRg1ylVWrUheSWMIMC67PQ7YNac4zKy+OnVKc3Y8/zxsuy2ccAKsuSbcfXfekVkZ5JU0lomIOQDZ9dK1lOskqVLSw5JqTSySRmTlKufOnVuKeM2suj590jAkd9wBCxbAdtvB0KEwc2bekVkJlSxpSJos6ZkaLkPqsZoFNgGCAAAP9klEQVReEVEB7AtcKGmlmgpFxJiIqIiIim7dujVK/GZWpB12gOnT03zld9wB/fvDWWfBV1/lHZmVQMmSRkQMiog1arhMAN6R1B0gu363lnXMzq5fBe4H1ilVvGbWAJ06wYknpiqr7bZLt9dcE+66K+/IrJHlVT01ERie3R4OTKheQNKSkjpmt7sCmwDPlS1CM6u/3r3hpptg0iSIgO23h913hzfeyDsyayR5JY1RwDaSZgDbZPeRVCFpbFamP1Ap6WngPmBURDhpmDUH22+fzrL6wx/gzjtTldUf/uAqqxZAEZF3DI2qoqIiKisr8w7DzKrMnAlHH52mm115Zbj44pRUrEmR9HjWhlwn9wg3s9Lq1QtuvDG1b7RpkxrOd9sNXn8978hsEThpmFl5bLstTJsGZ5+d+nT075/OuPryy7wjs3pw0jCz8unYMY1f9fzzsPPO8PvfwxprpFN1rVlw0jCz8uvVC264IR1xtGuXRtTddVdXWTUDThpmlp9ttklVVqNGwT33pCqr0093lVUT5qRhZvnq0CHNEvjCC2na2VNOgdVXTyPqWpPjpGFmTcPyy8P116cjjg4dUpvHllvCtde6f0cT4qRhZk3LoEHw9NNw/vmpJ/m++8Jyy6W+Hs8/n3d0rZ6Thpk1PR06pCTxyiupf8dWW6VOgautBptuCuPGweef5x1lq+SkYWZNV5s2qX/HDTfAW2/BOefAu+/CAQdAjx5w+OHpqMTKxknDzJqHpZeGY4+FF1+E++5Lp+mOHQsDBsAGG8Bf/wqffpp3lC2ek4aZNS9SaiAfPx5mz4YLL0xVVSNGpKOPn/8cHnssjbJrjc5Jw8yar6WWgqOOSpNAPfhgmjlw/Ph05LHOOnDJJfDRR3lH2aI4aZhZ8yfBxhvDP/4Bc+bApZem9pDDD09HH8OHp6Tio48Gc9Iws5ZliSXg0EPhiSegshL23z9NDLXppqnT4AUXwPvv5x1ls+WkYWYt13rrwWWXpaOPsWNh8cXTqbw9esA++8C998KCBXlH2aw4aZhZy9elCxx8MDz8cBrr6he/SDMKDhwIq6wCo0fDO+/kHWWz4KRhZq3LmmvCRRelM6+uuioddYwcCT17wh57pGQyf37eUTZZThpm1jp17gz77Qf//ncanuSoo9LtHXaAlVaCM86AN9/MO8omx0nDzGzVVeG881KSuO66NJf5ySdD795p5N2JE2HevLyjbBKcNMzMqnTsCHvuCZMnw8svpyHbKythyJCUQE46CV57Le8oc+WkYWZWk5VWgrPOgpkz4eab03AlZ52VHt9uO7jxRvj667yjLDsnDTOzurRvn6aivf32NB3tySenNpCf/jTNAXLccfDSS3lHWTZOGmZmxerVC049NVVR3X576oX+xz9Cv35p+PZrrmnxU9U6aZiZ1VfbtrDjjqnaatasVG31xhswbFiaMOrXv4Znn807ypJw0jAza4ju3eGEE1LD+T33pJkHL70U1lgjHYlccUWLmjDKScPMrDG0aZMSxnXXpQmjzjsPPvgADjwwJZbDDoMnn8w7ygZz0jAza2zdusFvf5sazKdOhcGD0wi8664LFRUwZkyznTDKScPMrFQk2HzzNFzJ7Nlp+JKvv05jX3XvDoccAo880qyGbHfSMDMrhyWXhCOOSHOaP/ww7L03XHstbLghrL02XHwxfPhh3lEulJOGmVk5SfCTn6Sh2ufMSUO3d+gARx6ZBk/cf3944IEme/ThpGFmlpfFF09VVZWVadKoAw+EW25JVVqrrQbnnw/vvZd3lN+TS9KQtJSkeyTNyK6XrKVcL0l3S3pe0nOS+pQ3UjOzMllnnXSq7pw58Pe/p+qsY45JRx977w1TpjSJCaPyOtIYCUyJiL7AlOx+Ta4Ezo2I/sAGwLtlis/MLB8//GE64vi//4Pp09OpunffnU7n7dsXzj47JZac5JU0hgDjstvjgF2rF5C0GtAuIu4BiIjPIqLl9JAxM1uYNdaACy9MZ16NH5/Guvrd79L1brvBpEllnzAqr6SxTETMAciul66hzCrAR5JukvSkpHMlta1pZZJGSKqUVDl37twShm1mloNOnWDffeH+++HFF9M85w8+mIYyWXFFOO20NJxJGZQsaUiaLOmZGi5DilxFO2Az4BhgfWBF4ICaCkbEmIioiIiKbt26NUr8ZmZN0iqrwDnnpAmjbrghTSB12mnQpw/stVfJz7pqV6oVR8Sg2pZJekdS94iYI6k7NbdVvAk8GRGvZs+5BdgQ+FtJAjYza046dIChQ9Pltdfgb39LCUMq6WZLljQWYiIwHBiVXU+oocxjwJKSukXEXGBroLJ8IZqZNRMrrABnnlmWTeXVpjEK2EbSDGCb7D6SKiSNBYiI+aSqqSmSpgMC/ppTvGZmRk5HGhHxPjCwhscrgUMK7t8DrFXG0MzMrA7uEW5mZkVz0jAzs6I5aZiZWdGcNMzMrGhOGmZmVjQnDTMzK5qiiU70sagkzQXeaMAqugJNawD7xHHVj+OqH8dVPy0xrt4RsdBxmFpc0mgoSZURUZF3HNU5rvpxXPXjuOqnNcfl6ikzMyuak4aZmRXNSeN/jck7gFo4rvpxXPXjuOqn1cblNg0zMyuajzTMzKxoThpmZla0Vpk0JG0v6UVJL0saWcPyjpKuy5Y/IqlPE4nrAElzJT2VXQ6paT0liOvvkt6V9EwtyyXpoizuaZLWbSJxbSnp44L9dXKZ4lpe0n2Snpf0rKSjaihT9n1WZFxl32eSOkl6VNLTWVyn1VCm7N/JIuPK5TuZbbutpCcl3VbDstLtr4hoVRegLfAKac7xDsDTwGrVyhwGXJbd3hu4ronEdQDw5xz22ebAusAztSzfEZhEmihrQ+CRJhLXlsBtOeyv7sC62e3FgJdqeC/Lvs+KjKvs+yzbB12y2+2BR4ANq5XJ4ztZTFy5fCezbR8NXFPT+1XK/dUajzQ2AF6OiFcj4mvgn8CQamWGAOOy2zcCA6UST7xbXFy5iIh/Ax/UUWQIcGUkDwM/yuZ+zzuuXETEnIh4Irv9KfA8sFy1YmXfZ0XGVXbZPvgsu9s+u1Q/Q6fs38ki48qFpJ7ATsDYWoqUbH+1xqSxHDCr4P6b/O8X59syETEP+Bj4cROIC2CPrDrjRknLlzimYhUbex42yqoXJklavdwbz6oF1iH9Sy2U6z6rIy7IYZ9lVS1PAe8C90RErfurjN/JYuKCfL6TFwLHAQtqWV6y/dUak0ZN2bb6v4diyjS2YrZ5K9AnItYCJvPdP4m85bG/ivEEaTydtYGLgVvKuXFJXYB/Ab+OiE+qL67hKWXZZwuJK5d9FhHzI2IA0BPYQNIa1Yrksr+KiKvs30lJOwPvRsTjdRWr4bFG2V+tMWm8CRT+G+gJzK6tjKR2wBKUvhpkoXFFxPsR8VV296/AeiWOqVjF7NOyi4hPqqoXIuIOoL2kruXYtqT2pB/m8RFxUw1FctlnC4srz32WbfMj4H5g+2qL8vhOLjSunL6TmwCDJb1OqsbeWtLV1cqUbH+1xqTxGNBX0gqSOpAaiSZWKzMRGJ7dHgrcG1mLUp5xVavzHkyqk24KJgL7Z2cEbQh8HBFz8g5K0rJV9biSNiB93t8vw3YF/A14PiL+WEuxsu+zYuLKY59J6ibpR9ntzsAg4IVqxcr+nSwmrjy+kxFxQkT0jIg+pN+JeyNiv2rFSra/2jXGSpqTiJgn6XDgLtIZS3+PiGclnQ5URsRE0hfrKkkvk7Lz3k0kriMlDQbmZXEdUOq4ACRdSzqrpqukN4FTSI2CRMRlwB2ks4FeBj4HDmwicQ0FDpU0D/gC2LsMyR/SP8GfAdOz+nCA3wG9CmLLY58VE1ce+6w7ME5SW1KSuj4ibsv7O1lkXLl8J2tSrv3lYUTMzKxorbF6yszMFpGThpmZFc1Jw8zMiuakYWZmRXPSMDOzojlpWO6yvgH/lPSKpOck3SFplbzjqo2kHpJuXMTnHiCpR8H9sZJWa4SYCkdbfUHSbxq6ziK2uWVNI6xay+akYbnKOpLdDNwfEStFxGqkvgPL5BtZ7SJidkQMXcSnHwB8mzQi4pCIeK5RAksjmQ4g9cc4sQmNTWYtiJOG5W0r4JusYxkAEfFURDyQ9ZY+V9IzkqZL2gu+/Yc7VdL1kl6SNErSMKW5D6ZLWikrd4WkvyjNIfGqpC2U5uB4XtIVVduT9FnB7aFVy7LnXyTp/7LnD80e76NsDg+lAe3Oy7Y7TdIR2eMnS3osi31M9lqGAhXA+OyIoLOk+yVVZM/ZJ1vPM5JGF8Yn6Q9Kgwg+LKnOhBoR75M6DXbPnt9b0pQsvimSehW8vm+TX9V+yPbv/UoD8L0gaXyW3KvmfHlB0n+A3Queu4W+m1PiSUmLFff2W3PjpGF5WwOobeC13YEBwNqkIRzO1XfDNqwNHAWsSerlvEpEbEAaKvqIgnUsCWwN/IY0uNwFwOrAmpIGFBFfd2BTYGdgVA3LRwArAOtkg9aNzx7/c0SsHxFrAJ2BnSPiRqASGBYRAyLii6qVZFVWo7NYBwDrS9o1W/xD4OFsEMF/Az+vK+AsKXQCplXFQhqGvSq+i4p43esAvwZWI83xsomkTqTxlXYBNgOWLSh/DPCr7EhnM1JvcmuBnDSsKdsUuDYbafQdYCqwfrbssWx+iK9Ik1fdnT0+HehTsI5bs2EwpgPvRMT0iFgAPFutXG1uiYgFWRVSTf/wB5Emu5kHEBFVg8JtpTRj2nRSIljYEOPrk6ro5mbrGk+aZArga6Cq7eDxOuLeS9KzwKvAnyLiy+zxjUiT9QBcRdqvC/NoRLyZ7aunsm2uCrwWETOyfVo4SN6DwB8lHQn8qGp/WMvjpGF5e5baRwata9KYrwpuLyi4v4Dvj6n2VQ1lqpcrHEunUx3bqSkeVXs+2T/yS4GhEbEm6d959fXWtJ7afFMw/tN8ah8z7rqIWJ30T/98ScvWUq5qXfPIfgOy6qcOBWUKX3fhNmscdygiRgGHkI6qHpa0au0vx5ozJw3L271AR0nfVrlIWl/SFqSqmL2ydoNupH/ej5Yghnck9ZfUBtitns+9G/il0vDTSFqK7xLEe0pzVxQ2mn9Kmmq1ukeALSR1VRogbx/SkVW9RcRDpCOKqjnA/4/vBqwbBvwnu/063yXsIWSDPdbhBWCFqjajLEYAJK2UHcWNJlXBOWm0UE4alqvsH/RuwDZKp9w+C5xKmlviZlK9/NOk5HJcRLxdgjBGkqp/7gXqOzz5WGAmME3S08C+2dwLfyVVid1CGva+yhXAZVUN4VUPZsOinwDcR3q9T0TEhEV7OUBqHzkwa5A+Mrs9jdT+U5VM/kpKVI8CPwH+W9cKs+quEcDtWUP4GwWLf5014D9Nas+Y1IDYrQnzKLdmZlY0H2mYmVnRnDTMzKxoThpmZlY0Jw0zMyuak4aZmRXNScPMzIrmpGFmZkX7f1xQ4MKNJuvcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# PLOTTING (optional)\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.use('Agg')\n",
    "\n",
    "# Plot Loss curve\n",
    "plt.figure()\n",
    "plt.title('Training Loss vs Communication rounds')\n",
    "plt.plot(range(len(train_loss)), train_loss, color='r')\n",
    "plt.ylabel('Training loss')\n",
    "plt.xlabel('Communication Rounds')\n",
    "# plt.savefig('../save/fed_{}_{}_{}_C[{}]_iid[{}]_E[{}]_B[{}]_loss.png'.\n",
    "#             format(args.dataset, args.model, args.epochs, args.frac,\n",
    "#                    args.iid, args.local_ep, args.local_bs))\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function matplotlib.pyplot.show(*args, **kw)>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xd4FOXax/HvTShSBaSKdBVpESWgokexcGwUCwoamogcOBQRRHpHpAgiAoIIIiCiWCgWUERFXkEJHkikKEWUJkR6h5D7/WMmcYkpG8hkUu7PdeW6dnZmZ3472Z175pnZeURVMcYYYwBy+B3AGGNMxmFFwRhjTDwrCsYYY+JZUTDGGBPPioIxxph4VhSMMcbEs6JgjLmAiJQTkeMiEuLBvI+LSKW0nq+fRKSCiKiI5PQ7S1qwonAJROQbETkkInn8zpKWRGSmiMSIyJV+Z8noRORaEZkvIn+JyBERiRSR7l5sUNOLqv6hqgVU9fylzMf9frRLMO8Cqrr90hIaL1lRuEgiUgH4F6BAY4+Wke57HiKSH3gUOAKEp/OyM9WelohUBn4AdgI1VfVy4DEgDCjoZ7bsLLN9jjIcVbW/i/gDBgL/B4wDPgl4/mbgTyAk4LmHgUj3cQ6gN7ANOAC8DxR1x1XAKTJPA38AK9zn57vzPAKsAKoHzPsKYDFwFFgDDAdWBoy/DvgSOAj8AjyewvtqhbORexb4OcG4EKCvm/0YsBYo646rHrCcfUBf9/mZwPCAedQHdgUM7wB6AZHAGSBnwPo5BmwEHk6Q4xlgU8D4G4GewIcJpnsNGJ/Ie+wNfJDguVeBCe7jNsB2d/6/AeFJrKs5wKcprM/GwAbgMPANUDXBe+/pvvcTwHSgJPC5u+xlQJEEn42n3P/PIaADUMd9/WFgYsC8BwNzAobjXp/THf4GGIbzGT4GfAEUS2LaosBbwB53uQvc54sAnwDR7vOfAFe5414EzgOngeNx2dz5Xu0+vhyY5b7+d6A/kCPgf7ASeNmd92/A/cms5x3883NU1X2fh93/QeOA6b8B2gUMt+HC742663eLu/xJgAR8D14G/sL5nHRKsL7aEMTnJ6P++R4gs/4BW4H/ArWBc0DJgHHbgAYBw/OB3u7jbsBq4CogDzAVeNcdF/dlnAXkB/K6z7fF2fPMA4wH1gXMe577lw+ohrPBWOmOy+8OP+V+SW50P8jVk3lfXwGjcTZOMcCNAeN6AlFAFUCA63GKUkFgL9ADuMwdvsl9zUxSLgrrgLIB7/cx4EqcAtoMZ4NZOmDcbpyNoQBXA+WB0u50hd3pcgL7gdqJvMfywEmgkDsc4ua/2V1nR4Eq7rjSSa0vnEL9VDLr8lo3UwMgF/CC+7nJHfDeV7vruoyb9yfgBvd/vRwYlOCzMcVdx//G2eAuAEoEvP4Od/rBpFwUtrkZ87rDI5OY9lPgPZwikCtgGVfgHFXmc//n83ELRsAy2iVYJ4FFYRaw0H1tBeBX4Gl3XBuc79Uz7v+nI05RkiTW9Q4CPkduzq04OzG5gbtwNtJVEstG4kXhE6AwUA6ncN3njusAbHaXVRT4Om59kYrPT0b98z1AZvwDbnM/sHF7VpuB5wLGDwdmuI8L4mwYyrvDm4C7A6Yt7c4rZ8CXsVIyyy7sTnO5+2U5F/cBDFh2XFFoBnyX4PVTcTc0icy7HBAL1HKHlwKvBoz/BWiSyOueAP6XxDxnknJRaJvC+l4Xt1w307NJTPc58Iz7uCGwMZl5rgRauY8bANvcx/lx9iwfxS1SyczjXNyGIonxA4D3A4Zz4BS0+gHvPTxg/IfA6wHDXfh7rzzus1EmYPwBoFmC13dzHw8m5aLQP2D8f4ElCad1P5+xuEcsKayPWsChgOFvSKIouJ/dM0C1gHH/Ab5xH7cBtgaMy+e+tlQSy77gc4TTtPsn7pGH+9y7wODEspF4UbgtYPh9/t6xWw50CBj3by4sCkF9fjLqn51TuDitgS9U9S93eK77HAHDj7gnoB8BflLV391x5YGPReSwiBzGKRLncfYW4+yMeyAiISIyUkS2ichRnA8/QDGgOM4HcWdir3WXdVPcstzlhQOlknhfLYFNqrrOHX4HeFJEcrnDZXH2LhNK6vlgBWZGRFqJyLqAzDVw3m9Ky3obaOE+bgHMTmaZc3GKGcCT7jCqegKnmHYA9orIpyJyXRLzOICz0UzKlTjNIrjzjsV5r2UCptkX8PhUIsMFEswztdMn58+AxyeTeG1Z4KCqHko4QkTyichUEfnd/WyuAAoHeZK9GM4e/O8Bz/3OhesmPp+qnnQfJvf+Aj9HVwI73XWe1PxTktT6uTLBsgL/x6n5/GRIVhRSSUTyAo8Dd4jInyLyJ/AccL2IXA+gqhtxPij3E7DBce3EaRstHPB3maruDphGAx4/CTQB7sE5OqgQFwXnkDYGpykqTtkEy/o2wbIKqGrHJN5eK6BSwPsah/PlvT9gfpUTeV1Sz4NzlJQvYDixghT/fkWkPDAN6AxcoaqFgZ9x3m9Ky1oAhIpIDZwjhXeSmA6cpo76InIVzjmf+P+Rqi5V1QY4G/zNbp7ELMPZI0zKHpzCHPfeBOf/szvJV6SdYNZ7MHYCRUWkcCLjeuA0Jd6kqoWA293n4/5Xmshr4vyFc6RVPuC5clzauglc3h6grIgEbuMC538p62cvF37Pyl0QIvjPT4ZkRSH1HsLZs6+Gc7hcC+eE1nc4G9U4c4GuOF+U+QHPTwFedDd+iEhxEWmSzPIK4hxmH8D5EI+IG6HOJYMfAYPdvbbrEmT4BLhWRFqKSC73r46IVE24EBG5BWdjWzfgfdXgwqOgN4FhInKNOEJF5Ap3OaVEpJuI5BGRgiJyk/uadcADIlJURErhnFNJTn6cL3e0m+spN0ecN4HnRaS2m+HquHWpqqeBD9zMP6rqH0ktRFWjcZoQ3gJ+U9VN7vJKikhj9yqsMzgnSZO6NHMQUE9ExrjvDTfPHHcj+j7woIjc7R5t9XDn+X0K6yAtrANud39zcDnQ52Jmoqp7cZrlJotIEfczFLfxL4hzdHJYRIrirI9A+4BEf5Pgfnbfx/kuFHT/h91xTt6nhR9wNvwvuJnrA41wzr+Bs34ecb83V+Nc3BGs94GuInKViBTBuXABSPXnJ0OyopB6rYG31LmW+8+4P2AiEB5wOdy7OO3nywOamcC5ymUR8IWIHMM50XgTSZuFc9SxG+dKm9UJxnfGOYL4E6e55F2cDyOqegynvbM5zp7Tn8AonJOYib2vhaoaleB9vQo0dL/043C+EF/gnEybjtNuegynXb6Ru4wtwJ3ufGcD63Gavb7AOWGZJPcoayywCmejUhPnCpm48fNxrmyZi3PicAHOyb44b7uvSa7pKM5cnCOwwCO5HDgb7z04V1LdgdPenljWbcAtOEdvG0TkCE67fgRwTFV/wWnGeg1nz7gR0EhVzwaR7ZKo6pc46zoS5yqxTy5hdi1x9uo345zMjivs43FO6v6F87lckuB1rwJN3d/yTEhkvl1wNtzbcc7xzAVmXELOeO46boxzlPsXMBnnHNJmd5JXgLM4n7G3Sf6oMqFpOOe21uNcGPBRwLigPz8ZVdwlViaLEJFROCfjWqc4cRYkIuVwNl6lVPWo33mMyWzsSCGTE5Hr3GYcEZG6OIfBH/udyw9u+3F3YJ4VBGMujv3yL/MriNNkdCXOof1YnGu/sxW3DXcfTlPbfT7HMSbTsuYjY4wx8az5yBhjTLxM13xUrFgxrVChgt8xjDEmU1m7du1fqlo8pekyXVGoUKECERERfscwxphMRUR+T3kqaz4yxhgTwIqCMcaYeFYUjDHGxLOiYIwxJp4VBWOMMfGsKBhjjIlnRcEYY0y8TPc7BWOMyS5iY2PZsWMHkZGRREZG0rBhQ2688UZPl2lFwRhjMoDDhw8TFRVFVFRUfBGIiori+PHjAIgIJUqUsKJgjDFZSUxMDFu3bo3f8EdGRrJ+/Xr++OPvjgKLFClCaGgoTz31FKGhoYSGhlK9enXy58/veT4rCsYY45G//vrrgo1/ZGQkGzZs4PTp0wCEhIRw3XXXceutt9KxY8f4AlCmTBmcLr3TnxUFY4y5RGfPnmXz5s3/KAB79+6Nn6ZkyZKEhobSqVOn+I1/1apVyZMnsd5x/eNpURCR+3D6aQ0B3lTVkYlM8zgwGKez9vWq+qSXmYwx5mKpKnv37v3Hxn/Tpk3ExMQAkDt3bqpXr86///3v+I1/zZo1KVmypM/pg+NZURCREGASTofuu4A1IrLI7Zg9bpprgD7Arap6SERKeJXHGGNS4+TJk2zcuPEfBeDAgQPx05QtW5bQ0FAaNmwYXwCuueYacuXK5WPyS+PlkUJdYKuqbgcQkXlAE2BjwDTPAJNU9RCAqu73MI8xxvyDqvL777//Y+O/ZcsWYmNjAciXLx81a9bkkUceuWDvv0iRIj6nT3teFoUywM6A4V3ATQmmuRZARP4Pp4lpsKouSTgjEWkPtAcoV66cJ2GNMVnf0aNH+fnnny/Y+EdFRXH06NH4aSpXrkxoaCjNmzePLwCVKlUiR47s8VtfL4tCYqfOE3YInRO4BqgPXAV8JyI1VPXwBS9SfQN4AyAsLMw6lTbGJOv8+fNs27btH3v/v/32W/w0l19+OaGhobRs2TJ+41+jRg0KFCjgY3L/eVkUdgFlA4avAvYkMs1qVT0H/CYiv+AUiTUe5jLGZCEHDhy44AdfkZGR/Pzzz5w6dQqAHDlyUKVKFerWrUu7du3iC0DZsmV9u+wzI/OyKKwBrhGRisBuoDmQ8MqiBcATwEwRKYbTnLTdw0zGmEzq3Llz/PLLL//Y+9+9e3f8NMWKFeP666+nQ4cOF1z2mTdvXh+TZy6eFQVVjRGRzsBSnPMFM1R1g4gMBSJUdZE77t8ishE4D/RU1QNJz9UYk9WpKvv27fvHxn/jxo2cO3cOgFy5clGtWjXuuuuu+I1/aGgoJUuWtL3/SySqmauJPiwsTCMiIvyOYYxJA6dPn070ss/o6Oj4acqUKXPBhj80NJQqVapk6ss+/SAia1U1LKXp7BfNxph0dfbsWWbOnMmkSZPYsGED58+fByBv3rzUqFGDxo0bX3DZ5xVXXOFz4uzFioIxJl2cPn2a6dOnM3LkSHbt2kWdOnXo27dvfAGoXLkyISEhfsfM9qwoGGM8dfLkSaZOncqYMWPYu3cvt912GzNmzOCee+6x9v8MyIqCMcYTx44dY/LkyYwdO5bo6Gjuuusu5s6dyx133GHFIAOzomCMSVOHDx/mtddeY/z48Rw8eJD77ruPAQMGUK9ePb+jmSBYUTDGpImDBw8yfvx4JkyYwJEjR2jUqBEDBgygTp06fkczqWBFwRhzSfbv38+4ceOYNGkSx48f59FHH6V///7UqlXL72jmIlhRMMZclL179zJmzBimTJnC6dOnad68Of369aN69ep+RzOXwIqCMSZVdu7cyejRo5k2bRoxMTGEh4fTt29fqlSp4nc0kwasKBhjgvLbb78xcuRI3nrrLQBat25Nnz59qFSpks/JTFqyomCMSdaWLVsYMWIEs2fPJiQkhGeeeYZevXpZ3yZZlBUFY0yiNm3axIsvvsi7775L7ty56dy5Mz179qRMmTJ+RzMesqJgjLlAZGQkw4cP54MPPiBfvnz06NGDHj16ZJqO582lsaJgjAFg7dq1DBs2jIULF1KoUCH69u1Lt27dKFasmN/RTDqyomBMNrd69WqGDRvGZ599RuHChRk8eDBdu3bNkp3Sm5RZUTAmm1qxYgXDhg1j2bJlFCtWjBEjRtCpUycKFSrkdzTjIysKxmQjqsry5csZOnQoK1asoGTJkrz88st06NCB/Pnz+x3PZABWFIzJBlSVJUuWMGzYMFatWkWZMmWYMGEC7dq1s/6LzQVy+B3AGOMdVWXhwoXUrVuXBx54gD179vD666+zbds2unTpYgXB/IMVBWOyoNjYWObPn0+tWrV46KGHOHToENOnT2fLli106NCBPHny+B3RZFBWFIzJQs6fP8/cuXOpWbMmjz/+OGfOnGH27Nls3ryZtm3bWmf3JkVWFIzJAs6dO8fMmTOpWrUq4eHh5MiRg3nz5rFhwwZatGhBzpx2+tAExz4pxmRiZ8+eZebMmbz00kvs2LGDG264gY8++ogmTZqQI4ft85nUs0+NMZnQ6dOnmTRpEpUrV+Y///kPJUqUYPHixaxdu5aHH37YCoK5aHakYEwmcvLkSaZOncqYMWPYu3cvt912G9OnT6dBgwaIiN/xTBZgRcGYTODYsWNMnjyZsWPHEh0dzV133cXcuXO54447rBiYNGVFwZgM7PDhw7z22muMHz+egwcPcu+99zJgwABuvfVWv6OZLMqKgjEZ0MGDBxk/fjwTJkzgyJEjNGrUiP79+1O3bl2/o5kszoqCMRnI/v37GTduHJMmTeL48eM8+uij9O/fn1q1avkdzWQTVhSMyQD27t3LmDFjmDJlCqdPn6ZZs2b069ePGjVq+B3NZDOeXrcmIveJyC8islVEeicyvo2IRIvIOvevnZd5jMlodu7cSZcuXahYsSITJkzgscceY9OmTbz77rtWEIwvPDtSEJEQYBLQANgFrBGRRaq6McGk76lqZ69yGJMR/fbbb4wcOZK33noLVaVNmzb07t2bypUr+x3NZHNeNh/VBbaq6nYAEZkHNAESFgVjso0tW7YwYsQIZs+eTUhICO3ataNXr16UL1/e72jGAN42H5UBdgYM73KfS+hREYkUkQ9EpGxiMxKR9iISISIR0dHRXmQ1xlObNm2iRYsWXHfddcybN4/OnTuzfft2Jk+ebAXBZCheFoXEflGjCYYXAxVUNRRYBryd2IxU9Q1VDVPVsOLFi6dxTGO8ExkZyeOPP0716tVZsGABPXr0YMeOHYwfP54yZRLbRzLGX142H+0CAvf8rwL2BE6gqgcCBqcBozzMY0y6Wbt2LcOGDWPhwoUULFiQvn370q1bN4oVK+Z3NGOS5WVRWANcIyIVgd1Ac+DJwAlEpLSq7nUHGwObPMxjjOdWr17NsGHD+OyzzyhcuDCDBw+ma9euFClSxO9oxgTFs6KgqjEi0hlYCoQAM1R1g4gMBSJUdRHQVUQaAzHAQaCNV3mM8dL333/PoEGDWLZsGVdccQUjRoygU6dOFCpUyO9oxqSKqCZs5s/YwsLCNCIiwu8YxgBOM9GAAQP4/PPPKVGiBD179qRDhw4UKFDA72jGXEBE1qpqWErT2S+ajbkIkZGRDBo0iAULFlC0aFFGjRpFp06dyJ8/v9/RjLkkVhSMSYXNmzczePBg3nvvPQoVKsTQoUN59tlnrZnIZBlWFIwJwrZt2xg6dChz5swhb9689OvXjx49etgJZJPlpFgURKSGqv6cHmGMyWj++OMPhg8fzowZM8iVKxfdu3fnhRdewH4vY7KqYI4UpohIbmAmMFdVD3sbyRj/7dmzhxEjRjBt2jQA/vvf/9KnTx9Kly7tczJjvJViUVDV20TkGqAtECEiPwJvqeqXnqczJp3t37+fUaNGMXnyZGJiYmjbti39+/enbNlE78BiTJYT1DkFVd0iIv2BCGACcIM4HcP2VdWPvAxoTHo4ePAgY8eO5dVXX+XUqVO0bNmSgQMHUqlSJb+jGZOugjmnEAo8BTwIfAk0UtWfRORKYBVgRcFkWkePHuWVV15h3LhxHDt2jGbNmjFo0CCuu+46v6MZ44tgjhQm4tyXqK+qnop7UlX3uEcPxmQ6J06cYOLEiYwePZqDBw/y8MMPM2TIEGrWrOl3NGN8FUxReAA4parnAUQkB3CZqp5U1dmepjMmjZ06dYopU6YwcuRI9u/fzwMPPMDQoUOpXbu239GMyRCCuXX2MiBvwHA+9zljMo2zZ88yefJkrr76arp3707NmjX5/vvv+fTTT60gGBMgmKJwmaoejxtwH+fzLpIxaScmJoYZM2Zw7bXX0qlTJypVqsTXX3/NsmXLuOWWW/yOZ0yGE0xROCEiN8YNiEht4FQy0xvju/PnzzNnzhyqVq3K008/TYkSJVi6dCkrVqygfv36fsczJsMK5pxCN2C+iMR1kFMaaOZdJGMuXmxsLB999BEDBw5k06ZNXH/99SxatIiGDRviXEVtjElOMD9eWyMi1wFVcLrY3Kyq5zxPZkwqqCqLFy9m4MCBrF+/nqpVqzJ//nweeeQRcuTwstdZY7KWYL8tVYBqwA3AEyLSyrtIxgRPVVm6dCk33XQTTZo04cSJE8yZM4eoqCiaNm1qBcGYVArmx2uDgPo4ReEz4H5gJTDL02TGpODbb7+lf//+rFy5kvLlyzN9+nRatWpFzpx2819jLlYwu1FNgbuBP1X1KeB6II+nqYxJxqpVq7jnnnuoX78+27dvZ/Lkyfz666+0bdvWCoIxlyiYonBKVWOBGBEpBOwH7IYwJt2tXbuWBx98kHr16hEVFcUrr7zC1q1b6dixI7lz5/Y7njFZQjC7VREiUhjnVhdrgePAj56mMiZAVFQUgwYN4uOPP6Zo0aKMHDmSzp07W9eXxngg2aLg3gn1JbcPhSkisgQopKqR6ZLOZGu//PJLfNeXBQsWZMiQIXTr1s26vjTGQ8kWBVVVEVkA1HaHd6RHKJO9bd++naFDhzJ79mzy5s1Lnz596NGjB0WLFvU7mjFZXjDNR6tFpI6qrvE8jcnWdu7cGd/1Zc6cOXnuuefo1auXdX1pTDoKpijcCfxHRH4HTuD8gE1VNdTTZCbb2Lt3Ly+99BJTp05FVenQoQN9+vThyiuv9DuaMdlOMEXhfs9TmGwpOjqa0aNHM2nSJM6ePRvf9WW5cuX8jmZMthVMUVDPU5hs5dChQ4wdO5bx48dz6tQpWrRowcCBA6lcubLf0YzJ9oIpCp/iFAYBLgMqAr8A1T3MZbKgo0eP8uqrrzJ27FiOHDlCs2bNGDx4sHV9aUwGEswN8S7on9C9jfZ/PEtkspwTJ04wadIkRo0axcGDB3nooYcYMmQIoaF2WsqYjCbVdwtT1Z+AOh5kMVnM6dOnefXVV6lUqRK9evXipptuYs2aNXz88cdWEIzJoIK5IV73gMEcwI1AtGeJTKZ39uxZZsyYwfDhw9m9ezd33XUXw4YNo169en5HM8akIJgjhYIBf3lwzjE0CWbmInKfiPwiIltFpHcy0zUVERWRsGDmazKmmJgY3nrrLapUqULHjh2pUKECy5cv56uvvrKCYEwmEcw5hSEXM2MRCQEmAQ2AXcAaEVmkqhsTTFcQ6Ar8cDHLMf47f/488+bNY8iQIWzZsoWwsDBef/117r33XuvtzJhMJsUjBRH50r0hXtxwERFZGsS86wJbVXW7qp4F5pH4EcYwYDRwOsjMJoOIjY3lww8/JDQ0lBYtWpA3b14WLlzIjz/+yH333WcFwZhMKJjmo+LuDfEAUNVDQIkgXlcG2BkwvMt9Lp6I3ACUVdVPkpuRiLQXkQgRiYiOttMZfovr+rJ27do0bdqU2NhY3nvvPf73v//RuHFjKwbGZGLBFIXzIhL/E1MRKU9wP2hLbMsQ/zoRyQG8AvRIaUaq+oaqhqlqmN0Hxz+qyhdffMHNN99M48aNOXbsGLNnz+bnn3/m8ccft64vjckCgvnxWj9gpYh86w7fDrQP4nW7gLIBw1cBewKGCwI1gG/cPctSwCIRaayqEUHM36SjFStW0L9/f7777jvKlSvHm2++SatWrciVK5ff0YwxaSiYE81L3B+s3Yyz9/+cqv4VxLzXANeISEVgN9AceDJgvkeAYnHDIvIN8LwVhIxl9erVDBgwgGXLllG6dGkmTpxIu3btyJPHemQ1JisK5kTzw8A5Vf1EVRfjdMv5UEqvU9UYoDOwFNgEvK+qG0RkqIg0vtTgxlt//vknDRs25JZbbmH9+vWMGzeObdu20alTJysIxmRhopr86QERWaeqtRI89z9VvcHTZEkICwvTiAg7mPBSbGwsDRo0YPXq1fTv358uXbpQoEABv2MZYy6BiKxV1RR/CxbMOYXEjiaCeZ3JpMaNG8fy5ct58803efrpp/2OY4xJR8FcLhIhIuNEpLKIVBKRV4C1Xgcz/li3bh19+/bl4Ycfpm3btn7HMcaks2CKQhfgLPAeMB/nR2b/9TKU8cepU6d48sknKVasGNOmTbPfGxiTDQVz9dEJIP6+RSJyGdAIp0CYLKRXr15s2rSJL774giuuuMLvOMYYHwT1ayMRCRGR+0VkFrADaOZpKpPulixZwmuvvUa3bt1o0KCB33GMMT5J9khBRG7H+W3Bg8CPwK1AJVU9mQ7ZTDqJjo6mTZs21KhRg5deesnvOMYYHyVZFERkF/AH8DrQU1WPichvVhCyFlWlXbt2HDp0iC+++ILLLrvM70jGGB8l13z0Ic4N7JoBjUQkP8Hd88hkItOmTWPRokWMHDnSekMzxiRdFFT1WaACMA64E/gVKC4ij4uI/ZIpC/j111957rnnuOeee3j22Wf9jmOMyQCSPdGsjuWq+gxOgXgSeAjnZLPJxM6dO0d4eDiXXXYZb7/9tt3h1BgDpOKXyap6DlgMLBaRvN5FMulhyJAhRERE8OGHH3LllVf6HccYk0Fc1O6hqp5K6yAm/axcuZKXXnqJtm3b8sgjj/gdxxiTgVibQTZz5MgRWrRoQcWKFRk/frzfcYwxGUzQzUcikt/9dbPJxDp37syuXbtYuXIlBQsW9DuOMSaDCaY/hXoishGnTwRE5HoRmex5MpPm5s2bx5w5cxgwYAA333yz33GMMRlQMM1HrwD3AgcAVHU9TpecJhP5448/6NChA7fccgv9+vXzO44xJoMK6pyCqu5M8NR5D7IYj5w/f55WrVpx/vx55syZQ86c1h2GMSZxwWwddopIPUBFJDfQFbcpyWQOY8eO5dtvv+Wtt96iUqVKfscxxmRgwRwpdAA64dzyYhdQyx02mcBPP/1E//79adq0Ka1bt/Y7jjEmgwumP4W/gPB0yGLS2MmTJwkPD6d48eJMmTLFOs0xxqQoxaIgIhMSefoIEKGqC9M+kkkrPXv2ZPPmzXz55ZfWaY4xJijBNB9dhtNktMX9CwWKAk+LiP36KYP69NMZmnIoAAAV7klEQVRPmTx5Mt27d+eee+7xO44xJpMQ1eTvhi0iy4F/q2qMO5wT+AJoAESpajXPUwYICwvTiIiI9FxkprN//35q1qxJqVKl+PHHH8mTJ4/fkYwxPhORtaoaltJ0wVx9VAbIj9NkhPv4SlU9LyJnLiGj8YCq8vTTT3PkyBGWL19uBcEYkyrBFIXRwDoR+QYQnB+ujXA73VnmYTZzEaZOnconn3zCq6++SvXq1f2OY4zJZFJsPgIQkdJAXZyi8KOq7vE6WFKs+Shpmzdv5sYbb+Rf//oXn3/+ufWRYIyJF2zzUbBbjdPAXuAgcLWI2G0uMpizZ88SHh5Ovnz5mDlzphUEY8xFCeaS1HbAs8BVwDrgZmAVcJe30UxqDBo0iJ9++omPP/6Y0qVL+x3HGJNJBbM7+SxQB/hdVe8EbgCiPU1lUuXbb79l1KhRtGvXjoceesjvOMaYTCyYonBaVU8DiEgeVd0MVPE2lgnW4cOHadWqFZUrV+aVV17xO44xJpMLpijsEpHCwALgSxFZCAR1ollE7hORX0Rkq4j0TmR8BxGJEpF1IrJSRNL1Nw9ZQadOndi9ezfvvPMOBQoU8DuOMSaTC+beRw+7DweLyNfA5cCSlF4nIiHAJJwfue0C1ojIIlXdGDDZXFWd4k7fGBgH3Je6t5B9zZ07l7lz5zJs2DDq1q3rdxxjTBaQbFEQkRxApKrWAFDVb1Mx77rAVlXd7s5rHtAEiC8Kqno0YPr8QMrXxxoAfv/9dzp27Ei9evXo3fsfB2HGGHNRkm0+UtVYYL2IlLuIeZcBAjvn2eU+dwER6SQi23B+JNc1sRmJSHsRiRCRiOhoO8d9/vx5WrZsiapapznGmDQVzDmF0sAGEflKRBbF/QXxusTu0/yPIwFVnaSqlYFeQP/EZqSqb6hqmKqGFS9ePIhFZ22jR4/mu+++Y+LEiVSsWNHvOMaYLCSYXcwhFznvXUDZgOGrSP4E9Tzg9YtcVrYRERHBwIEDefzxx2nZsqXfcYwxWUyKRwrueYQdQC738RrgpyDmvQa4RkQqut14NgcuOMIQkWsCBh/EuTW3ScKJEycIDw+nVKlS1mmOMcYTwfyi+RmgPU4fCpVxzgtMAe5O7nWqGiMinYGlQAgwQ1U3iMhQnA56FgGdReQe4BxwCLD+IpPx/PPPs2XLFr766iuKFCnidxxjTBYUTPNRJ5wriX4AUNUtIlIimJmr6mfAZwmeGxjw+Nngo2ZvixcvZsqUKTz//PPceeedfscxxmRRwZxoPqOqZ+MG3E527NLRdLRv3z6efvppatWqxfDhw/2OY4zJwoIpCt+KSF8gr4g0AOYDi72NZeKoKm3btuXYsWO888471mmOMcZTwRSF3jg3wIsC/oPTHJTopaMm7U2ePJnPPvuMMWPGUK2a3QXEGOOtYM4pNAFmqeo0r8OYC23cuJHnn3+e+++/n06dOvkdxxiTDQRzpNAY+FVEZovIg+45BeOxuE5zChQowIwZM+zyU2NMugjmdwpPAVfjnEt4EtgmIm96HSy7GzBgAOvWrWP69OmUKlXK7zjGmGwiqL1+VT0nIp/jXHWUF6dJqZ2XwbKzb775hjFjxtC+fXsaN27sdxxjTDaS4pGC2yfCTGAr0BR4E+d+SMYDhw4domXLllxzzTWMGzfO7zjGmGwmmCOFNjj3JfqPqp7xNk72pqp07NiRP//8k1WrVpE/f36/IxljsplgOtlpHjgsIrcCT6qqXQ6TxubMmcN7773Hiy++SFhYmN9xjDHZUFDnFESkFs5J5seB34CPvAyVHe3YsYNOnTrxr3/9i169evkdxxiTTSVZFETkWpw7mz4BHADeA0RV7cY7aSyu0xwRYdasWYSEhPgdyRiTTSV3pLAZ+A5opKpbAUTkuXRJlc2MHDmSlStXMnv2bCpUqOB3HGNMNpbc1UePAn8CX4vINBG5m8R7UzOXYM2aNQwePJjmzZsTHh7udxxjTDaXZFFQ1Y9VtRlwHfAN8BxQUkReF5F/p1O+LO348eOEh4dTunRpXn/9dfvVsjHGd8H8ovmEqr6jqg1xutRch3OTPHOJunfvztatW5k9ezaFCxf2O44xxgR176N4qnpQVaeq6l1eBcouFixYwLRp03jhhRe44447/I5jjDFAKouCSRt79+6lXbt23HDDDQwdOtTvOMYYE8+KQjqL6zTnxIkTvPPOO+TOndvvSMYYE89ug53OJk6cyJIlS5g0aRJVq1b1O44xxlzAjhTS0YYNG+jZsycPPvggHTt29DuOMcb8gxWFdHLmzBmefPJJChUqxPTp0+3yU2NMhmTNR+mkX79+REZGsnjxYkqWLOl3HGOMSZQdKaSDr776irFjx9KxY0caNmzodxxjjEmSFQWPHTx4kNatW1OlShVefvllv+MYY0yyrPnIQ6pKhw4d2LdvHwsXLiRfvnx+RzLGmGRZUfDQrFmzmD9/Pi+99BK1a9f2O44xxqTImo88sn37djp37sztt99Oz549/Y5jjDFBsaLggZiYGFq0aEFISAizZ8+2TnOMMZmGNR95YMSIEaxatYq5c+dSrlw5v+MYY0zQPD1SEJH7ROQXEdkqIv+43baIdBeRjSISKSJfiUh5L/Okhx9++IGhQ4cSHh7OE0884XccY4xJFc+KgoiEAJOA+4FqwBMiUi3BZP8DwlQ1FPgAGO1VnvRw/PhxWrRoQZkyZZg4caLfcYwxJtW8PFKoC2xV1e2qehaYBzQJnEBVv1bVk+7gapxOfDKtbt26sW3bNus0xxiTaXlZFMoAOwOGd7nPJeVp4PPERohIexGJEJGI6OjoNIyYdj7++GOmT59O7969uf322/2OY4wxF8XLopDYHd800QlFWgBhwJjExqvqG6oapqphxYsXT8OIaWPPnj20a9eO2rVrM3jwYL/jGGPMRfPy6qNdQNmA4auAPQknEpF7gH7AHap6xsM8noiNjaVNmzacPn3aOs0xxmR6XhaFNcA1IlIR2A00B54MnEBEbgCmAvep6n4Ps3hmwoQJfPnll0yZMoUqVar4HccYYy6JZ81HqhoDdAaWApuA91V1g4gMFZHG7mRjgALAfBFZJyKLvMrjhaioKHr37k2jRo1o376933GMMeaSiWqizfwZVlhYmEZERPgdg9OnT1O3bl327dtHVFQUJUqU8DuSMcYkSUTWqmpYStPZL5ovUt++fYmKiuLTTz+1gmCMyTLs3kcX4csvv+SVV16hU6dOPPDAA37HMcaYNGNFIZUOHDhAmzZtqFq1KmPGJHoFrTHGZFrWfJQKqkr79u2Jjo7m008/JW/evH5HMsaYNGVFIRVmzpzJRx99xOjRo6lVq5bfcYwxJs1Z81GQtm3bRteuXalfvz7du3f3O44xxnjCikIQ4jrNyZkzJ7NmzbJOc4wxWZY1HwVh+PDhrF69mnnz5lG2bNmUX2CMMZmUHSmkYNWqVQwbNoyWLVvSrFkzv+MYY4ynrCgk49ixY7Ro0YJy5cpZpznGmGzBmo+S0bVrV3bs2MGKFSsoVKiQ33GMMcZzdqSQhA8++ICZM2fSt29fbr31Vr/jGGNMurCikIjdu3fTvn176tSpw8CBA/2OY4wx6caKQgKxsbG0bt2aM2fOMGfOHHLlyuV3JGOMSTd2TiGB8ePH89VXX/HGG29w7bXX+h3HGGPSlR0pBFi/fj19+vShSZMmtGvXzu84xhiT7qwouE6dOkV4eDhFixblzTffRET8jmSMMenOmo9cvXv3ZsOGDSxZsoRixYr5HccYY3xhRwrA0qVLmTBhAl26dOHee+/1O44xxvgm2xeFv/76izZt2lC9enVGjRrldxxjjPFVtm4+UlWeeeYZDh48yJIlS6zTHGNMtpeti8L06dNZsGABL7/8Mtdff73fcYwxxnfZtvloy5YtPPvss9x9990899xzfscxxpgMIVsWhXPnzhEeHk6ePHmYOXMmOXJky9VgjDH/kC2bj4YNG8aaNWt4//33ueqqq/yOY4wxGUa220X+v//7P1588UVat27NY4895nccY4zJULJVUTh69CgtWrSgfPnyTJgwwe84xhiT4WSr5qMuXbrwxx9/8N1331mnOcYYk4hsc6Tw/vvvM2vWLPr370+9evX8jmOMMRlStikKRYoUoUmTJvTv39/vKMYYk2F5WhRE5D4R+UVEtopI70TG3y4iP4lIjIg09TJLgwYNWLBggXWaY4wxyfCsKIhICDAJuB+oBjwhItUSTPYH0AaY61UOY4wxwfPyRHNdYKuqbgcQkXlAE2Bj3ASqusMdF+thDmOMMUHysvmoDLAzYHiX+1yqiUh7EYkQkYjo6Og0CWeMMeafvCwKiXVdphczI1V9Q1XDVDWsePHilxjLGGNMUrwsCruAsgHDVwF7PFyeMcaYS+RlUVgDXCMiFUUkN9AcWOTh8owxxlwiz4qCqsYAnYGlwCbgfVXdICJDRaQxgIjUEZFdwGPAVBHZ4FUeY4wxKfP0Nheq+hnwWYLnBgY8XoPTrGSMMSYDENWLOvfrGxGJBn6/yJcXA/5KwzhpxXKljuVKvYyazXKlzqXkKq+qKV6pk+mKwqUQkQhVDfM7R0KWK3UsV+pl1GyWK3XSI1e2ufeRMcaYlFlRMMYYEy+7FYU3/A6QBMuVOpYr9TJqNsuVOp7nylbnFIwxxiQvux0pGGOMSYYVBWOMMfGyZFEIonOfPCLynjv+BxGpkEFytRGRaBFZ5/61S6dcM0Rkv4j8nMR4EZEJbu5IEbkxg+SqLyJHAtbXwMSmS+NMZUXkaxHZJCIbROTZRKZJ9/UVZC4/1tdlIvKjiKx3cw1JZJp0/z4GmcuX76O77BAR+Z+IfJLIOG/Xl6pmqT8gBNgGVAJyA+uBagmm+S8wxX3cHHgvg+RqA0z0YZ3dDtwI/JzE+AeAz3HufHsz8EMGyVUf+CSd11Vp4Eb3cUHg10T+j+m+voLM5cf6EqCA+zgX8ANwc4Jp/Pg+BpPLl++ju+zuOJ2P/eP/5fX6yopHCvGd+6jqWSCuc59ATYC33ccfAHeLSGK3+k7vXL5Q1RXAwWQmaQLMUsdqoLCIlM4AudKdqu5V1Z/cx8dw7uuVsJ+QdF9fQeZKd+46OO4O5nL/El7dku7fxyBz+UJErgIeBN5MYhJP11dWLArBdO4TP406N+47AlyRAXIBPOo2OXwgImUTGe+HNOswyQO3uE0An4tI9fRcsHvYfgPOXmYgX9dXMrnAh/XlNoWsA/YDX6pqkusrHb+PweQCf76P44EXgKR6pPR0fWXFohBM5z5p1gFQKgSzzMVABVUNBZbx996A3/xYX8H4Ced+LtcDrwEL0mvBIlIA+BDopqpHE45O5CXpsr5SyOXL+lLV86paC+fml3VFpEaCSXxZX0HkSvfvo4g0BPar6trkJkvkuTRbX1mxKATTuU/8NCKSE7gc75spUsylqgdU9Yw7OA2o7XGmYGXIDpNU9WhcE4A6d+TNJSLFvF6uiOTC2fC+o6ofJTKJL+srpVx+ra+A5R8GvgHuSzDKj+9jirl8+j7eCjQWkR04Tcx3icicBNN4ur6yYlEIpnOfRUBr93FTYLm6Z238zJWg3bkxTrtwRrAIaOVeVXMzcERV9/odSkRKxbWlikhdnM/zAY+XKcB0YJOqjktisnRfX8Hk8ml9FReRwu7jvMA9wOYEk6X79zGYXH58H1W1j6pepaoVcLYRy1W1RYLJPF1fnvan4AdVjRGRuM59QoAZ6nbuA0So6iKcL89sEdmKU2GbZ5BcXcXpgCjGzdXG61wAIvIuzpUpxcTp9GgQzok3VHUKTp8YDwBbgZPAUxkkV1Ogo4jEAKeA5ulQ3G8FWgJRbns0QF+gXEAuP9ZXMLn8WF+lgbdFJASnCL2vqp/4/X0MMpcv38fEpOf6sttcGGOMiZcVm4+MMcZcJCsKxhhj4llRMMYYE8+KgjHGmHhWFIwxxsSzomA8514fP09EtonIRhH5TESu9TtXUkTkShH54CJf20ZErgwYflNEqqVBpsA7dm4WkecudZ5BLLN+YnfpNFmbFQXjKffHUh8D36hqZVWthnP9fEl/kyVNVfeoatOLfHkbIL4oqGo7Vd2YJsGcu2HWwvlNQr8MdG8sk4VYUTBeuxM45/54CgBVXaeq37m/+B0jIj+LSJSINIP4PdRvReR9EflVREaKSLg497+PEpHK7nQzReR1cfoR2C4id4jTB8MmEZkZtzwROR7wuGncOPf1E0Tke/f1Td3nK4jbh4M4N0172V1upIh0cZ8fKCJr3OxvuO+lKRAGvOPu0ecVkW9EJMx9zRPufH4WkVGB+UTkRXFuVLdaRJItmKp6AOeHcaXd15cXka/cfF+JSLmA9xdf3OLWg7t+vxHnJm+bReQdt3jH9fmxWURWAo8EvPYO+btfgf+JSMHg/v0ms7GiYLxWA0jq5l6PALWA63FuMzBG/r61wPXAs0BNnF/qXquqdXFuJ9wlYB5FgLuA53BuYPYKUB2oKSK1gshXGrgNaAiMTGR8e6AicIN7Y7R33OcnqmodVa0B5AUaquoHQAQQrqq1VPVU3EzcJqVRbtZaQB0RecgdnR9Y7d6obgXwTHKB3Y3+ZUBkXBacW3XH5ZsQxPu+AegGVMPp4+NWEbkM5x4/jYB/AaUCpn8e6OQeqfwL5xfRJguyomD8dBvwrnu3yn3At0Add9wat4+AMzidE33hPh8FVAiYx2L3Vg1RwD5VjVLVWGBDgumSskBVY90mnsT20O/B6dAkBkBV4248dqc4vV5F4WzoU7oNdR2cJrRod17v4HQiBHAWiGu7X5tM7mYisgHYDryqqqfd52/B6ZAFYDbOek3Jj6q6y11X69xlXgf8pqpb3HUaeCO2/wPGiUhXoHDc+jBZjxUF47UNJH13yeQ6BjkT8Dg2YDiWC+/ZdSaRaRJOF3gvl8uSWU5ieSTB63H3qCcDTVW1Js7edcL5JjafpJwLuAfReZK+J9l7qlodZ099rIiUSmK6uHnF4H7H3eah3AHTBL7vwGUmet8bVR0JtMM5KlotItcl/XZMZmZFwXhtOZBHROKbRESkjojcgdNU0sxtty+Os+f8owcZ9olIVRHJATycytd+AXQQ5xbFiEhR/i4Af4nTf0HgSeljON1hJvQDcIeIFBPnJmxP4BwZpZqqrsI5Iojrh/l7/r4pWjiw0n28g78LchPcmwkmYzNQMe6cjZsRABGp7B6FjcJpIrOikEVZUTCecveAHwYaiHNJ6gZgME7/Ah/jtIuvxykeL6jqnx7E6I3TPLMcSO0trN8E/gAiRWQ98KR7//1pOE1WC3Buix5nJjAl7kRz3JPurbP7AF/jvN+fVHXhxb0dwDk/8ZR7wrer+zgS5/xLXLGYhlOIfgRuAk4kN0O3Oao98Kl7ovn3gNHd3BPk63HOJ3x+CdlNBmZ3STXGGBPPjhSMMcbEs6JgjDEmnhUFY4wx8awoGGOMiWdFwRhjTDwrCsYYY+JZUTDGGBPv/wGPqah/n/609gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot Average Accuracy vs Communication rounds\n",
    "plt.figure()\n",
    "plt.title('Average Accuracy vs Communication rounds')\n",
    "plt.plot(range(len(train_accuracy)), train_accuracy, color='k')\n",
    "plt.ylabel('Average Accuracy')\n",
    "plt.xlabel('Communication Rounds')\n",
    "# plt.savefig('../save/fed_{}_{}_{}_C[{}]_iid[{}]_E[{}]_B[{}]_acc.png'.\n",
    "#             format(args.dataset, args.model, args.epochs, args.frac,\n",
    "#                    args.iid, args.local_ep, args.local_bs))\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fl_pytorch",
   "language": "python",
   "name": "fl_pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}