FedLSR.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sL0Pth1es3wE",
    "outputId": "c7b54fd3-e8ed-4e15-f871-80efb6233234"
   },
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cDIYcrDiELXo"
   },
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "mJNawC4Y4QYm"
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import pstats\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "import time\n",
    "\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils, datasets\n",
    "import torch.nn.functional as F\n",
    "from torch.cuda.amp import autocast as autocast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vhTkGZhsENLN"
   },
   "source": [
    "# Experimental Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UA2hJgPEGI5f",
    "outputId": "dcfca8bb-602a-442d-c121-20f30d271c15"
   },
   "outputs": [],
   "source": [
    "# Start time\n",
    "start = time.time()\n",
    "local_start_time = time.asctime(time.localtime(time.time()))\n",
    "print(\n",
    "    f\"==========================================================START experiment time: {local_start_time}==========================================================\")\n",
    "\n",
    "# Torch Version\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print('torch version: ', torch.__version__)\n",
    "print('torchvision version: ', torchvision.__version__)\n",
    "\n",
    "# Seed\n",
    "seed = 1\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "np.random.seed(seed)\n",
    "\n",
    "# dataset\n",
    "datasetType = 'fashionmnist' \n",
    "# possible choice: mnist fashionmnist cifar, for Clothing1M dataset, please refer to:https://github.com/Cysu/noisy_label to request the official dataset (not open source)\n",
    "\n",
    "# TODO:\n",
    "# Settings\n",
    "Is_IID = True                                       # IID or Non-IID\n",
    "# if Non-IID, how many classes per client\n",
    "classes_per_client = 5\n",
    "K = 100                                             # Total Clients Number\n",
    "Clients_all_idxs = [i for i in range(K)]            # All Clients Indexes\n",
    "\n",
    "\n",
    "\n",
    "adding_noise = True                                 # adding noise or not\n",
    "add_noise_type = \"symmetric\"                        # Noise Type: \"symmetric\" or \"pairflip\"\n",
    "noise_rate = 0.5                                    # Noise Rate\n",
    "\n",
    "\n",
    "\n",
    "total_rounds = 100                                  # Total Global Rounds\n",
    "C = 0.05                                            # percentage of fraction for client selection in each global round\n",
    "local_epochs = 5                                    # local epochs\n",
    "lr = 0.15                                           # a relatively bigger learning rate helps to prevent overfitting, please refer to the paper: JOINT OPTIMIZATION, CVPR 2018\n",
    "momentum = 0.9                                      # momentum\n",
    "batchSize = 60                                      # local batch size\n",
    "weight_decay = 0.0001                               # l2 regularization\n",
    "\n",
    "feature_dim = 128                                   # intermediate feature dimension\n",
    "\n",
    "test_accuracies = []                                # to store the test accuracies for each round\n",
    "TotalClients = []                                   # Initialize the total clients\n",
    "\n",
    "\n",
    "cachedWeights, cachedDatasizes = {}, {}             # cached weights and cached data sizes for conducting model aggregation at the server side\n",
    "for i in range(len(Clients_all_idxs)):\n",
    "    cachedWeights[i] = 0\n",
    "    cachedDatasizes[i] = 0\n",
    "\n",
    "\n",
    "#TODO: Hyperparameters\n",
    "gamma = 0.4\n",
    "lambda_e = 0.6\n",
    "T_d_reverse = 3                                     # equals to 1/T_d\n",
    "global_t_w = 0.2 * total_rounds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YeiYdBFyG3nr"
   },
   "source": [
    "# Dataset loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 446,
     "referenced_widgets": [
      "7f41b0e329ef4643969535cc1bdb1849",
      "5a4e812effcc441da2c697df2dffb77f",
      "1e3d90f6a4c5450f9227108a8c0c0073",
      "26cb1e324a944c3a8c8bc4b947616001",
      "026d32bf649d4fd1ae920793000c316d",
      "563be6df77d54a3caa581edb00bf6610",
      "76d822fe77024b388fac02c3602e8b08",
      "0b337e477e384335bf14dc59becd23e8",
      "7e474bd6f3e24182b200dc8809846fdf",
      "c2d4377ef1534bf59935075873b5a310",
      "60a9169a2a914776a86a0c2d0f6c7fe6",
      "24b3a13d33704212b75014e951b98839",
      "0dee152ab00341a18248ce1305e234c6",
      "5676d839d95749888a61d6fa62ef9943",
      "84eafe0c3ba54cc1a5b2c3f9fbd7c0c3",
      "385f57f15c4b4556bc21a002ed020260",
      "b9b26d5a07574b01bfc87ce48fcf6ce3",
      "b59c01b3c75d4ba3a6f9c22df4dd7329",
      "f34c2c577c1b486c81a282b19e868bb0",
      "a227f20160b5431194950cb390dffde4",
      "9e84ac3a93244c1b997e5fb5ca31e4fd",
      "8ae9cbd4686e4bc6920cd0f5963bbb00",
      "57f441857d574675b8b9fa826951fb59",
      "0203dea442b540a985261aa7d4e3957e",
      "3c6797dfef9a4e98bd0c23a3ad0645d2",
      "1abae7f57e724fada16a5e890bb2e486",
      "3df4e8f938444bd58c5d9f13c7c55679",
      "cdf33b1bb538419392c2df62e8e207c6",
      "1c482bf77cdf4cc6b45dd8c6f12f96c1",
      "4a45fd88f07b47ef807a982ab88df881",
      "73ae715314a641ec891e5b7367cf8336",
      "f71addb7f12a4c4cadc94c28f4efadd4",
      "39ba2f0bd219473dac800d7b219b8204",
      "fe619bcfa5374ddca5cfd24aafcb5180",
      "16172331a24e4409b44c57a251aed5dc",
      "f9fcd89428814c4fa8ac245a131180e7",
      "608ca2f0bf0f483b85fa5ce77d18e688",
      "a46a35f97cce44a8a8803b75f90892fe",
      "91cca096431f46e3858ded1a4425350c",
      "d7edda791ce348c782b89e46dc0be5cf",
      "4f84caabba5340ab8ce078294a2ba380",
      "4309da5dde32429eaf97366ffb10f307",
      "e7613d8c26b64b8b8ed643c48c3ba301",
      "a14c04188aad4fd1af4b35b5e4838370"
     ]
    },
    "id": "O2m9MEmBG5J2",
    "outputId": "cf98f048-7835-4424-f165-0618e0d01cae"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "from torchvision.transforms.transforms import Resize\n",
    "\"\"\"# Dataset loading\"\"\"\n",
    "\n",
    "if datasetType == 'fashionmnist':\n",
    "    transforms_mnist = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,)),\n",
    "    ])\n",
    "    dataset_train = datasets.FashionMNIST(\n",
    "        './data/fashionmnist_data/', train=True, download=True, transform=transforms_mnist)\n",
    "    dataset_test = datasets.FashionMNIST(\n",
    "        './data/fashionmnist_data/', train=False, download=True, transform=transforms_mnist)\n",
    "    input_channel = 1                           # input color channel num\n",
    "\n",
    "\n",
    "elif datasetType == 'mnist':\n",
    "    transforms_mnist = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.1307,), (0.3081,))\n",
    "    ])\n",
    "    dataset_train = datasets.MNIST(\n",
    "        './data/mnist/', train=True, download=True, transform=transforms_mnist)\n",
    "    dataset_test = datasets.MNIST(\n",
    "        './data/mnist/', train=False, download=True, transform=transforms_mnist)\n",
    "    input_channel = 1\n",
    "\n",
    "\n",
    "elif datasetType == 'cifar':\n",
    "    transforms_cifar = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[\n",
    "                             0.229, 0.224, 0.225])\n",
    "    ])\n",
    "    dataset_train = datasets.CIFAR10(\n",
    "        './data/cifar/', train=True, download=True, transform=transforms_cifar)\n",
    "    dataset_test = datasets.CIFAR10(\n",
    "        './data/cifar/', train=False, download=True, transform=transforms_cifar)\n",
    "    input_channel = 3\n",
    "\n",
    "    \n",
    "else:\n",
    "    raise NotImplementedError('Error: unrecognized dataset')\n",
    "\n",
    "\n",
    "Num_classes = 10                                            # number of classes in the dataset\n",
    "labels_array = [i for i in range(Num_classes)]              #\n",
    "labels = np.array(dataset_train.targets)                    # retrieve the real labels\n",
    "true_Labels = copy.deepcopy(labels)                         # cache the real labels\n",
    "\n",
    "\n",
    "# noise_or_not = []\n",
    "\n",
    "img_size = dataset_train[0][0].shape                        # image size\n",
    "classes = np.array(list(dataset_train.class_to_idx.values()))\n",
    "\n",
    "\n",
    "local_datasize = len(labels)/K                              # local dataset size\n",
    "\n",
    "\n",
    "if datasetType == 'fashionmnist':\n",
    "    tt_transform = transforms.Compose([\n",
    "            transforms.RandomRotation(30)])\n",
    "elif datasetType == 'mnist':\n",
    "    tt_transform = transforms.Compose([\n",
    "            transforms.RandomRotation(30)])\n",
    "elif datasetType == 'cifar':\n",
    "    s = 1\n",
    "    ssize=28\n",
    "    color_jitter = torchvision.transforms.ColorJitter(0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s)\n",
    "    tt_transform = transforms.Compose([\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.RandomApply([color_jitter], p=0.8),\n",
    "        transforms.RandomGrayscale(p=0.2)])\n",
    "\n",
    "\n",
    "# retrieve the data according to distributed data indexes\n",
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, dataset, idxs):\n",
    "        self.dataset = dataset\n",
    "        self.idxs = list(idxs)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.idxs)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        image, label = self.dataset[self.idxs[item]]\n",
    "        return image, label\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"add noise\"\"\"\n",
    "def noisify_label(true_label, num_classes=10, noise_type=\"symmetric\"):\n",
    "    if noise_type == \"symmetric\":\n",
    "        label_lst = list(range(num_classes))\n",
    "        label_lst.remove(true_label)\n",
    "        return random.sample(label_lst, k=1)[0]\n",
    "\n",
    "    elif noise_type == \"pairflip\":\n",
    "        return (true_label - 1) % num_classes\n",
    "\n",
    "\n",
    "# NonIID\n",
    "classes_mapper = {}\n",
    "for item in classes:\n",
    "    # numpy to int\n",
    "    classes_mapper[int(item)] = []\n",
    "\n",
    "# TODO: Data Parititioning\n",
    "\n",
    "\n",
    "def customizedIdxsGenerator(Bias_array, local_datasize):\n",
    "    \"\"\"\n",
    "    @Params:\n",
    "    Bias_array reprensents the ground-truth label distribution of each client\n",
    "    local_datasize represents the local dataset size\n",
    "    \"\"\"\n",
    "    # the data nums for each class\n",
    "    Bias_num_array = copy.deepcopy(Bias_array)\n",
    "    for i in range(len(Bias_array)):\n",
    "        Bias_num_array[i] = int(local_datasize*Bias_array[i])\n",
    "\n",
    "    local_dataitem_idxs = np.array([])\n",
    "\n",
    "    for key in classes_mapper.keys():\n",
    "        # concatenate\n",
    "        local_dataitem_idxs = np.concatenate((local_dataitem_idxs, np.random.choice(\n",
    "            classes_mapper[key], int(Bias_num_array[key]), replace=False)), axis=0)\n",
    "\n",
    "    np.random.shuffle(local_dataitem_idxs)\n",
    "    local_dataitem_idxs = local_dataitem_idxs.astype(int)\n",
    "\n",
    "    # return the data indexes for each client in Python list Type\n",
    "    return local_dataitem_idxs.tolist()\n",
    "\n",
    "\n",
    "def classesIdxsGenerator(classes, dataset):\n",
    "    for i in range(len(dataset)):\n",
    "        # tensor to int\n",
    "        targetForOnce = int(dataset.targets[i])\n",
    "        classes_mapper[targetForOnce].append(i)\n",
    "\n",
    "    for key in classes_mapper.keys():\n",
    "        classes_mapper[key] = np.array(classes_mapper[key])\n",
    "        np.random.seed(seed)\n",
    "        np.random.shuffle(classes_mapper[key])\n",
    "    return\n",
    "\n",
    "\n",
    "classesIdxsGenerator(classes=classes, dataset=dataset_train)\n",
    "\n",
    "# Adding noise\n",
    "if adding_noise == True:\n",
    "    len_of_Dataset = len(dataset_train)\n",
    "\n",
    "    for key in classes_mapper.keys():\n",
    "        len_of_Datasize_thisClass = int(len(classes_mapper[key]) * noise_rate)\n",
    "        # inject equal number of noisy samples for each class\n",
    "        # randomly select samples and add noise\n",
    "        np.random.seed(seed)\n",
    "\n",
    "        sampleIdxs_random_choice = np.random.choice(\n",
    "            classes_mapper[key], len_of_Datasize_thisClass, replace=False)\n",
    "        for i in range(len(sampleIdxs_random_choice)):\n",
    "            changed_item_index = sampleIdxs_random_choice[i]\n",
    "            true_label = dataset_train.targets[changed_item_index]\n",
    "            dataset_train.targets[changed_item_index] = noisify_label(\n",
    "                true_label, num_classes=Num_classes, noise_type=add_noise_type)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "V5KB8QODjLvj"
   },
   "source": [
    "# Base Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "80CILrMXjMrb"
   },
   "outputs": [],
   "source": [
    "def call_bn(bn, x):\n",
    "    return bn(x)\n",
    "\n",
    "\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self, input_channels=3, n_outputs=10, dropout_rate=0.25, top_bn=False):\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.top_bn = top_bn\n",
    "        super(CNN, self).__init__()\n",
    "        self.c1 = nn.Conv2d(input_channels, 128,\n",
    "                            kernel_size=3, stride=1, padding=1)\n",
    "        self.c2 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)\n",
    "        self.c3 = nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1)\n",
    "        self.c4 = nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1)\n",
    "        self.c5 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)\n",
    "        self.c6 = nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1)\n",
    "        self.c7 = nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=0)\n",
    "        self.c8 = nn.Conv2d(512, 256, kernel_size=3, stride=1, padding=0)\n",
    "        self.c9 = nn.Conv2d(256, 128, kernel_size=3, stride=1, padding=0)\n",
    "        self.l_c1 = nn.Linear(128, n_outputs)\n",
    "        self.bn1 = nn.BatchNorm2d(128)\n",
    "        self.bn2 = nn.BatchNorm2d(128)\n",
    "        self.bn3 = nn.BatchNorm2d(128)\n",
    "        self.bn4 = nn.BatchNorm2d(256)\n",
    "        self.bn5 = nn.BatchNorm2d(256)\n",
    "        self.bn6 = nn.BatchNorm2d(256)\n",
    "        self.bn7 = nn.BatchNorm2d(512)\n",
    "        self.bn8 = nn.BatchNorm2d(256)\n",
    "        self.bn9 = nn.BatchNorm2d(128)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h = x\n",
    "        h = self.c1(h)\n",
    "        h = F.leaky_relu(self.bn1(h), negative_slope=0.01)\n",
    "        h = self.c2(h)\n",
    "        h = F.leaky_relu(self.bn2(h), negative_slope=0.01)\n",
    "        h = self.c3(h)\n",
    "        h = F.leaky_relu(self.bn3(h), negative_slope=0.01)\n",
    "        h = F.max_pool2d(h, kernel_size=2, stride=2)\n",
    "        h = F.dropout2d(h, p=self.dropout_rate)\n",
    "\n",
    "        h = self.c4(h)\n",
    "        h = F.leaky_relu(self.bn4(h), negative_slope=0.01)\n",
    "        h = self.c5(h)\n",
    "        h = F.leaky_relu(self.bn5(h), negative_slope=0.01)\n",
    "        h = self.c6(h)\n",
    "        h = F.leaky_relu(self.bn6(h), negative_slope=0.01)\n",
    "        h = F.max_pool2d(h, kernel_size=2, stride=2)\n",
    "        h = F.dropout2d(h, p=self.dropout_rate)\n",
    "\n",
    "        h = self.c7(h)\n",
    "        h = F.leaky_relu(self.bn7(h), negative_slope=0.01)\n",
    "        h = self.c8(h)\n",
    "        h = F.leaky_relu(self.bn8(h), negative_slope=0.01)\n",
    "        h = self.c9(h)\n",
    "        h = F.leaky_relu(self.bn9(h), negative_slope=0.01)\n",
    "        h = F.avg_pool2d(h, kernel_size=h.data.shape[2])\n",
    "\n",
    "        feature = h.view(h.size(0), h.size(1))\n",
    "\n",
    "        logit = self.l_c1(feature)\n",
    "        if self.top_bn:\n",
    "            logit = call_bn(self.bn_c1, logit)\n",
    "        return logit, feature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tGs7adkwjSU3"
   },
   "source": [
    "# Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "ZT2ZwCLijc3j"
   },
   "outputs": [],
   "source": [
    "\"\"\"# Client\"\"\"\n",
    "\n",
    "\n",
    "class Client(object):\n",
    "    def __init__(self, client_index, dataset, batchsize, learning_rate, local_epochs, data_idxs, bias_array):\n",
    "        self.client_index = client_index\n",
    "        self.bias_array = torch.Tensor(bias_array)\n",
    "        self.data_idxs = data_idxs\n",
    "        self.batchsize = batchsize\n",
    "        self.train_loader = DataLoader(CustomDataset(\n",
    "            dataset, data_idxs), batch_size=batchsize, shuffle=True)\n",
    "        self.learning_rate = learning_rate\n",
    "        self.momentum = momentum\n",
    "        self.epochs = local_epochs\n",
    "        self.local_datasize = len(data_idxs)\n",
    "        self.t_w = global_t_w\n",
    "\n",
    "    def printProfile(self):\n",
    "        print(\n",
    "            f\"ClientNumber: {self.client_index},learningRate: {self.learning_rate},local_epochs {self.epochs},and its bias is {self.bias_array}\")\n",
    "        print(\n",
    "            f\"Type of dataidxs is {type(self.data_idxs)} and its length is {len(self.data_idxs)}, wrong labels length is {len(self.wrong_labels_idxs)}\")\n",
    "        return\n",
    "    \n",
    "    def similarloss(self, predict, target):\n",
    "        \"\"\"\n",
    "        :param predict: model prediction for original data\n",
    "        :param target: model prediction for mild augmented data\n",
    "        :return: loss\n",
    "        \"\"\"\n",
    "        return 2-2*torch.cosine_similarity(predict, target, dim=-1)\n",
    "\n",
    "    def L1loss(self,predict,target):\n",
    "        \"\"\"\n",
    "        :param predict: model prediction for original data\n",
    "        :param target: model prediction for mildly augmented data\n",
    "        :return: loss\n",
    "        \"\"\"\n",
    "        loss_f = nn.L1Loss(reduction='mean')\n",
    "        l1_loss = loss_f(predict,target)\n",
    "        return l1_loss\n",
    "    \n",
    "    def L2loss(self,predict,target):\n",
    "        \"\"\"\n",
    "        :param predict: model prediction for original data\n",
    "        :param target: model prediction for mildly augmented data\n",
    "        :return: loss\n",
    "        \"\"\"\n",
    "        loss_f_mse = nn.MSELoss(reduction='mean')\n",
    "        loss_mse = loss_f_mse(predict,target)\n",
    "        return loss_mse\n",
    "\n",
    "    def js(self, p_output, q_output):\n",
    "        \"\"\"\n",
    "        :param predict: model prediction for original data\n",
    "        :param target: model prediction for mildly augmented data\n",
    "        :return: loss\n",
    "        \"\"\"\n",
    "        KLDivLoss = nn.KLDivLoss(reduction='mean')\n",
    "        log_mean_output = ((p_output + q_output )/2).log()\n",
    "        return (KLDivLoss(log_mean_output, p_output) + KLDivLoss(log_mean_output, q_output))/2\n",
    "\n",
    "\n",
    "    def train(self, model, current_round):\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        \n",
    "        sm = torch.nn.Softmax(dim=1)\n",
    "        lsm = torch.nn.LogSoftmax(dim=1)\n",
    "\n",
    "\n",
    "        optimizer = torch.optim.SGD(model.parameters(\n",
    "        ), lr=self.learning_rate, momentum=self.momentum, weight_decay=weight_decay)\n",
    "        e_loss = []\n",
    "        for _ in range(1, self.epochs+1):  # _ means each local epoch\n",
    "            train_loss = 0.0\n",
    "            model.train()\n",
    "            for data, labels in self.train_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    data, labels = data.cuda(), labels.cuda()\n",
    "                \n",
    "                # conduct augmentation on GPU to accelerate training\n",
    "                data_aug = tt_transform(data)\n",
    "                data_aug = data_aug.cuda()\n",
    "                \n",
    "                optimizer.zero_grad()  # clear the gradients\n",
    "                \n",
    "                with autocast():\n",
    "                    output1, f1 = model(data)  # make a forward pass\n",
    "                    output2, f2 = model(data_aug) # make a forward pass\n",
    "\n",
    "                    mix_1 = np.random.beta(1,1) # mixing predict1 and predict2\n",
    "                    # mix_1 = 1\n",
    "                    mix_2 = 1-mix_1\n",
    "\n",
    "\n",
    "\n",
    "                    # to further conduct self distillation, *3 means the temperature T_d is 1/3\n",
    "                    logits1,logits2=torch.softmax(output1*3, dim=1),torch.softmax(output2*3, dim=1)\n",
    "                    \n",
    "                    \n",
    "                    # for training stability to conduct clamping to avoid exploding gradients, which is also used in Symmetric CE, ICCV 2019\n",
    "                    logits1,logits2 = torch.clamp(logits1, min=1e-6, max=1.0), torch.clamp(logits2, min=1e-6, max=1.0) \n",
    "                    \n",
    "                    # to conduct self entropy regularization (discussed in Discussion Section of the paper)\n",
    "                    L_e = - (torch.mean(torch.sum(sm(logits1) * lsm(logits1), dim=1))+torch.mean(torch.sum(sm(logits1) * lsm(logits1), dim=1))) * 0.5\n",
    "                    \n",
    "                    # to mix up the two predictions\n",
    "                    p = torch.softmax(output1, dim=1)*mix_1 + torch.softmax(output2, dim=1)*mix_2\n",
    "\n",
    "                    # to get sharpened prediction p_s\n",
    "                    pt = p**(2)\n",
    "                    # normalize the prediction\n",
    "                    pred_mix = pt / pt.sum(dim=1, keepdim=True)\n",
    "\n",
    "                    \n",
    "                    betaa = gamma\n",
    "                    if(current_round<self.t_w):\n",
    "                        betaa = gamma * current_round/self.t_w                    \n",
    "                    \n",
    "                \n",
    "                    loss = criterion(pred_mix, labels)  # to compute cross entropy loss\n",
    "\n",
    "                    loss +=  self.js(logits1,logits2) * betaa\n",
    "\n",
    "                    loss += L_e * lambda_e\n",
    "    #                 loss +=  self.similarloss(logits1,logits2).mean() * betaa\n",
    "                \n",
    "                loss.backward()  # do a backwards pass\n",
    "                optimizer.step()  # perform a single optimization step\n",
    "                train_loss += loss.item()*data.size(0)  # update training loss\n",
    "\n",
    "            # average losses\n",
    "            train_loss = train_loss/len(self.train_loader.dataset)\n",
    "            e_loss.append(train_loss)\n",
    "\n",
    "        total_loss = sum(e_loss)/len(e_loss)\n",
    "        return model.state_dict(), total_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TestGlobalModel(model, dataset, criterion, test_batchsize, num_classes, classes):\n",
    "    # test loss\n",
    "    test_loss = 0.0\n",
    "    correct_class = list(0. for i in range(num_classes))\n",
    "    total_class = list(0. for i in range(num_classes))\n",
    "\n",
    "    test_loader = DataLoader(dataset, batch_size=test_batchsize)\n",
    "    l = len(test_loader.dataset)\n",
    "    model.eval()\n",
    "    correcttt = 0\n",
    "\n",
    "    for data, labels in test_loader:\n",
    "        if torch.cuda.is_available():\n",
    "            data, labels = data.cuda(), labels.cuda()\n",
    "\n",
    "        output, _ = model(data)\n",
    "        loss = criterion(output, labels)\n",
    "        test_loss += loss.item()*data.size(0)\n",
    "\n",
    "        _, pred = torch.max(output, 1)\n",
    "\n",
    "        correcttt += pred.eq(labels.data.view_as(pred)).float().sum().item()\n",
    "        correct_tensor = pred.eq(labels.data.view_as(pred))\n",
    "        correct = np.squeeze(correct_tensor.numpy()) if not torch.cuda.is_available(\n",
    "        ) else np.squeeze(correct_tensor.cpu().numpy())\n",
    "\n",
    "        # test accuracy for each object class\n",
    "        for i in range(num_classes):\n",
    "            label = labels.data[i]\n",
    "            correct_class[label] += correct[i].item()\n",
    "            total_class[label] += 1\n",
    "\n",
    "    # avg test loss\n",
    "    test_loss = test_loss/len(test_loader.dataset)\n",
    "    print(\"Test Loss: {:.6f}\\n\".format(test_loss))\n",
    "\n",
    "    # print test accuracy\n",
    "    for i in range(10):\n",
    "        if total_class[i] > 0:\n",
    "            print('Test Accuracy of %5s: %2d%% (%2d/%2d)' %\n",
    "                  (classes[i], 100 * correct_class[i] / total_class[i],\n",
    "                   np.sum(correct_class[i]), np.sum(total_class[i])))\n",
    "        else:\n",
    "            print('Test Accuracy of %5s: N/A (no training examples)' %\n",
    "                  (classes[i]))\n",
    "\n",
    "    print('\\nTEST ACC {:.3f} Final Test  Accuracy: {:.3f} ({}/{})'.format(100.0*correcttt/l,\n",
    "                                                                          100. *\n",
    "                                                                          np.sum(\n",
    "                                                                              correct_class) / np.sum(total_class),\n",
    "                                                                          np.sum(correct_class), np.sum(total_class)))\n",
    "\n",
    "    test_accuracies.append(100.0*correcttt/l)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vbi8cb83kAk4"
   },
   "source": [
    "# Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "bDB-IEVDkClj"
   },
   "outputs": [],
   "source": [
    "def averaging(w_group, selected_client_idxs):\n",
    "    w_avg = copy.deepcopy(w_group[selected_client_idxs[0]])\n",
    "    for k in w_avg.keys():\n",
    "        for id in selected_client_idxs[1:]:\n",
    "            w_avg[k] += w_group[id][k]\n",
    "        w_avg[k] = torch.div(w_avg[k], len(w_group))\n",
    "\n",
    "    return w_avg\n",
    "\n",
    "\n",
    "\"\"\"## Overall FedAvg\"\"\"\n",
    "\n",
    "def FedAvg(model, rounds):\n",
    "    global_weights = model.state_dict()\n",
    "    last_round_timing = time.time()\n",
    "    print(f\"Start experiment time: {last_round_timing}\")\n",
    "    for i in range(rounds):\n",
    "        print(f\"\\nGLOBAL ROUND {i}, Start Training\")\n",
    "        selected_client_num = max(int(C*len(Clients_all_idxs)), 1)\n",
    "        selected_client_idxs = np.random.choice(\n",
    "            Clients_all_idxs, selected_client_num, replace=False)\n",
    "        local_Loss = []\n",
    "        tempTotalDatasize = 0\n",
    "        for k in selected_client_idxs:\n",
    "            local_update = TotalClients[k]\n",
    "            weights, _ = local_update.train(\n",
    "                model=copy.deepcopy(model), current_round=i)  # _ means loss\n",
    "            cachedWeights[k] = copy.deepcopy(weights)\n",
    "            cachedDatasizes[k] = local_update.local_datasize\n",
    "            tempTotalDatasize += local_update.local_datasize\n",
    "            local_Loss.append(copy.deepcopy(_))\n",
    "        loss_avg = sum(local_Loss) / len(local_Loss)\n",
    "\n",
    "\n",
    "        weights_avg = copy.deepcopy(cachedWeights[selected_client_idxs[0]])\n",
    "        for key in weights_avg.keys():\n",
    "            rr = cachedDatasizes[selected_client_idxs[0]]/tempTotalDatasize\n",
    "            weights_avg[key] = torch.mul(weights_avg[key], rr)\n",
    "        for key in weights_avg.keys():\n",
    "            for k in selected_client_idxs[1:]:\n",
    "                ratio = cachedDatasizes[k]/tempTotalDatasize\n",
    "                weights_avg[key] += cachedWeights[k][key]*ratio\n",
    "\n",
    "        global_weights = weights_avg\n",
    "        print('\\n')\n",
    "        model.load_state_dict(weights_avg)\n",
    "        Criterion = nn.CrossEntropyLoss()\n",
    "        TestGlobalModel(model=model, dataset=dataset_test, criterion=Criterion,\n",
    "                        test_batchsize=128, num_classes=Num_classes, classes=classes)\n",
    "        current_round_timing = time.time()\n",
    "        print(\n",
    "            f\"Duration for this round is : {current_round_timing-last_round_timing}\")\n",
    "        last_round_timing = current_round_timing\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Z1fO9vCWkG61"
   },
   "source": [
    "# Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "g4Urv5bAkKl7"
   },
   "outputs": [],
   "source": [
    "\"\"\"# Initialization\"\"\"\n",
    "\n",
    "\n",
    "def initialization():\n",
    "    print(f\"Generating {K} clients and sampling random generation profile...\")\n",
    "    for i in range(K):\n",
    "        if Is_IID == False:\n",
    "            selected_labels = np.random.choice(\n",
    "                labels_array, classes_per_client, replace=False)\n",
    "            client_distribution = np.array(\n",
    "                [0. for i in labels_array]) \n",
    "            for each in selected_labels:\n",
    "                client_distribution[each] = random.uniform(0, 1)\n",
    "            client_distribution /= client_distribution.sum()\n",
    "            biasArrayTemp = client_distribution\n",
    "\n",
    "            # generate all indexes based on the ground truth label distribution\n",
    "            clientDatasetIdxs = customizedIdxsGenerator(\n",
    "                Bias_array=biasArrayTemp, local_datasize=local_datasize)\n",
    "            wrong_label_mapping_temp = {}\n",
    "\n",
    "            clientTemp = Client(client_index=i, dataset=dataset_train, batchsize=batchSize, learning_rate=lr,\n",
    "                                local_epochs=local_epochs, data_idxs=clientDatasetIdxs, bias_array=biasArrayTemp)\n",
    "            clientTemp.wrong_labels_idxs = wrong_label_mapping_temp\n",
    "            if(i % 11 == 0):\n",
    "                clientTemp.printProfile()\n",
    "                if(adding_noise == True):\n",
    "                    print(\n",
    "                        f\"【Noise Added, its type is {add_noise_type} and noise ration is {noise_rate}】\")\n",
    "            TotalClients.append(clientTemp)\n",
    "        else:\n",
    "            client_distribution = np.array([0.1 for i in labels_array])\n",
    "            biasArrayTemp = client_distribution\n",
    "            # generate all indexes based on the ground truth label distribution\n",
    "            clientDatasetIdxs = customizedIdxsGenerator(\n",
    "                Bias_array=biasArrayTemp, local_datasize=local_datasize)\n",
    "            wrong_label_mapping_temp = {}\n",
    "\n",
    "            clientTemp = Client(client_index=i, dataset=dataset_train, batchsize=batchSize, learning_rate=lr,\n",
    "                                local_epochs=local_epochs, data_idxs=clientDatasetIdxs, bias_array=biasArrayTemp)\n",
    "            clientTemp.wrong_labels_idxs = wrong_label_mapping_temp\n",
    "\n",
    "            if(i % 11 == 0):\n",
    "                clientTemp.printProfile()\n",
    "                if(adding_noise == True):\n",
    "                    print(\n",
    "                        f\"【Noise Added, its type is {add_noise_type} and noise ration is {noise_rate}】\")\n",
    "            TotalClients.append(clientTemp)\n",
    "\n",
    "    # noise_or_not = np.transpose(\n",
    "    #     dataset_train.targets) == np.transpose(true_Labels)\n",
    "    print(f\"Done, {K} clients are generated for initialization...\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oJe_qSs9kU8h"
   },
   "source": [
    "# Main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EvDMNl9WkgEz",
    "outputId": "9bbc7573-7ea1-4068-ad4b-ce0ce52d09fd"
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "    # load model\n",
    "    cnn_net = CNN(input_channels=input_channel)\n",
    "    if torch.cuda.is_available():\n",
    "        cnn_net.cuda()\n",
    "    initialization()\n",
    "    FedAvg(model=cnn_net, rounds=total_rounds)\n",
    "\n",
    "\n",
    "    print(\"==========================Final Results================================\")\n",
    "    print(test_accuracies)\n",
    "    print(f\"Last 10 rounds average test accuracy: {sum(test_accuracies[-10:])/10} %\")\n",
    "    # print(train_losses)\n",
    "    # print(test_accuracies)\n",
    "    txtfile = './'+'FedAvgLSR+'+'_dataset-'+datasetType+'_NoiseType_'+add_noise_type+'_NoiseRate_'+str(noise_rate)+'_IID_'+str(adding_noise)+'.txt'\n",
    "    with open(txtfile,'a') as myfile:\n",
    "        text = str(test_accuracies)\n",
    "        myfile.write(text[1:-1])\n",
    "    print(\"=======================================================================\")\n",
    "\n",
    "main()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "[fmnist]FLNLL（FedAvg+SS）noniid.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "88279d2366fe020547cde40dd65aa0e3aa662a6ec1f3ca12d88834876c85e1a6"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0203dea442b540a985261aa7d4e3957e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cdf33b1bb538419392c2df62e8e207c6",
      "placeholder": "​",
      "style": "IPY_MODEL_1c482bf77cdf4cc6b45dd8c6f12f96c1",
      "value": ""
     }
    },
    "026d32bf649d4fd1ae920793000c316d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0b337e477e384335bf14dc59becd23e8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0dee152ab00341a18248ce1305e234c6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b9b26d5a07574b01bfc87ce48fcf6ce3",
      "placeholder": "​",
      "style": "IPY_MODEL_b59c01b3c75d4ba3a6f9c22df4dd7329",
      "value": ""
     }
    },
    "16172331a24e4409b44c57a251aed5dc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_91cca096431f46e3858ded1a4425350c",
      "placeholder": "​",
      "style": "IPY_MODEL_d7edda791ce348c782b89e46dc0be5cf",
      "value": ""
     }
    },
    "1abae7f57e724fada16a5e890bb2e486": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f71addb7f12a4c4cadc94c28f4efadd4",
      "placeholder": "​",
      "style": "IPY_MODEL_39ba2f0bd219473dac800d7b219b8204",
      "value": " 4422656/? [00:02&lt;00:00, 3592525.89it/s]"
     }
    },
    "1c482bf77cdf4cc6b45dd8c6f12f96c1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "1e3d90f6a4c5450f9227108a8c0c0073": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0b337e477e384335bf14dc59becd23e8",
      "max": 26421880,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7e474bd6f3e24182b200dc8809846fdf",
      "value": 26421880
     }
    },
    "24b3a13d33704212b75014e951b98839": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0dee152ab00341a18248ce1305e234c6",
       "IPY_MODEL_5676d839d95749888a61d6fa62ef9943",
       "IPY_MODEL_84eafe0c3ba54cc1a5b2c3f9fbd7c0c3"
      ],
      "layout": "IPY_MODEL_385f57f15c4b4556bc21a002ed020260"
     }
    },
    "26cb1e324a944c3a8c8bc4b947616001": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c2d4377ef1534bf59935075873b5a310",
      "placeholder": "​",
      "style": "IPY_MODEL_60a9169a2a914776a86a0c2d0f6c7fe6",
      "value": " 26422272/? [00:04&lt;00:00, 9373079.37it/s]"
     }
    },
    "385f57f15c4b4556bc21a002ed020260": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "39ba2f0bd219473dac800d7b219b8204": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3c6797dfef9a4e98bd0c23a3ad0645d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4a45fd88f07b47ef807a982ab88df881",
      "max": 4422102,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_73ae715314a641ec891e5b7367cf8336",
      "value": 4422102
     }
    },
    "3df4e8f938444bd58c5d9f13c7c55679": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4309da5dde32429eaf97366ffb10f307": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "4a45fd88f07b47ef807a982ab88df881": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4f84caabba5340ab8ce078294a2ba380": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "563be6df77d54a3caa581edb00bf6610": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5676d839d95749888a61d6fa62ef9943": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f34c2c577c1b486c81a282b19e868bb0",
      "max": 29515,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a227f20160b5431194950cb390dffde4",
      "value": 29515
     }
    },
    "57f441857d574675b8b9fa826951fb59": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_0203dea442b540a985261aa7d4e3957e",
       "IPY_MODEL_3c6797dfef9a4e98bd0c23a3ad0645d2",
       "IPY_MODEL_1abae7f57e724fada16a5e890bb2e486"
      ],
      "layout": "IPY_MODEL_3df4e8f938444bd58c5d9f13c7c55679"
     }
    },
    "5a4e812effcc441da2c697df2dffb77f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_563be6df77d54a3caa581edb00bf6610",
      "placeholder": "​",
      "style": "IPY_MODEL_76d822fe77024b388fac02c3602e8b08",
      "value": ""
     }
    },
    "608ca2f0bf0f483b85fa5ce77d18e688": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e7613d8c26b64b8b8ed643c48c3ba301",
      "placeholder": "​",
      "style": "IPY_MODEL_a14c04188aad4fd1af4b35b5e4838370",
      "value": " 6144/? [00:00&lt;00:00, 261843.01it/s]"
     }
    },
    "60a9169a2a914776a86a0c2d0f6c7fe6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "73ae715314a641ec891e5b7367cf8336": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "76d822fe77024b388fac02c3602e8b08": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7e474bd6f3e24182b200dc8809846fdf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "7f41b0e329ef4643969535cc1bdb1849": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_5a4e812effcc441da2c697df2dffb77f",
       "IPY_MODEL_1e3d90f6a4c5450f9227108a8c0c0073",
       "IPY_MODEL_26cb1e324a944c3a8c8bc4b947616001"
      ],
      "layout": "IPY_MODEL_026d32bf649d4fd1ae920793000c316d"
     }
    },
    "84eafe0c3ba54cc1a5b2c3f9fbd7c0c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9e84ac3a93244c1b997e5fb5ca31e4fd",
      "placeholder": "​",
      "style": "IPY_MODEL_8ae9cbd4686e4bc6920cd0f5963bbb00",
      "value": " 29696/? [00:00&lt;00:00, 55998.06it/s]"
     }
    },
    "8ae9cbd4686e4bc6920cd0f5963bbb00": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "91cca096431f46e3858ded1a4425350c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9e84ac3a93244c1b997e5fb5ca31e4fd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a14c04188aad4fd1af4b35b5e4838370": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a227f20160b5431194950cb390dffde4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "a46a35f97cce44a8a8803b75f90892fe": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b59c01b3c75d4ba3a6f9c22df4dd7329": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b9b26d5a07574b01bfc87ce48fcf6ce3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c2d4377ef1534bf59935075873b5a310": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cdf33b1bb538419392c2df62e8e207c6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d7edda791ce348c782b89e46dc0be5cf": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e7613d8c26b64b8b8ed643c48c3ba301": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f34c2c577c1b486c81a282b19e868bb0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f71addb7f12a4c4cadc94c28f4efadd4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f9fcd89428814c4fa8ac245a131180e7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4f84caabba5340ab8ce078294a2ba380",
      "max": 5148,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_4309da5dde32429eaf97366ffb10f307",
      "value": 5148
     }
    },
    "fe619bcfa5374ddca5cfd24aafcb5180": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_16172331a24e4409b44c57a251aed5dc",
       "IPY_MODEL_f9fcd89428814c4fa8ac245a131180e7",
       "IPY_MODEL_608ca2f0bf0f483b85fa5ce77d18e688"
      ],
      "layout": "IPY_MODEL_a46a35f97cce44a8a8803b75f90892fe"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}