src/codebase/iPython/Cumulative_performance/AWA2-Resnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\\n\\nimport os\\nimport sys\\n\\nsys.path.append(\\n    os.path.abspath(\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/codebase\\\")\\n)\\nimport torch.backends.cudnn as cudnn\\nimport random\\nimport time\\nfrom collections import OrderedDict\\nimport pickle\\nimport numpy as np\\nimport pandas as pd\\nimport torch\\nimport torch.nn.functional as F\\nfrom torch.utils.data import DataLoader\\nfrom tqdm import tqdm\\nfrom torch.nn.functional import one_hot\\nimport sklearn.metrics as metrics\\nimport utils\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss\\nfrom Explainer.models.Gated_Logic_Net import Gated_Logic_Net\\nfrom Explainer.models.explainer import Explainer\\nfrom Explainer.models.pi import Pi\\nfrom dataset.dataset_cubs import Dataset_cub_for_explainer\\nfrom dataset.utils_dataset import get_dataset_with_image_and_attributes\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss, Selective_Distillation_Loss\\nfrom Explainer.models.concepts import Conceptizator\\nimport Plots.plot_for_paper as plot\\nimport matplotlib.pyplot as plt\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\\n\\nimport os\\nimport sys\\n\\nsys.path.append(\\n    os.path.abspath(\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/codebase\\\")\\n)\\nimport torch.backends.cudnn as cudnn\\nimport random\\nimport time\\nfrom collections import OrderedDict\\nimport pickle\\nimport numpy as np\\nimport pandas as pd\\nimport torch\\nimport torch.nn.functional as F\\nfrom torch.utils.data import DataLoader\\nfrom tqdm import tqdm\\nfrom torch.nn.functional import one_hot\\nimport sklearn.metrics as metrics\\nimport utils\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss\\nfrom Explainer.models.Gated_Logic_Net import Gated_Logic_Net\\nfrom Explainer.models.explainer import Explainer\\nfrom Explainer.models.pi import Pi\\nfrom dataset.dataset_cubs import Dataset_cub_for_explainer\\nfrom dataset.utils_dataset import get_dataset_with_image_and_attributes\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss, Selective_Distillation_Loss\\nfrom Explainer.models.concepts import Conceptizator\\nimport Plots.plot_for_paper as plot\\nimport matplotlib.pyplot as plt\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append(\n",
    "    os.path.abspath(\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/codebase\")\n",
    ")\n",
    "import torch.backends.cudnn as cudnn\n",
    "import random\n",
    "import time\n",
    "from collections import OrderedDict\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "from torch.nn.functional import one_hot\n",
    "import sklearn.metrics as metrics\n",
    "import utils\n",
    "from Explainer.loss_F import loss_fn_kd, entropy_loss\n",
    "from Explainer.models.Gated_Logic_Net import Gated_Logic_Net\n",
    "from Explainer.models.explainer import Explainer\n",
    "from Explainer.models.pi import Pi\n",
    "from dataset.dataset_cubs import Dataset_cub_for_explainer\n",
    "from dataset.utils_dataset import get_dataset_with_image_and_attributes\n",
    "from Explainer.loss_F import loss_fn_kd, entropy_loss, Selective_Distillation_Loss\n",
    "from Explainer.models.concepts import Conceptizator\n",
    "import Plots.plot_for_paper as plot\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"device = utils.get_device()\\nprint(f\\\"Device: {device}\\\")\";\n",
       "                var nbb_formatted_code = \"device = utils.get_device()\\nprint(f\\\"Device: {device}\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "device = utils.get_device()\n",
    "print(f\"Device: {device}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "lr = 0.001\n",
    "cov = 0.4\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_{lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\",\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        \"iter3\",\n",
    "        \"explainer\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr)\n",
    "print(args.cov)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print(args.arch)\n",
    "print(args.layer)\n",
    "\n",
    "print(args.checkpoint_model)\n",
    "print(args.checkpoint_residual)\n",
    "\n",
    "print(args.iter)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "cov = args.cov[0]\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{cov}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    "    f\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\"\n",
    ")\n",
    "\n",
    "print(experiment_folder)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "lr = 0.001\n",
    "cov = 0.4\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_{lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\",\n",
    "        \"iter1\",\n",
    "        \"explainer\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr)\n",
    "print(args.cov)\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    "    f\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/\"\n",
    "experiment = f\"explainer/ResNet101/{experiment_folder}\"\n",
    "iteration = \"iter1\"\n",
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "\n",
    "test_tensor_preds_1 = torch.load(\n",
    "    os.path.join(\n",
    "        root, experiment, iteration, expert_type, output, \"test_tensor_preds.pt\"\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_preds_bb_1 = torch.load(\n",
    "    os.path.join(\n",
    "        root, experiment, iteration, expert_type, output, \"test_tensor_preds_bb.pt\"\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_y_1 = torch.load(\n",
    "    os.path.join(root, experiment, iteration, expert_type, output, \"test_tensor_y.pt\")\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Test sizes >>\")\n",
    "print(test_tensor_preds_1.size())\n",
    "print(test_tensor_preds_bb_1.size())\n",
    "print(test_tensor_y_1.size())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "base_lr = 0.001\n",
    "base_cov = 0.4\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_{base_lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\",\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        \"iter2\",\n",
    "        \"explainer\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    "    f\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/\"\n",
    "experiment = f\"explainer/ResNet101/{experiment_folder}\"\n",
    "iteration = \"iter2\"\n",
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "\n",
    "test_tensor_preds_2 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_preds_bb_2 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_y_2 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Test sizes >>\")\n",
    "print(test_tensor_preds_2.size())\n",
    "print(test_tensor_preds_bb_2.size())\n",
    "print(test_tensor_y_2.size())\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "base_lr = 0.001\n",
    "base_cov = 0.4\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_{base_lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\",\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        \"iter3\",\n",
    "        \"explainer\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    "    f\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/\"\n",
    "experiment = f\"explainer/ResNet101/{experiment_folder}\"\n",
    "iteration = \"iter3\"\n",
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "\n",
    "test_tensor_preds_3 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_preds_bb_3 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_y_3 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Test sizes >>\")\n",
    "print(test_tensor_preds_3.size())\n",
    "print(test_tensor_preds_bb_3.size())\n",
    "print(test_tensor_y_3.size())\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.001\n",
      "0.4\n",
      "\n",
      "\n",
      " << Test sizes >>\n",
      "torch.Size([643, 50])\n",
      "torch.Size([643, 50])\n",
      "torch.Size([643])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.001\\nbase_cov = 0.4\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_{base_lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\\\",\\n        \\\"cov_0.4_lr_0.001\\\",\\n        \\\"iter4\\\",\\n        \\\"explainer\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n    f\\\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/\\\"\\nexperiment = f\\\"explainer/ResNet50/{experiment_folder}\\\"\\niteration = \\\"iter4\\\"\\nexpert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\n\\ntest_tensor_preds_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        \\\"cov_0.4_lr_0.001\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"test_tensor_preds.pt\\\",\\n    )\\n)\\n\\ntest_tensor_preds_bb_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        \\\"cov_0.4_lr_0.001\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"test_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\ntest_tensor_y_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        \\\"cov_0.4_lr_0.001\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"test_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Test sizes >>\\\")\\nprint(test_tensor_preds_4.size())\\nprint(test_tensor_preds_bb_4.size())\\nprint(test_tensor_y_4.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.001\\nbase_cov = 0.4\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_{base_lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\\\",\\n        \\\"cov_0.4_lr_0.001\\\",\\n        \\\"iter4\\\",\\n        \\\"explainer\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n    f\\\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/\\\"\\nexperiment = f\\\"explainer/ResNet50/{experiment_folder}\\\"\\niteration = \\\"iter4\\\"\\nexpert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\n\\ntest_tensor_preds_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        \\\"cov_0.4_lr_0.001\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"test_tensor_preds.pt\\\",\\n    )\\n)\\n\\ntest_tensor_preds_bb_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        \\\"cov_0.4_lr_0.001\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"test_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\ntest_tensor_y_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        \\\"cov_0.4_lr_0.001\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"test_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Test sizes >>\\\")\\nprint(test_tensor_preds_4.size())\\nprint(test_tensor_preds_bb_4.size())\\nprint(test_tensor_y_4.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.001\n",
    "base_cov = 0.4\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_{base_lr}_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none\",\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        \"iter4\",\n",
    "        \"explainer\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_use-concepts-as-pi-input_{use_concepts_as_pi_input}_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    "    f\"_layer_{args.layer}_explainer_init_{explainer_init if not args.explainer_init else args.explainer_init}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/\"\n",
    "experiment = f\"explainer/ResNet101/{experiment_folder}\"\n",
    "iteration = \"iter4\"\n",
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "\n",
    "test_tensor_preds_4 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_preds_bb_4 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_y_4 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        \"cov_0.4_lr_0.001\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"test_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Test sizes >>\")\n",
    "print(test_tensor_preds_4.size())\n",
    "print(test_tensor_preds_bb_4.size())\n",
    "print(test_tensor_y_4.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"expert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\npreds = torch.cat((test_tensor_preds_1, test_tensor_preds_2, test_tensor_preds_3, \\n                   test_tensor_preds_4, \\n#                    test_tensor_preds_5, test_tensor_preds_6, \\n#                    test_tensor_preds_7,  \\n#                    test_tensor_preds_8\\n                  ), dim=0)\\ngt = torch.cat((\\n    test_tensor_y_1, test_tensor_y_2, test_tensor_y_3 ,\\n    test_tensor_y_4, \\n#     test_tensor_y_5, test_tensor_y_6,\\n#                test_tensor_y_7, \\n#     test_tensor_y_8\\n               ),\\n               dim=0)\\nbb = torch.cat((\\n    test_tensor_preds_bb_1.cpu(), test_tensor_preds_bb_2.cpu(), test_tensor_preds_bb_3.cpu(),\\n    test_tensor_preds_bb_4.cpu(), \\n#     test_tensor_preds_bb_5.cpu(), test_tensor_preds_bb_6.cpu(), \\n#     test_tensor_preds_bb_7.cpu(),\\n#     test_tensor_preds_bb_8.cpu()\\n), \\n               dim=0)\";\n",
       "                var nbb_formatted_code = \"expert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\npreds = torch.cat(\\n    (\\n        test_tensor_preds_1,\\n        test_tensor_preds_2,\\n        test_tensor_preds_3,\\n        test_tensor_preds_4,\\n        #                    test_tensor_preds_5, test_tensor_preds_6,\\n        #                    test_tensor_preds_7,\\n        #                    test_tensor_preds_8\\n    ),\\n    dim=0,\\n)\\ngt = torch.cat(\\n    (\\n        test_tensor_y_1,\\n        test_tensor_y_2,\\n        test_tensor_y_3,\\n        test_tensor_y_4,\\n        #     test_tensor_y_5, test_tensor_y_6,\\n        #                test_tensor_y_7,\\n        #     test_tensor_y_8\\n    ),\\n    dim=0,\\n)\\nbb = torch.cat(\\n    (\\n        test_tensor_preds_bb_1.cpu(),\\n        test_tensor_preds_bb_2.cpu(),\\n        test_tensor_preds_bb_3.cpu(),\\n        test_tensor_preds_bb_4.cpu(),\\n        #     test_tensor_preds_bb_5.cpu(), test_tensor_preds_bb_6.cpu(),\\n        #     test_tensor_preds_bb_7.cpu(),\\n        #     test_tensor_preds_bb_8.cpu()\\n    ),\\n    dim=0,\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "preds = torch.cat((test_tensor_preds_1, test_tensor_preds_2, test_tensor_preds_3, \n",
    "                   test_tensor_preds_4, \n",
    "#                    test_tensor_preds_5, test_tensor_preds_6, \n",
    "#                    test_tensor_preds_7,  \n",
    "#                    test_tensor_preds_8\n",
    "                  ), dim=0)\n",
    "gt = torch.cat((\n",
    "    test_tensor_y_1, test_tensor_y_2, test_tensor_y_3 ,\n",
    "    test_tensor_y_4, \n",
    "#     test_tensor_y_5, test_tensor_y_6,\n",
    "#                test_tensor_y_7, \n",
    "#     test_tensor_y_8\n",
    "               ),\n",
    "               dim=0)\n",
    "bb = torch.cat((\n",
    "    test_tensor_preds_bb_1.cpu(), test_tensor_preds_bb_2.cpu(), test_tensor_preds_bb_3.cpu(),\n",
    "    test_tensor_preds_bb_4.cpu(), \n",
    "#     test_tensor_preds_bb_5.cpu(), test_tensor_preds_bb_6.cpu(), \n",
    "#     test_tensor_preds_bb_7.cpu(),\n",
    "#     test_tensor_preds_bb_8.cpu()\n",
    "), \n",
    "               dim=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.8539), torch.Size([6634]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"torch.sum(preds.argmax(dim=1) == gt) / gt.size(0), gt.size()\";\n",
       "                var nbb_formatted_code = \"torch.sum(preds.argmax(dim=1) == gt) / gt.size(0), gt.size()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(preds.argmax(dim=1) == gt) / gt.size(0), gt.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "g_tot_acc: 85.39342779620138 (%)\n",
      "bb_tot_acc: 91.82996683750378 (%)\n",
      "g_tot_f1: 85.39342779620138 (%)\n",
      "g_tot_precision: 85.39342779620138 (%)\n",
      "g_tot_recall: 85.39342779620138 (%)\n",
      "bb_tot_recall: 91.82996683750378 (%)\n",
      "total samples covered by g: 6634 (out of 7465)\n",
      "total coverage by g: 0.8886805090421969\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"preds_tot_np = preds.argmax(dim=1).numpy()\\ngt_tot_np = gt.numpy()\\nbb_tot_np = bb.argmax(dim=1).numpy()\\nbb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\\ng_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\\ng_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\nbb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\ng_tot_cov = preds.size(0) / 7465\\n\\n\\nprint(f\\\"g_tot_acc: {g_tot_acc * 100} (%)\\\")\\nprint(f\\\"bb_tot_acc: {bb_tot_acc * 100} (%)\\\")\\nprint(f\\\"g_tot_f1: {g_tot_f1 * 100} (%)\\\")\\nprint(f\\\"g_tot_precision: {g_tot_precision * 100} (%)\\\")\\nprint(f\\\"g_tot_recall: {g_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_recall: {bb_tot_recall * 100} (%)\\\")\\nprint(f\\\"total samples covered by g: {preds.size(0)} (out of {7465})\\\")\\nprint(f\\\"total coverage by g: {g_tot_cov}\\\")\";\n",
       "                var nbb_formatted_code = \"preds_tot_np = preds.argmax(dim=1).numpy()\\ngt_tot_np = gt.numpy()\\nbb_tot_np = bb.argmax(dim=1).numpy()\\nbb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\\ng_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\\ng_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\nbb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\ng_tot_cov = preds.size(0) / 7465\\n\\n\\nprint(f\\\"g_tot_acc: {g_tot_acc * 100} (%)\\\")\\nprint(f\\\"bb_tot_acc: {bb_tot_acc * 100} (%)\\\")\\nprint(f\\\"g_tot_f1: {g_tot_f1 * 100} (%)\\\")\\nprint(f\\\"g_tot_precision: {g_tot_precision * 100} (%)\\\")\\nprint(f\\\"g_tot_recall: {g_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_recall: {bb_tot_recall * 100} (%)\\\")\\nprint(f\\\"total samples covered by g: {preds.size(0)} (out of {7465})\\\")\\nprint(f\\\"total coverage by g: {g_tot_cov}\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds_tot_np = preds.argmax(dim=1).numpy()\n",
    "gt_tot_np = gt.numpy()\n",
    "bb_tot_np = bb.argmax(dim=1).numpy()\n",
    "bb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\n",
    "g_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\n",
    "g_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "g_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "g_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "bb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\"micro\")\n",
    "g_tot_cov = preds.size(0) / 7465\n",
    "\n",
    "\n",
    "print(f\"g_tot_acc: {g_tot_acc * 100} (%)\")\n",
    "print(f\"bb_tot_acc: {bb_tot_acc * 100} (%)\")\n",
    "print(f\"g_tot_f1: {g_tot_f1 * 100} (%)\")\n",
    "print(f\"g_tot_precision: {g_tot_precision * 100} (%)\")\n",
    "print(f\"g_tot_recall: {g_tot_recall * 100} (%)\")\n",
    "print(f\"bb_tot_recall: {bb_tot_recall * 100} (%)\")\n",
    "print(f\"total samples covered by g: {preds.size(0)} (out of {7465})\")\n",
    "print(f\"total coverage by g: {g_tot_cov}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " << Test sizes >>\n",
      "torch.Size([831, 50])\n",
      "torch.Size([831, 50])\n",
      "torch.Size([831])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"test_tensor_preds_residual = torch.load(\\n    os.path.join(\\n        \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\\\",\\n        \\\"test_tensor_preds_residual.pt\\\",\\n    )\\n)\\n\\ntest_tensor_preds_bb_residual = torch.load(\\n    os.path.join(\\n        \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\\\",\\n        \\\"test_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\ntest_tensor_y_residual = torch.load(\\n    os.path.join(\\n        \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\\\",\\n        \\\"test_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Test sizes >>\\\")\\nprint(test_tensor_preds_residual.size())\\nprint(test_tensor_preds_bb_residual.size())\\nprint(test_tensor_y_residual.size())\";\n",
       "                var nbb_formatted_code = \"test_tensor_preds_residual = torch.load(\\n    os.path.join(\\n        \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\\\",\\n        \\\"test_tensor_preds_residual.pt\\\",\\n    )\\n)\\n\\ntest_tensor_preds_bb_residual = torch.load(\\n    os.path.join(\\n        \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\\\",\\n        \\\"test_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\ntest_tensor_y_residual = torch.load(\\n    os.path.join(\\n        \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet50/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\\\",\\n        \\\"test_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Test sizes >>\\\")\\nprint(test_tensor_preds_residual.size())\\nprint(test_tensor_preds_bb_residual.size())\\nprint(test_tensor_y_residual.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_tensor_preds_residual = torch.load(\n",
    "    os.path.join(\n",
    "        \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\",\n",
    "        \"test_tensor_preds_residual.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_preds_bb_residual = torch.load(\n",
    "    os.path.join(\n",
    "        \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\",\n",
    "        \"test_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "test_tensor_y_residual = torch.load(\n",
    "    os.path.join(\n",
    "        \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/awa2/explainer/ResNet101/lr_0.001_epochs_500_temperature-lens_0.7_use-concepts-as-pi-input_True_input-size-pi_2048_cov_0.4_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1_layer_layer4_explainer_init_none/cov_0.4_lr_0.001/iter4/bb/residual_outputs\",\n",
    "        \"test_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Test sizes >>\")\n",
    "print(test_tensor_preds_residual.size())\n",
    "print(test_tensor_preds_bb_residual.size())\n",
    "print(test_tensor_y_residual.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7196)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 15;\n",
       "                var nbb_unformatted_code = \"torch.sum(\\n    test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual\\n) / test_tensor_preds_residual.size(0)\";\n",
       "                var nbb_formatted_code = \"torch.sum(\\n    test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual\\n) / test_tensor_preds_residual.size(0)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(\n",
    "    test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual\n",
    ") / test_tensor_preds_residual.size(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "g_tot_acc: 83.89819156061621 (%)\n",
      "bb_tot_acc: 91.69457468184862 (%)\n",
      "g_tot_f1: 83.89819156061621 (%)\n",
      "g_tot_precision: 83.89819156061621 (%)\n",
      "g_tot_recall: 83.89819156061621 (%)\n",
      "bb_tot_recall: 91.69457468184862 (%)\n",
      "total samples covered by g: 7465 (out of 7465)\n",
      "total coverage by g: 1.0\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"expert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\npreds = torch.cat((test_tensor_preds_1, test_tensor_preds_2, test_tensor_preds_3, \\n                   test_tensor_preds_4, \\n                     test_tensor_preds_residual,\\n#                    test_tensor_preds_7, test_tensor_preds_8\\n                  ), dim=0)\\ngt = torch.cat((test_tensor_y_1, test_tensor_y_2, test_tensor_y_3 ,\\n                test_tensor_y_4, \\n                test_tensor_y_residual\\n#                test_tensor_y_7, test_tensor_y_8\\n               ), dim=0)\\nbb = torch.cat((\\n    test_tensor_preds_bb_1.cpu(), test_tensor_preds_bb_2.cpu(), test_tensor_preds_bb_3.cpu(),\\n    test_tensor_preds_bb_4.cpu(),\\n    test_tensor_preds_bb_residual.cpu()\\n#     test_tensor_preds_bb_7.cpu(), test_tensor_preds_bb_8.cpu()\\n), \\n               dim=0)\\n\\npreds_tot_np = preds.argmax(dim=1).numpy()\\ngt_tot_np = gt.numpy()\\nbb_tot_np = bb.argmax(dim=1).numpy()\\nbb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\\ng_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\\ng_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\nbb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\ng_tot_cov = preds.size(0) / 7465\\n\\n\\nprint(f\\\"g_tot_acc: {g_tot_acc * 100} (%)\\\")\\nprint(f\\\"bb_tot_acc: {bb_tot_acc * 100} (%)\\\")\\nprint(f\\\"g_tot_f1: {g_tot_f1 * 100} (%)\\\")\\nprint(f\\\"g_tot_precision: {g_tot_precision * 100} (%)\\\")\\nprint(f\\\"g_tot_recall: {g_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_recall: {bb_tot_recall * 100} (%)\\\")\\nprint(f\\\"total samples covered by g: {preds.size(0)} (out of {7465})\\\")\\nprint(f\\\"total coverage by g: {g_tot_cov}\\\")\";\n",
       "                var nbb_formatted_code = \"expert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\npreds = torch.cat(\\n    (\\n        test_tensor_preds_1,\\n        test_tensor_preds_2,\\n        test_tensor_preds_3,\\n        test_tensor_preds_4,\\n        test_tensor_preds_residual,\\n        #                    test_tensor_preds_7, test_tensor_preds_8\\n    ),\\n    dim=0,\\n)\\ngt = torch.cat(\\n    (\\n        test_tensor_y_1,\\n        test_tensor_y_2,\\n        test_tensor_y_3,\\n        test_tensor_y_4,\\n        test_tensor_y_residual\\n        #                test_tensor_y_7, test_tensor_y_8\\n    ),\\n    dim=0,\\n)\\nbb = torch.cat(\\n    (\\n        test_tensor_preds_bb_1.cpu(),\\n        test_tensor_preds_bb_2.cpu(),\\n        test_tensor_preds_bb_3.cpu(),\\n        test_tensor_preds_bb_4.cpu(),\\n        test_tensor_preds_bb_residual.cpu()\\n        #     test_tensor_preds_bb_7.cpu(), test_tensor_preds_bb_8.cpu()\\n    ),\\n    dim=0,\\n)\\n\\npreds_tot_np = preds.argmax(dim=1).numpy()\\ngt_tot_np = gt.numpy()\\nbb_tot_np = bb.argmax(dim=1).numpy()\\nbb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\\ng_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\\ng_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\nbb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\ng_tot_cov = preds.size(0) / 7465\\n\\n\\nprint(f\\\"g_tot_acc: {g_tot_acc * 100} (%)\\\")\\nprint(f\\\"bb_tot_acc: {bb_tot_acc * 100} (%)\\\")\\nprint(f\\\"g_tot_f1: {g_tot_f1 * 100} (%)\\\")\\nprint(f\\\"g_tot_precision: {g_tot_precision * 100} (%)\\\")\\nprint(f\\\"g_tot_recall: {g_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_recall: {bb_tot_recall * 100} (%)\\\")\\nprint(f\\\"total samples covered by g: {preds.size(0)} (out of {7465})\\\")\\nprint(f\\\"total coverage by g: {g_tot_cov}\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "preds = torch.cat((test_tensor_preds_1, test_tensor_preds_2, test_tensor_preds_3, \n",
    "                   test_tensor_preds_4, \n",
    "                     test_tensor_preds_residual,\n",
    "#                    test_tensor_preds_7, test_tensor_preds_8\n",
    "                  ), dim=0)\n",
    "gt = torch.cat((test_tensor_y_1, test_tensor_y_2, test_tensor_y_3 ,\n",
    "                test_tensor_y_4, \n",
    "                test_tensor_y_residual\n",
    "#                test_tensor_y_7, test_tensor_y_8\n",
    "               ), dim=0)\n",
    "bb = torch.cat((\n",
    "    test_tensor_preds_bb_1.cpu(), test_tensor_preds_bb_2.cpu(), test_tensor_preds_bb_3.cpu(),\n",
    "    test_tensor_preds_bb_4.cpu(),\n",
    "    test_tensor_preds_bb_residual.cpu()\n",
    "#     test_tensor_preds_bb_7.cpu(), test_tensor_preds_bb_8.cpu()\n",
    "), \n",
    "               dim=0)\n",
    "\n",
    "preds_tot_np = preds.argmax(dim=1).numpy()\n",
    "gt_tot_np = gt.numpy()\n",
    "bb_tot_np = bb.argmax(dim=1).numpy()\n",
    "bb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\n",
    "g_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\n",
    "g_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "g_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "g_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "bb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\"micro\")\n",
    "g_tot_cov = preds.size(0) / 7465\n",
    "\n",
    "\n",
    "print(f\"g_tot_acc: {g_tot_acc * 100} (%)\")\n",
    "print(f\"bb_tot_acc: {bb_tot_acc * 100} (%)\")\n",
    "print(f\"g_tot_f1: {g_tot_f1 * 100} (%)\")\n",
    "print(f\"g_tot_precision: {g_tot_precision * 100} (%)\")\n",
    "print(f\"g_tot_recall: {g_tot_recall * 100} (%)\")\n",
    "print(f\"bb_tot_recall: {bb_tot_recall * 100} (%)\")\n",
    "print(f\"total samples covered by g: {preds.size(0)} (out of {7465})\")\n",
    "print(f\"total coverage by g: {g_tot_cov}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8689819268963117"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 48;\n",
       "                var nbb_unformatted_code = \"(0.3022103116405305+\\n 0.2321902284331555+\\n 0.1832953830238005+\\n 0.07117883688478923+\\n 0.08010716691403603)\";\n",
       "                var nbb_formatted_code = \"(\\n    0.3022103116405305\\n    + 0.2321902284331555\\n    + 0.1832953830238005\\n    + 0.07117883688478923\\n    + 0.08010716691403603\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(0.3022103116405305+\n",
    " 0.2321902284331555+\n",
    " 0.1832953830238005+\n",
    " 0.07117883688478923+\n",
    " 0.08010716691403603)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.3022103116405305,\n",
       " 0.2121902284331555,\n",
       " 0.1632953830238005,\n",
       " 0.08117883688478923,\n",
       " 0.08010716691403603)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 39;\n",
       "                var nbb_unformatted_code = \"(\\n    (\\n        torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)\\n        / test_tensor_y_1.size(0)\\n    ).item() * (test_tensor_preds_1.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_2.argmax(dim=1) == test_tensor_y_2)\\n        / test_tensor_y_2.size(0)\\n    ).item() * (test_tensor_preds_2.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_3.argmax(dim=1) == test_tensor_y_3)\\n        / test_tensor_y_3.size(0)\\n    ).item() * (test_tensor_preds_3.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_4.argmax(dim=1) == test_tensor_y_4)\\n        / test_tensor_y_4.size(0)\\n    ).item() * (test_tensor_preds_4.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual)\\n        / test_tensor_preds_residual.size(0)\\n    ).item() * (test_tensor_preds_residual.size(0) / 7465),\\n)\";\n",
       "                var nbb_formatted_code = \"(\\n    (\\n        torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)\\n        / test_tensor_y_1.size(0)\\n    ).item()\\n    * (test_tensor_preds_1.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_2.argmax(dim=1) == test_tensor_y_2)\\n        / test_tensor_y_2.size(0)\\n    ).item()\\n    * (test_tensor_preds_2.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_3.argmax(dim=1) == test_tensor_y_3)\\n        / test_tensor_y_3.size(0)\\n    ).item()\\n    * (test_tensor_preds_3.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_4.argmax(dim=1) == test_tensor_y_4)\\n        / test_tensor_y_4.size(0)\\n    ).item()\\n    * (test_tensor_preds_4.size(0) / 7465),\\n    (\\n        torch.sum(test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual)\\n        / test_tensor_preds_residual.size(0)\\n    ).item()\\n    * (test_tensor_preds_residual.size(0) / 7465),\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)\n",
    "        / test_tensor_y_1.size(0)\n",
    "    ).item()\n",
    "    * (test_tensor_preds_1.size(0) / 7465),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_2.argmax(dim=1) == test_tensor_y_2)\n",
    "        / test_tensor_y_2.size(0)\n",
    "    ).item()\n",
    "    * (test_tensor_preds_2.size(0) / 7465),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_3.argmax(dim=1) == test_tensor_y_3)\n",
    "        / test_tensor_y_3.size(0)\n",
    "    ).item()\n",
    "    * (test_tensor_preds_3.size(0) / 7465),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_4.argmax(dim=1) == test_tensor_y_4)\n",
    "        / test_tensor_y_4.size(0)\n",
    "    ).item()\n",
    "    * (test_tensor_preds_4.size(0) / 7465),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual)\n",
    "        / test_tensor_preds_residual.size(0)\n",
    "    ).item()\n",
    "    * (test_tensor_preds_residual.size(0) / 7465),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8433644771575928,\n",
       " 0.8457021117210388,\n",
       " 0.8447678685188293,\n",
       " 0.9424572587013245,\n",
       " 0.7196149230003357)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 40;\n",
       "                var nbb_unformatted_code = \"(\\n    (\\n        torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)\\n        / test_tensor_y_1.size(0)\\n    ).item()\\n    ,\\n    (\\n        torch.sum(test_tensor_preds_2.argmax(dim=1) == test_tensor_y_2)\\n        / test_tensor_y_2.size(0)\\n    ).item()\\n    ,\\n    (\\n        torch.sum(test_tensor_preds_3.argmax(dim=1) == test_tensor_y_3)\\n        / test_tensor_y_3.size(0)\\n    ).item()\\n    ,\\n    (\\n        torch.sum(test_tensor_preds_4.argmax(dim=1) == test_tensor_y_4)\\n        / test_tensor_y_4.size(0)\\n    ).item()\\n    ,\\n    (\\n        torch.sum(test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual)\\n        / test_tensor_preds_residual.size(0)\\n    ).item()\\n    \\n)\";\n",
       "                var nbb_formatted_code = \"(\\n    (\\n        torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)\\n        / test_tensor_y_1.size(0)\\n    ).item(),\\n    (\\n        torch.sum(test_tensor_preds_2.argmax(dim=1) == test_tensor_y_2)\\n        / test_tensor_y_2.size(0)\\n    ).item(),\\n    (\\n        torch.sum(test_tensor_preds_3.argmax(dim=1) == test_tensor_y_3)\\n        / test_tensor_y_3.size(0)\\n    ).item(),\\n    (\\n        torch.sum(test_tensor_preds_4.argmax(dim=1) == test_tensor_y_4)\\n        / test_tensor_y_4.size(0)\\n    ).item(),\\n    (\\n        torch.sum(test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual)\\n        / test_tensor_preds_residual.size(0)\\n    ).item(),\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)\n",
    "        / test_tensor_y_1.size(0)\n",
    "    ).item(),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_2.argmax(dim=1) == test_tensor_y_2)\n",
    "        / test_tensor_y_2.size(0)\n",
    "    ).item(),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_3.argmax(dim=1) == test_tensor_y_3)\n",
    "        / test_tensor_y_3.size(0)\n",
    "    ).item(),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_4.argmax(dim=1) == test_tensor_y_4)\n",
    "        / test_tensor_y_4.size(0)\n",
    "    ).item(),\n",
    "    (\n",
    "        torch.sum(test_tensor_preds_residual.argmax(dim=1) == test_tensor_y_residual)\n",
    "        / test_tensor_preds_residual.size(0)\n",
    "    ).item(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.3583389149363697,\n",
       " 0.2509042196918955,\n",
       " 0.19330207635632954,\n",
       " 0.08613529805760214,\n",
       " 0.11131949095780308)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 41;\n",
       "                var nbb_unformatted_code = \"(\\n    test_tensor_preds_1.size(0) / 7465,\\n    test_tensor_preds_2.size(0) / 7465,\\n    test_tensor_preds_3.size(0) / 7465,\\n    test_tensor_preds_4.size(0) / 7465,\\n    test_tensor_preds_residual.size(0) / 7465\\n)\";\n",
       "                var nbb_formatted_code = \"(\\n    test_tensor_preds_1.size(0) / 7465,\\n    test_tensor_preds_2.size(0) / 7465,\\n    test_tensor_preds_3.size(0) / 7465,\\n    test_tensor_preds_4.size(0) / 7465,\\n    test_tensor_preds_residual.size(0) / 7465,\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(\n",
    "    test_tensor_preds_1.size(0) / 7465,\n",
    "    test_tensor_preds_2.size(0) / 7465,\n",
    "    test_tensor_preds_3.size(0) / 7465,\n",
    "    test_tensor_preds_4.size(0) / 7465,\n",
    "    test_tensor_preds_residual.size(0) / 7465,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08010716691403603"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 42;\n",
       "                var nbb_unformatted_code = \"0.7196149230003357 * 0.11131949095780308\";\n",
       "                var nbb_formatted_code = \"0.7196149230003357 * 0.11131949095780308\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "0.7196149230003357 * 0.11131949095780308"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 4)\n",
      "(50, 4)\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 32;\n",
       "                var nbb_unformatted_code = \"expert_1_gt, expert_1_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_1, pred_tensor=test_tensor_preds_1, n_classes=50\\n)\\n\\nexpert_2_gt, expert_2_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_2, pred_tensor=test_tensor_preds_2, n_classes=50\\n)\\nexpert_3_gt, expert_3_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_3, pred_tensor=test_tensor_preds_3, n_classes=50\\n)\\nexpert_4_gt, expert_4_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_4, pred_tensor=test_tensor_preds_4, n_classes=50\\n)\\n# expert_5_gt, expert_5_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_5, pred_tensor=test_tensor_preds_5, n_classes=50\\n# )\\n# expert_6_gt, expert_6_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_6, pred_tensor=test_tensor_preds_6, n_classes=50\\n# )\\n# expert_7_gt, expert_7_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_7, pred_tensor=test_tensor_preds_7, n_classes=200\\n# )\\n# expert_8_gt, expert_8_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_8, pred_tensor=test_tensor_preds_8, n_classes=200\\n# )\\n\\n\\nexpert_1_gt =np.array([expert_1_gt]).T\\nexpert_2_gt =np.array([expert_2_gt]).T\\nexpert_3_gt =np.array([expert_3_gt]).T\\nexpert_4_gt =np.array([expert_4_gt]).T\\n# expert_5_gt =np.array([expert_5_gt]).T\\n# expert_6_gt =np.array([expert_6_gt]).T\\n# expert_7_gt =np.array([expert_7_gt]).T\\n# expert_8_gt =np.array([expert_8_gt]).T\\n\\nexpert_gt = np.concatenate(\\n    (\\n        expert_1_gt, \\n        expert_2_gt,\\n        expert_3_gt,\\n        expert_4_gt,\\n#         expert_5_gt,\\n#         expert_6_gt,\\n#         expert_7_gt,\\n#         expert_8_gt\\n    ), axis=1)\\n\\nexpert_1_correct_pred =np.array([expert_1_correct_pred]).T\\nexpert_2_correct_pred =np.array([expert_2_correct_pred]).T\\nexpert_3_correct_pred =np.array([expert_3_correct_pred]).T\\nexpert_4_correct_pred =np.array([expert_4_correct_pred]).T\\n# expert_5_correct_pred =np.array([expert_5_correct_pred]).T\\n# expert_6_correct_pred =np.array([expert_6_correct_pred]).T\\n# expert_7_correct_pred =np.array([expert_7_correct_pred]).T\\n# expert_8_correct_pred =np.array([expert_8_correct_pred]).T\\n\\nexpert_correct_preds = np.concatenate(\\n    (\\n        expert_1_correct_pred, \\n        expert_2_correct_pred,\\n        expert_3_correct_pred,\\n        expert_4_correct_pred,\\n#         expert_5_correct_pred,\\n#         expert_6_correct_pred,\\n#         expert_7_correct_pred,\\n#         expert_8_correct_pred\\n    ), axis=1)\\nprint(expert_gt.shape)\\nprint(expert_correct_preds.shape)\\n\\n\\n\\n\\n\\n\\n# print(exper_gt.shape)\\n# print(expert_correct_preds.shape)\";\n",
       "                var nbb_formatted_code = \"expert_1_gt, expert_1_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_1, pred_tensor=test_tensor_preds_1, n_classes=50\\n)\\n\\nexpert_2_gt, expert_2_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_2, pred_tensor=test_tensor_preds_2, n_classes=50\\n)\\nexpert_3_gt, expert_3_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_3, pred_tensor=test_tensor_preds_3, n_classes=50\\n)\\nexpert_4_gt, expert_4_correct_pred = plot.create_data_for_expert_label_plot(\\n    GT_tensor=test_tensor_y_4, pred_tensor=test_tensor_preds_4, n_classes=50\\n)\\n# expert_5_gt, expert_5_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_5, pred_tensor=test_tensor_preds_5, n_classes=50\\n# )\\n# expert_6_gt, expert_6_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_6, pred_tensor=test_tensor_preds_6, n_classes=50\\n# )\\n# expert_7_gt, expert_7_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_7, pred_tensor=test_tensor_preds_7, n_classes=200\\n# )\\n# expert_8_gt, expert_8_correct_pred = plot.create_data_for_expert_label_plot(\\n#     GT_tensor=test_tensor_y_8, pred_tensor=test_tensor_preds_8, n_classes=200\\n# )\\n\\n\\nexpert_1_gt = np.array([expert_1_gt]).T\\nexpert_2_gt = np.array([expert_2_gt]).T\\nexpert_3_gt = np.array([expert_3_gt]).T\\nexpert_4_gt = np.array([expert_4_gt]).T\\n# expert_5_gt =np.array([expert_5_gt]).T\\n# expert_6_gt =np.array([expert_6_gt]).T\\n# expert_7_gt =np.array([expert_7_gt]).T\\n# expert_8_gt =np.array([expert_8_gt]).T\\n\\nexpert_gt = np.concatenate(\\n    (\\n        expert_1_gt,\\n        expert_2_gt,\\n        expert_3_gt,\\n        expert_4_gt,\\n        #         expert_5_gt,\\n        #         expert_6_gt,\\n        #         expert_7_gt,\\n        #         expert_8_gt\\n    ),\\n    axis=1,\\n)\\n\\nexpert_1_correct_pred = np.array([expert_1_correct_pred]).T\\nexpert_2_correct_pred = np.array([expert_2_correct_pred]).T\\nexpert_3_correct_pred = np.array([expert_3_correct_pred]).T\\nexpert_4_correct_pred = np.array([expert_4_correct_pred]).T\\n# expert_5_correct_pred =np.array([expert_5_correct_pred]).T\\n# expert_6_correct_pred =np.array([expert_6_correct_pred]).T\\n# expert_7_correct_pred =np.array([expert_7_correct_pred]).T\\n# expert_8_correct_pred =np.array([expert_8_correct_pred]).T\\n\\nexpert_correct_preds = np.concatenate(\\n    (\\n        expert_1_correct_pred,\\n        expert_2_correct_pred,\\n        expert_3_correct_pred,\\n        expert_4_correct_pred,\\n        #         expert_5_correct_pred,\\n        #         expert_6_correct_pred,\\n        #         expert_7_correct_pred,\\n        #         expert_8_correct_pred\\n    ),\\n    axis=1,\\n)\\nprint(expert_gt.shape)\\nprint(expert_correct_preds.shape)\\n\\n\\n# print(exper_gt.shape)\\n# print(expert_correct_preds.shape)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "expert_1_gt, expert_1_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "    GT_tensor=test_tensor_y_1, pred_tensor=test_tensor_preds_1, n_classes=50\n",
    ")\n",
    "\n",
    "expert_2_gt, expert_2_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "    GT_tensor=test_tensor_y_2, pred_tensor=test_tensor_preds_2, n_classes=50\n",
    ")\n",
    "expert_3_gt, expert_3_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "    GT_tensor=test_tensor_y_3, pred_tensor=test_tensor_preds_3, n_classes=50\n",
    ")\n",
    "expert_4_gt, expert_4_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "    GT_tensor=test_tensor_y_4, pred_tensor=test_tensor_preds_4, n_classes=50\n",
    ")\n",
    "# expert_5_gt, expert_5_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "#     GT_tensor=test_tensor_y_5, pred_tensor=test_tensor_preds_5, n_classes=50\n",
    "# )\n",
    "# expert_6_gt, expert_6_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "#     GT_tensor=test_tensor_y_6, pred_tensor=test_tensor_preds_6, n_classes=50\n",
    "# )\n",
    "# expert_7_gt, expert_7_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "#     GT_tensor=test_tensor_y_7, pred_tensor=test_tensor_preds_7, n_classes=200\n",
    "# )\n",
    "# expert_8_gt, expert_8_correct_pred = plot.create_data_for_expert_label_plot(\n",
    "#     GT_tensor=test_tensor_y_8, pred_tensor=test_tensor_preds_8, n_classes=200\n",
    "# )\n",
    "\n",
    "\n",
    "expert_1_gt =np.array([expert_1_gt]).T\n",
    "expert_2_gt =np.array([expert_2_gt]).T\n",
    "expert_3_gt =np.array([expert_3_gt]).T\n",
    "expert_4_gt =np.array([expert_4_gt]).T\n",
    "# expert_5_gt =np.array([expert_5_gt]).T\n",
    "# expert_6_gt =np.array([expert_6_gt]).T\n",
    "# expert_7_gt =np.array([expert_7_gt]).T\n",
    "# expert_8_gt =np.array([expert_8_gt]).T\n",
    "\n",
    "expert_gt = np.concatenate(\n",
    "    (\n",
    "        expert_1_gt, \n",
    "        expert_2_gt,\n",
    "        expert_3_gt,\n",
    "        expert_4_gt,\n",
    "#         expert_5_gt,\n",
    "#         expert_6_gt,\n",
    "#         expert_7_gt,\n",
    "#         expert_8_gt\n",
    "    ), axis=1)\n",
    "\n",
    "expert_1_correct_pred =np.array([expert_1_correct_pred]).T\n",
    "expert_2_correct_pred =np.array([expert_2_correct_pred]).T\n",
    "expert_3_correct_pred =np.array([expert_3_correct_pred]).T\n",
    "expert_4_correct_pred =np.array([expert_4_correct_pred]).T\n",
    "# expert_5_correct_pred =np.array([expert_5_correct_pred]).T\n",
    "# expert_6_correct_pred =np.array([expert_6_correct_pred]).T\n",
    "# expert_7_correct_pred =np.array([expert_7_correct_pred]).T\n",
    "# expert_8_correct_pred =np.array([expert_8_correct_pred]).T\n",
    "\n",
    "expert_correct_preds = np.concatenate(\n",
    "    (\n",
    "        expert_1_correct_pred, \n",
    "        expert_2_correct_pred,\n",
    "        expert_3_correct_pred,\n",
    "        expert_4_correct_pred,\n",
    "#         expert_5_correct_pred,\n",
    "#         expert_6_correct_pred,\n",
    "#         expert_7_correct_pred,\n",
    "#         expert_8_correct_pred\n",
    "    ), axis=1)\n",
    "print(expert_gt.shape)\n",
    "print(expert_correct_preds.shape)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# print(exper_gt.shape)\n",
    "# print(expert_correct_preds.shape)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[37  8 46  2 17 49 36 22 15  4 20 48 11  3 35  0 30 23 41  6 39 34 29 43\n",
      " 32  9 33  5 40 21 25 28 26  1 10 45 16 42  7 31 14 13 44 47 24 27 19 18\n",
      " 38 12]\n",
      "50\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0],\n",
       "       [ 35,   0,   0,   0],\n",
       "       [ 43,   0,   0,   0],\n",
       "       [ 58,   0,   0,   0],\n",
       "       [142,   0,   0,   0],\n",
       "       [189,   0,   0,   0],\n",
       "       [158,  10,   0,   0],\n",
       "       [252,  10,   0,   0],\n",
       "       [101,  37,   0,   0],\n",
       "       [  2, 104,   0,   0],\n",
       "       [ 35, 107,   0,   0],\n",
       "       [ 43, 223,   0,   0],\n",
       "       [ 13,   0,   1,   0],\n",
       "       [ 16,   0,   1,   0],\n",
       "       [ 78,   0,   1,   0],\n",
       "       [  5, 202,   1,   0],\n",
       "       [  0, 239,   1,   0],\n",
       "       [194,   0,   2,   0],\n",
       "       [ 88,  11,   3,   0],\n",
       "       [286,  40,   3,   0],\n",
       "       [  4, 262,   3,   0],\n",
       "       [  3,   3,  11,   0],\n",
       "       [ 62,   0,  12,   0],\n",
       "       [  3,   1,  22,   0],\n",
       "       [  8,  71,  31,   0],\n",
       "       [ 14,  18,  37,   0],\n",
       "       [  5,   1,  40,   0],\n",
       "       [ 17,  17, 112,   0],\n",
       "       [  2,   7, 113,   0],\n",
       "       [  1,   3, 118,   0],\n",
       "       [  2,  15, 126,   0],\n",
       "       [  1,   2, 209,   0],\n",
       "       [  2,   0, 234,   0],\n",
       "       [150,   1,   1,   1],\n",
       "       [  4,   0,   3,   1],\n",
       "       [ 21, 166,  11,   1],\n",
       "       [ 31,   2,  17,   1],\n",
       "       [  4, 174,  23,   1],\n",
       "       [ 49, 107,  46,   1],\n",
       "       [  8,   7,  93,   1],\n",
       "       [  0,   0, 143,   1],\n",
       "       [128,   0,   0,   2],\n",
       "       [145,   0,   1,   3],\n",
       "       [  5,   0,  15,   3],\n",
       "       [135,   4,   1,   4],\n",
       "       [ 12,   9,   0,  57],\n",
       "       [ 65,   0,   1, 107],\n",
       "       [ 50,  20,   1, 128],\n",
       "       [  6,   0,   0, 164],\n",
       "       [  0,   0,   6, 167]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 33;\n",
       "                var nbb_unformatted_code = \"order = np.lexsort(\\n    [\\n        expert_gt[:, 0],\\n        expert_gt[:, 1],\\n        expert_gt[:, 2],\\n        expert_gt[:, 3],\\n#         expert_gt[:, 4],\\n#         expert_gt[:, 5],\\n#         expert_gt[:, 6],\\n#         expert_gt[:, 7],\\n    ]\\n)\\nprint(order)\\nprint(len(order))\\nsorted_cols = []\\nfor idx in order:\\n    sorted_cols.append(args.labels[idx])\\n# print(args.labels)\\n# print(sorted_cols)\\nexpert_gt[order]\";\n",
       "                var nbb_formatted_code = \"order = np.lexsort(\\n    [\\n        expert_gt[:, 0],\\n        expert_gt[:, 1],\\n        expert_gt[:, 2],\\n        expert_gt[:, 3],\\n        #         expert_gt[:, 4],\\n        #         expert_gt[:, 5],\\n        #         expert_gt[:, 6],\\n        #         expert_gt[:, 7],\\n    ]\\n)\\nprint(order)\\nprint(len(order))\\nsorted_cols = []\\nfor idx in order:\\n    sorted_cols.append(args.labels[idx])\\n# print(args.labels)\\n# print(sorted_cols)\\nexpert_gt[order]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "order = np.lexsort(\n",
    "    [\n",
    "        expert_gt[:, 0],\n",
    "        expert_gt[:, 1],\n",
    "        expert_gt[:, 2],\n",
    "        expert_gt[:, 3],\n",
    "#         expert_gt[:, 4],\n",
    "#         expert_gt[:, 5],\n",
    "#         expert_gt[:, 6],\n",
    "#         expert_gt[:, 7],\n",
    "    ]\n",
    ")\n",
    "print(order)\n",
    "print(len(order))\n",
    "sorted_cols = []\n",
    "for idx in order:\n",
    "    sorted_cols.append(args.labels[idx])\n",
    "# print(args.labels)\n",
    "# print(sorted_cols)\n",
    "expert_gt[order]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvgAAAK5CAYAAADKJloqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAACMm0lEQVR4nOzdeZwcVbn/8c83i2SDRNYrKgQRGiFAIAkaZAmIXBcui6CIXCGAjoiKqNHrBUVQERFEZE/LhaCgV9kU4SeLYQmELUP2AA0KUa6ggEAgkGAyeX5/1Glpxp6ZTjIz1cv3zatfU33qVNVTXXReT586dY4iAjMzMzMzaw4D8g7AzMzMzMx6jxN8MzMzM7Mm4gTfzMzMzKyJOME3MzMzM2siTvDNzMzMzJrIoLwDsD7hoZHMzMystynvAPbXfrnnONfHDbl/Dj1xgt+klnesyjuEljVkYHZjzNcgX74O+fM1yF/5GphZa/E338zMzMysibgFPweSFgPjI+K5vGMxMzMzaxQD3DZdE39KdUqSf3yZmZmZ2Wpzgt9LJB0raW56PSHpdkn7SrpX0mxJV0kaUbHJVyU9kF7vTPuYJulsSbcDZ0jaRdI9kuakv4V8zs7MzMzMGoUT/F4SERdHxFhgAvB/wKXAN4B9ImJnoB34csUmL0XELsD5wDkV5Vunbb4CPALsERE7AScD3+vr8zAzMzOrV5JyfzUCJ/i978fAbcALwLbATElzgSOBzSvq/aLi78SK8qsioiMtjwSukrQQ+BGwXVcHldQmqV1Se7FY7JUTMTMzM7PG437evUjSZLIk/vPAh4FbI+KwLqpHF8uvVCx/B7g9Ig6SNBq4o6tjR0QRKGf24WHpzMzMrNn4Idva+FPqJZLGAVOA/4yIVcB9wHsr+tcPk7R1xSaHVvy9t4vdjgT+kpYn93rQZmZmZtZ03ILfez4PrA/cnvpntZMl5b+QtE6q8w3g0bS8jqT7yX5kddXK/wPgcklfJuv2Y2ZmZmbWLUXkPuOv9T530cmRZ++sD74O+fM1yJ9nsrVelvsTph8d+JHcE9erOq7N/XPoib/5ZmZmZmZNxF10mpRbbfLna1AffB3y52tgZr1FbpuuiT8lMzMzM7Mm4hb8JvXnv7/ScyXrE5ttMBxwv+O8uf93/nwN8ue7J2atqam/+ZJGp0miqq27Q9L4fohhkqQbVnObaZIO6auYzMzMzBrRACn3VyNo6gTfzMzMzKzVtEKCP0jS5ZLmS7pa0rDOFSQtrVg+RNK0tLyRpGskzUqv91bZ9v9J2iEtz5F0clr+jqRPpWoj0rEfkXSl0kD5kk5O+10oqVgu77T/cZLulPSgpJslvaU3PhQzMzOzRiMG5P5qBI0R5dopAMWI2AF4CThuNbb9MfCjiJgAHAxcUqXODGB3SesBK4Hyj4DdgLvS8k7ACcC2wDsq6pwfERMiYgwwFNivcseSBgPnAYdExDjgUuC01YjfzMzMzFpMKyT4T0bEzLR8BVniXat9gPMlzQWuB9aTtG6nOncBe6T93kjWWj8MGB0RpVTngYj4v4hYBcwFRqfyvSTdL2kBsDewXad9F4AxwK0phm8Ab6sWqKQ2Se2S2ovF4mqcopmZmZk1k1YYRafzjGfVZkCrLBtSsTwAmBgRy7rZ/yxgPPA4cCuwIfBp4MGKOq9VLHeQdRsaAlwIjI+IJyWd0unYkM0YtygiJnZz/OwEIopAObMPj6JjZmZmzaZRHnLNWyu04G8mqZwgHwbcXaXO3yS9S9IA4KCK8luAz5ffSBrbecOI+AfwJPAx4D6yFv0pvN49pyvlZP45SSOAaqPmlICNyvFLGiypcyu/mZmZmdk/tUKC/zBwpKT5wPrARVXqfB24AbgNeLqi/HhgfHpA9yHg2C6OcRfwt4h4NS2/jR4S/Ih4EfgJsAD4NdmdgM51/kGW+J8haR5Z955du9uvmZmZWbMaUAf/NQJFVOuxYg3OXXRy5Imu6oMnWcqfr0H+PNGV9bLc+8d8cvAnck9cf7bi57l/Dj3xN9/MzMzMrIm0wkO2Lancimz5cctZffB1yJ+vgZn1lipTBlkV/lfXzMzMzKyJuAW/ST3215fyDqFlbfVv6wHud5w39//On69B/nz3xKw1OcHvR5LuAKZERHvesZiZmZk1mkYZxSZv/pTqgKSBecdgZmZmZs3BCf4akPQ1Scen5R9Jui0tv0/SFZIuktQuaZGkU7vYx1JJ35Z0PzBR0mJJG6Z141NrP5L2lDQ3veZIWrd/ztLMzMysvgyQcn81Aif4a2YGsHtaHg+MkDQY2I1sgquTImI8sAOwp6QdquxjOLAwIt4dEdVm1y2bAnwuIsamYy7rpXMwMzMzsybkBH/NPAiMS63prwH3kiX6u5Ml+B+TNBuYA2wHbFtlHx3ANTUcayZwdrpjMCoiVlarJKkt3TVoLxaLq31CZmZmZtYc/JDtGoiIFZIWA0cB9wDzgb2ALcla2KcAEyLiBUnTgCFVdrM8Ijoq3q/k9R9c/6wfEd+XdCPwIeA+SftExCNVYioC5cw+PIqOmZmZNRu5bbom/pTW3AyyRH4GWav9scBcYD3gFWCJpE2AD9a4v8XAuLR8cLlQ0pYRsSAizgDagW16I3gzMzMza05uwV9zdwEnAfdGxCuSlgN3RcQ8SXOARcDjZF1sanEq8D+STgTuryg/QdJeZF16HgJ+12tnYGZmZtZABsht07Vwgr+GImI6MLji/dYVy5O72GZSxfKITuvuArauss0X1j5aMzMzM2sV/hlkZmZmZtZE3ILfpLb6t/XyDqHleYr4+uDrkD9fAzPrLaIxxqHPm//VNTMzMzNrIm7Bb1IPPv73vENoWePesQEAyztW5RxJayu3Gvs65MfXIH++e2LNxg/Z1qbpPiVJoyUtrFI+XtK5aXmypPPT8imSpvRhPJMk3bCa20yTdEhfxWRmZmZmzatlWvAjop1sHPm1ImlQV7PJmpmZmZnlrela8CtJeoekOZIm1NKSLmlLSTdJelDSXZK2SeXTJJ0t6XbgjE7b/D9JO6TlOZJOTsvfkfSpVG2EpKslPSLpSklKdU6WNEvSQknFcnmn/Y+TdGeK6WZJb1n7T8bMzMys8QxAub8aQdMm+JIKwDXAURExq8bNisAXImIc2Sy1F1as2xrYJyK+0mmbGcDuktYDVgLvTeW7kU2GBbATcAKwLfCOijrnR8SEiBgDDAX263QOg4HzgENSTJcCp9V4LmZmZmbWgpq1i85GwG+AgyNiUS0bSBoB7ApcVdGQvk5FlasioqPKpncBxwNPADcC75c0DBgdEaXU4v5ARPxfOs5cYDRwN7CXpK8Bw4D1yWa//W3FvgvAGODWFNNA4Oku4m8D2gCmTp3KuH0OruW0zczMzBqGmrdtulc1a4K/BHiSrKW8pgSf7G7GixExtov1r3RRPgsYDzwO3ApsCHwaeLCizmsVyx3AIElDyO4QjI+IJyWdAgzptG8BiyJiYk/BR0SR7A4EQHgUHTMzM7PW1Kw/g/4BHAgcIekTtWwQES8BT0j6KIAyO9aw3T/Ifkx8DLiPrEV/Cq93z+lKOZl/Lt09qDZqTgnYSNLEFNNgSdvVcj5mZmZm1pqatQWfiHhF0n5k3VteIWvV78nhwEWSvgEMBv4XmFfDdncB74uIVyXdBbyNHhL8iHhR0k+ABcBisjsBnev8Iw2Xea6kkWTX6xxqvythZmZm1jQG/Ot4JFaFIiLvGKz3uYtOjjzRVX3wJEv58zXInye6sl6We3b9hWGfzT1xPe/Vi3L/HHrStC34ZmZmZtZcBjRt7/Le5QS/SZVbkS0/bjmrD74O+fM1MDPrX/5X18zMzMysibgFv0n98OJ78w6hZX3l2GxUU/c7zpf7f+fP1yB/vntizUZ+yLYm/fLNlzRa0sL+OFbFMU+RNKUX9jMtjWTTr/uQtHRtjmlmZmZmrck/7c3MzMzMmkh/JvgDJf1E0iJJt0gaCiDpDknj0/KGkhan5cmSfi3pt5KekPR5SV+WNEfSfZLWr9j+HEn3SFooaZeKY+4o6TZJj0n6dKo/QtJ0SbMlLZB0QLmypCMkzZc0T9LPOp+ApO+k1vgBFWUbS3owLe8oKSRtlt7/UdKwVHWPFOPj5db87mLpdNyvSpqVYjt1DT9/MzMzs4Y2oA7+awT9GeVWwAURsR3wInBwDduMAT4B7AKcBrwaETsB9wJHVNQbHhG7AscBl1aU7wB8GJgInCxpU2A5cFBE7AzsBfwwzVq7HXASsHdE7Ah8sTIQST8ANgaOioh/diiNiGeAIZLWA3YH2oHdJW0OPBMRr6aqbwF2A/YDvp/KqsbS6bj7ps9uF2AsME7SHjV8dmZmZmbWgvrzIdsnImJuWn4QGF3DNrdHxMvAy5KWAL9N5QvIkveyXwBExAxJ60kalcp/ExHLgGWSbidLkm8EvpeS5FXAW4FNgL2BqyPiubSv5yv2/03g/oho6yLOe4D3AnsA3wM+QDYZROVstr9OPwwekrRJKlMXsfy1Yrt902tOej+CLOGfURmApDagDWDq1KnA9l2EamZmZtaYPJNtbfozwX+tYrkDGJqWV/L6nYQh3WyzquL9Kt4Ye+dZzaKb8sOBjYBxEbEidQkaQpZsdzU72iyylvP1OyX+ZXeRtd5vDvwG+K+0rxu6OJfy/51dxUKnuqdHxNQuYstOLKIIFMtvPYqOmZmZWWuqh45Ei4FxaXlNR6s5FEDSbsCSiFiSyg+QNETSBsAkskR9JFnXmRWS9iJLygGmAx9LdSn38U9uIutWc6Okdascfwbwn8BjqZX+eeBDwMwe4u4qlko3A0dLGpHiequkjXvYr5mZmZm1qHoYB/8s4FeSPgnctob7eEHSPcB6wNEV5Q+QdcnZDPhORDwl6Urgt5LagbnAIwARsUjSacCdkjrIusRMLu8oIq5Kyf31kj6Uuv6U1y1OXefL3WbuBt4WES/0EHfVWCpFxC2S3gXcm46xlOzHxDM9fipmZmZmTUR10TZd/xTRVa+UxiDpDmBKRLTnHUsdcRedHHmiq/rgSZby52uQP090Zb0s9w7wXx3xpdwT1zOX/ij3z6En9dCCb2ZmZmbWIz9kW5uGT/AjYlLeMdSjciuy5cctZ/XB1yF/vgZmZv3L/+qamZmZmTWRhm/Bt+rOOvLqvENoWVMuzwaDcr/jfLn/d/58DfLnuyfWbPyQbW38Ka0BSadImtLN+mmSuh3ys6s6kjaV5OzczMzMzNaIW/DrTEQ8xZrPB2BmZmbWtAbIbdO18KdUI0knSSpJ+j1QSGVjJd0nab6k6yS9ucp2iyWdIemB9Hpnxeo9JN0j6fFya76k0ZIWpuXJkq6VdJOkxyT9oD/O1czMzMwalxP8GkgaB3wc2An4CDAhrfop8F8RsQOwAPhWF7t4KSJ2Ac4HzqkofwuwG7Af2Uy51Ywlm6l3e+BQSW9f4xMxMzMzs6bnBL82uwPXRcSrEfEScD0wHBgVEXemOpcDe3Sx/S8q/laOX/nriFgVEQ8Bm3Sx7fSIWBIRy4GHgM2rVZLUJqldUnuxWKz9zMzMzMwaxACU+6sRuA9+7dZm5rToYvm1iuWu/o+prNNBF9csIopAObOPs2b6OV0zMzOzVuQW/NrMAA6SNFTSusB/AK8AL0jaPdX5JHBnF9sfWvH33j6N1MzMzKxJSQNyfzUCt+DXICJmS/olMBf4E3BXWnUkcLGkYcDjwFFd7GIdSfeT/aA6rI/DNTMzM7MW5gS/RhFxGnBalVXvqVJ3cqeiCyLi1O7qRMSI9HcxMCYtTwOmVdTZb3XjNjMzM7PW4gTfzMzMzBpCozzkmjcn+H0sIkbncdwpl3uurLx5ivj64OuQP18DM7P+5QTfzMzMzBpCI81kWygUTgI+CmwFvABcC5xYKpWWpvWTgNs7bTavVCqNrdjHCLJ5lD4CrCDruv21UqnU0d2xneA3qR+edlveIbSsr5y0NwAvvbYy50ha23rrZP+8Le9YlXMkravccu9rkB/fPTHL1a7AD4AHyeY7KgLr8q+DsrylYnlFp3UXAOOBfYARwBXAEuDb3R3YCf5qkjQauCEixtRYfx3gRmBD4HTgKeBisgs4MSKWdbHd0vKDt2ZmZmbWWEql0ocr3xYKhW8CU6vU+2u17QuFwpuBw4H3l0qlB1LZN4DTC4XCd0ulUpetJ07w+95OwOCIGAsg6WLgrIi4LNeozMzMzBqM6uAh20KhMAoYVWXVi6VS6cVuNt0Q+Jf1hUJhMdlQ6veQdb/5c1o1jmyC1BkV1acDGwNbAH/s6kC+d7dmBkm6XNJ8SVdLGiZpsaQNASSNl3SHpI3JbqWMlTRX0meAjwEnS7pS0ghJ0yXNlrRA0gGdD6TMmZIWpjqHdq5jZmZmZv3mBOCJKq8TutqgUCiMBKYAl1YUPw0cAxxANrfSxsDthUJhaFq/MfB8p/72z1as65Jb8NdMATgmImZKuhQ4rlqliHhG0qeAKeUx7CVNJOvic7WkQcBBEfFS+nFwn6TrIyIqdvMRYCywI9kvv1mSZkTE0313emZmZmbWhXOomKeowovVKhcKhXWAa8gmRf1+ubxUKpWAUkW9duDPwH7AVVD1dkVUKfsXTvDXzJMRMTMtXwEcv4b7EfA9SXsAq4C3kj2EUdkXazfgFxHRAfxN0p3ABOD6N+xIagPaAKZOnQq8cw1DMjMzM6tTA/LvopO64bxYS91CoTAI+F+yh2vfVyqVuhyBo1QqvVwoFP4AjE5FfwPWLxQKAyta8cst9890d1x30VkznX89BbCS1z/PITXu53BgI2Bc6qP/tyrb1vR/ckQUI2J8RIxva2ur8fBmZmZm1hcKhcIA4Kdkra4fLA+P2U39ocA7gD+lotlkeeDuFdX2Jkvun+huX27BXzObSZoYEfcChwF3k/0yGwf8Dji4xv2MBJ6JiBWS9gI2r1JnBvAZSZcD6wN7AF9d2xMwMzMzazjKvwV/NRSBScCHgDcVCoV/S+XPlkqljkKh0EbWa2MR2UO73wKWAv8PoFQqPV8oFH4OnFcoFI4BhgPfBS7obgQdcIK/ph4GjpQ0FXgMuAh4APgfSScC99e4nyuB30pqB+YCj1Spcx0wEZhHdqfgaxFRdTglMzMzM6sbx6S/czqVbwEsBgYDZwNvIxvbfiZZN57Klv7jyMbCn87rE12d1tOB9cbnOa1JhCe6yo8nuqoPnugqf57oKn+e6Mp6We7N59/e4Fu5J64n//3U3D+HnrgF38zMzMwagurgIdtG4AS/SZVbkS0/5RZky5dbMPPna2Bm1r+cgZiZmZlZY2ish2xz4wS/SX1r5Il5h9CyTl3yPcB98PPmPvj5cx/8/PnuiVlr8je/D0hanGamNTMzMzPrV27BNzMzM7PG4Idsa+IW/LUkabikGyXNk7RQ0qFp1RckzZa0QNI2FXUvlTRL0hxJB6TygZLOTOXzJX0mlU+SNEPSdZIeknSxJF8zMzMzM+uSk8W19wHgqYjYMSLGADel8uciYmeySbCmpLKTgNsiYgKwF3CmpOFkEyEsSeUTgE9L2iJtswvwFWB7YEvgI/1xUmZmZmZ1Z4DyfzUAJ/hrbwGwj6QzJO0eEUtS+bXp74PA6LS8L/B1SXOBO4AhwGap/IhUfj+wAbBV2uaBiHg8IjqAXwC7VQtCUpukdkntxWKxF0/PzMzMzBqJ++CvpYh4VNI44EPA6ZJuSateS387eP1zFnBwRJQq9yFJwBci4uZO5ZOAzjO2VZ3BLSKKQDmzj2991aPomJmZmbUit+CvJUmbAq9GxBXAWcDO3VS/maxvvtK2O1WUf1bS4FS+deq6A7CLpC1S3/tDgbv74jzMzMzM6p2k3F+NwC34a297sr70q4AVwGeBq7uo+x3gHGB+SvIXA/sBl5B145mdyp8FDkzb3At8Px1nBnBdH5yDmZmZmTUJJ/hrKXWrublT8eiK9e3ApLS8DPhMlX2sAk5Mr39KvxJfjYhDO29jZmZm1nIa5CHXvLmLjpmZmZlZE3ELfh2LiDvIRttZbacu+V6vxmKrb711/PWqB0MGuh0jb74GZmb9yxmImZmZmTWGBnnINW9O8JvUlXf+Me8QWtbhe24JwOLnXsk5ktY2esNsIKrlHatyjqR1lVvufQ3y47snZq3J33wzMzMzsybiFnwzMzMzawweRacmbsFfQ5JGS3pE0iWSFkq6UtI+kmZKekzSLpLWl/RrSfMl3Sdph7RtV+V7SpqbXnMkrZvKvyppVqp/ap7nbWZmZmb1zS34a+edwEeBNmAW8AlgN2B/sjHtnwTmRMSBkvYGfgqMBU7tonwK8LmImClpBLBc0r7AVsAugIDrJe0RETP67zTNzMzM6oDcNl0Lf0pr54mIWJAmqloETI+IABaQTXa1G/AzgIi4DdhA0shuymcCZ0s6HhgVESuBfdNrDjAb2IYs4X8DSW2S2iW1F4vFvjxnMzMzM6tjbsFfO69VLK+qeL+K7LNdWWWbIGuJ/5fyiPi+pBuBDwH3Sdon1T09IqZ2F0hEFIFyZh8eRcfMzMysNbkFv2/NAA4HkDQJeC4iXuqqXNKW6Y7AGUA7WWv9zcDRqcsOkt4qaeN+Pg8zMzOz3GmAcn81Arfg961TgMskzQdeBY7sofwESXsBHcBDwO8i4jVJ7wLuVTa5w1LgP4Fn+uskzMzMzKxxOMFfQxGxGBhT8X5yF+sOqLLt812Uf6GLY/0Y+PHaxGtmZmbW8BqkBT1v7qJjZmZmZtZE3ILfpA7fc8u8Q2h5ozccnncIBgwZ6HaMvPkamJn1Lyf4ZmZmZtYY5C46tXCC36SWrVyVdwgta+igrLXyp7f/IedIWtsRe70TgOUd/i7kpdxy72uQH989MWtNLf/Nl3SKpClrun41j3Vip/f39MZ+zczMzFrCAOX/agAtn+D3szck+BGxa16BmJmZmVlzaskEX9JJkkqSfg8UUtmnJc2SNE/SNZKGVdnuDkk/kjRD0sOSJki6VtJjkr5bUe/Xkh6UtEhSWyr7PjBU0lxJV6aypenvCEnTJc2WtEDSAal8dDrOT9K+bpE0tO8/ITMzMzNrVC2X4EsaB3wc2An4CDAhrbo2IiZExI7Aw8AxXeziHxGxB3Ax8Bvgc2Rj3k+WtEGqc3REjAPGA8dL2iAivg4si4ixEXF4p30uBw6KiJ2BvYAfSv98imQr4IKI2A54ETh4bc7fzMzMrFFJyv3VCFouwQd2B66LiFcj4iXg+lQ+RtJdkhYAhwPbdbF9uf4CYFFEPB0RrwGPA29P646XNA+4L5Vt1UNMAr6XZrb9PfBWYJO07omImJuWHwRGV92B1CapXVJ7sVjs4XBmZmZm1qxadRSdqFI2DTgwIuZJmgxM6mLb19LfVRXL5feDJE0C9gEmRsSrku4AhvQQz+HARsC4iFghaXHFNpXH6ACqdtGJiCJQzuzDo+iYmZlZ02mQh1zz1oot+DOAgyQNlbQu8B+pfF3gaUmDyRLuNTUSeCEl99sA76lYtyLtv9o2z6Tkfi9g87U4vpmZmZm1sJZrwY+I2ZJ+CcwF/gTclVZ9E7g/lS0gS/jXxE3Asam7TYmsm05ZEZgvaXanfvhXAr+V1J7iemQNj21mZmZmLU4R1XqrWINzF50ceaKr+uCJrvLnia7y54murJfl3j/me2POyT1xPXHhCbl/Dj3xN9/MzMzMrIm0XBedVlFuRbb8lFuQLV9uwcyfr4GZ9Ro/ZFsT/6trZmZmZtZE3ILfpP7891fyDqFlbbbBcACmTX8s50ha2+T3ZdNPuP93ftwHP3++e2LWmpzgm5mZmVljcBedmvinvZmZmZlZE3GCX2ckfVnSwvQ6Ib2/NK3bPpUPyztOMzMzM6tP7qJTRySNA44C3k021uz9wBHAgZIOAk4CPhMRr+YXpZmZmVk+JHfRqYVb8OvLbsB1EfFKRCwFrgXeC0wGfgbcGREzq20oqU1Su6T2YrHYbwGbmZmZWX1xC3596epn6VbAUmDTrjaMiCJQzuzDo+iYmZlZ0/FDtjVxC359mUHWHWeYpOHAQcA84MfAHsAGkg7JM0AzMzMzq29uwa8jETFb0jTggVR0CXAkcGFEPCrpGOB2STMi4pm84jQzMzOz+uUEv85ExNnA2V2sexJ4Z/9GZGZmZlYn/JBtTdxFx8zMzMysibgFv0lttsHwvENoeZPft1XeIRgwZKDbMfLma2BmvcYP2dbE/+qamZmZmTURt+A3qcf++lLeIbSsrf5tPQBO3+XCnCNpbf/9wHEALO9YlXMkravccu9rkB/fPTFrTU7wzczMzKwheCbb2vinvZmZmZlZE3GC388kHSFpvqR5kn4maXNJ01PZdEmbSRoo6XFlRklaJWmPtP1dkjxUppmZmbWeAcr/1QCc4PcjSdsBJwF7R8SOwBeB84GfRsQOwJXAuRHRATwKbAvsBjwI7C5pHeBtEfGHXE7AzMzMzOqeE/z+tTdwdUQ8BxARzwMTgZ+n9T8jS+gB7gL2SK/TU/kEYFZ/BmxmZmZmjcUJfv8SED3UKa+/C9gd2AX4f8AoYBIwo+qOpTZJ7ZLai8VirwRrZmZmVlfy7p7jLjpWxXTgY5I2AJC0PnAP8PG0/nDg7rR8P7ArsCoilgNzgc+QJf7/IiKKETE+Isa3tbX13RmYmZmZWV3zMJn9KCIWSToNuFNSBzAHOB64VNJXgWeBo1Ld1yQ9CdyXNr8LOAxY0P+Rm5mZmdUBD5NZEyf4/SwiLgcu71S8dxd1d69Y/jmv99U3MzMzM6vKXXTMzMzMzJqIW/Cb1Fb/tl7eIbS8/37guLxDMGDIQLdj5M3XwMx6TYM85Jo3/6trZmZmZtZE3ILfpF5ctiLvEFrWqKGDAVjesSrnSFpbudXY1yE/vgb5890TazbyQ7Y18Te/DkmaLOn8vOMwMzMzs8bjBN/MzMzMrIk4we8jkkZLekTSJZIWSrpS0j6SZkp6TNIuktaX9GtJ8yXdJ2mHKvvZSNI1kmal13vzOB8zMzOz3OU9i22DPOTrPvh9653AR4E2YBbwCWA3YH/gROBJYE5EHChpb+CnwNhO+/gx8KOIuFvSZsDNwLv6J3wzMzMzazRO8PvWExGxAEDSImB6RISkBcBoYHPgYICIuE3SBpJGdtrHPsC2FQ+VrCdp3Yh4uV/OwMzMzMwairvo9K3XKpZXVbxfRfbjqtp9nuj0fgAwMSLGptdbqyX3ktoktUtqLxaLvRG7mZmZWX2R8n81ACf4+ZoBHA4gaRLwXES81KnOLcDny28kja22o4goRsT4iBjf1tbWJ8GamZmZWf1zF518nQJcJmk+8CpwZJU6xwMXpDqDyH4UHNtvEZqZmZnViwZ5yDVvTvD7SEQsBsZUvJ/cxboDqmw7DZiWlp8DDu2rOM3MzMysubiLjpmZmZlZE3ELfpMaNXRw3iG0PE8RXx98HfLna2BmvcY9dGrif3XNzMzMzJqIW/Cb1POv/iPvEFrW+sPeBMDyjlU5R9Layq3Gvg758TXIn++eWNNpkGEq8+Zv/mqSNFrSwrzjMDMzMzOrxgl+A5LkOy9mZmZmVpUT/DUzSNLlkuZLulrSMEnjJN0p6UFJN0t6C4CkT0uaJWmepGtS3ZGSFksakOoMk/SkpMGStpR0U9rPXZK2SXWmSTpb0u3AGTmeu5mZmVkuNEC5vxqBE/w1UwCKEbED8BLwOeA84JCIGAdcCpyW6l4bERMiYkfgYeCYiFgCzAP2THX+A7g5IlYAReALaT9TgAsrjrs1sE9EfKVvT8/MzMzMGpW7eqyZJyNiZlq+AjiRbOKqW5U9/DEQeDqtHyPpu8AoYARwcyr/JdkEVrcDHwculDQC2BW4Sq8/RLJOxXGvioiOvjghMzMzs7rXGA3ouXML/pqJTu9fBhZFxNj02j4i9k3rpgGfj4jtgVOBIan8euCDktYHxgG3kV2PFyv2MzYi3lVxnFe6CkhSm6R2Se3FYnHtz9DMzMzMGpIT/DWzmaSJafkw4D5go3JZ6ku/XVq/LvC0pMHA4eUdRMRS4AHgx8ANEdERES8BT0j6aNqPJO1YS0ARUYyI8RExvq2trTfO0czMzMwakBP8NfMwcKSk+cD6pP73wBmS5gFzybraAHwTuB+4FXik035+Cfxn+lt2OHBM2s8i4IA+OgczMzOzxiLl/2oA7oO/miJiMbBtlVVzgT2q1L8IuKiLfV1Np95kEfEE8IEqdSevdrBmZmZm1nKc4JuZmZlZY2iQYSrz5gS/Sa0/7E15h9DyPEV8ffB1yJ+vgZlZ//K/umZmZmZmTcQt+E3q1rl/yTuElvX+sW8FYHnHqpwjaW3lVmNfh/z4GuTPd0+s6biHTk38ze9HkkZJOq7i/WhJn8gzJjMzMzNrLk7w+9co4LiK96OB1UrwJQ3sxXjMzMzMGkfeQ2R6mEyT9GXg6PT2EuA9wJaS5pKNi7878K70/nLgXOD7wCRgHeCCiJgqaRLwLeBpYCzVh+k0MzMzM3OC31ckjQOOAt5N1mPsfrJJrcZExNhUZxIwJSL2S+/bgCURMUHSOsBMSbekXe6Stn2iP8/DzMzMzBqLE/y+sxtwXUS8AiDpWrIW++7sC+wg6ZD0fiSwFfAP4AEn92ZmZtbS3Lm8Jv6Y+s6adNIS8IWIGJteW0REuQX/lW43lNoktUtqLxaLa3BoMzMzM2sGTvD7zgzgQEnDJA0HDgJmAutW1Hm50/ubgc9KGgwgaeu0bY8iohgR4yNifFtbW++cgZmZmZk1HHfR6SMRMVvSNOCBVHRJRDwoaaakhcDvgBOBlZLmAdOAH5ONrDNbkoBngQP7OXQzMzOz+tQgo9jkzQl+H4qIs4GzO5V1HhbzfZ3en5hele5ILzMzMzNrAIVC4STgo2TPU74AXAucWCqVllbUeTdwATAGeBz4SqlU+l3F+hHA+cBHgBVkDcJfK5VKHd0d2110zMzMzKwhSMr9tRp2BX4A7Ew279G+wHnllYVCYQOyHh0zU52fAdcVCoWtKvZxATAB2Ifsx8JhwEk9Hdgt+E3q/WPfmncILc9TxNcHX4f8+RqYWSsqlUofrnxbKBS+CUytKDsceAk4oVQqBfBQoVD4IPAZYEqhUHhzqvP+Uqn0AEChUPgGcHqhUPhuqVRa1dWxneCbmZmZmdWoUCiMAkZVWfViqVR6sZtNNwQq1+8C3J6S+7LpZK31AOOAIBu4pXL9xsAWwB+7OpAT/Cb1yopuu2ZZHxo+eCAAPzjoypwjaW1fu+5wAJZ3dNnAYX2s3HLva5Af3z2xplMfz9ieAHyrSvmpwCnVNigUCiOBKcClFcUbA3M6VX02lZfXP9+pv/2zFeuc4PcFSQcCj0bEQz3UOwVYGhFn9UdcZmZmZtZnziF72LWzF6tVLhQK6wDXkD1E+/2KVT39XKm2PqqU/Qsn+GvnQOAGoNsE38zMzMx6wYD8m/BTN5wXa6lbKBQGAf9LNu/R+0ql0sqK1X/j9db6so2AZyrWr18oFAZWtOKX6z9DN3zvrhNJv5b0oKRFktpS2VJJp0maJ+k+SZtI2hXYHzhT0lxJW6bXTWn7uyRtU2X/Y9M+5ku6TtKbU/kdks6RdI+khZJ2SeXDJV0qaZakOZIO6M/Pw8zMzMxWX6FQGAD8FHgn8MHK4TGTB4BJncr2Bu5Py7PJWvF377T+GeCJ7o7tBP9fHR0R44DxwPGSNgCGA/dFxI5kDzp8OiLuAa4HvhoRYyPij0AR+ELafgpwYZX9/xT4r4jYAVjAG/twDY+IXYHjeL2P1knAbRExAdiL7AdFTbPbmpmZmVluimQJ/CeBNxUKhX9Lr4Fp/ZXAyEKhcE6hUHhXoVD4L+DdpJF2SqXS88DPgfMKhcIuhUJhL+C7wAXdjaAD7qJTzfGSDkrLbyebnOAfZF1xAB4E3t95I0kjyMY7vapijNR1OtUZCYyKiDtT0eXAVRVVfgEQETMkrSdpFNmYqftLmpLqDAE2Ax5e0xM0MzMza0iNNZPtMelv5wdptwAWl0qlvxcKhQ+RjXX/WbI++h8plUqPVdQ9Lq2fzusTXZ3W04Gd4FeQNIlsaKKJEfGqpDvIEuoVEVF+qKGD6p/bAODFiBi7FiF0fnAiyG7NHBwRpe42TN2J2gCmTp3K4Ucd0111MzMzM+tDpVKpx18jpVLpPrLhMLtavxQ4Mr1q5i46bzQSeCEl99sA7+mh/stkD00QES8BT0j6KIAyO1ZWjoglwAuSyn2pPgncWVHl0LTtbsCSVP9m4AtKtwUk7VQtkIgoRsT4iBjf1tZW+xmbmZmZNQrVwasBOMF/o5uAQZLmA98B7uuh/v8CX00Pv25JNtvYMZLmAYuAag/EHknWj34+MBb4dsW6FyTdA1zM67d1vgMMBuZLWpjem5mZmZlV5S46FSLiNeCDVVaNqKhzNXB1Wp4JbNup7geq7PeUiuW5dH1n4JqI+O9O2y4jm7LYzMzMzKxHTvDNzMzMrDHUwTj4jcAJfp2IiEm9ub/hgwf2XMn61NeuOzzvEAwYMtA9EfPma2Bm1r+c4JuZmZlZY3ADfk2c4Dep5R3dzn9gfajcWvnd0afnHElr+8bi7HEWfxfyU/4u+Brkx3dPzFqTv/m9SNK3Je2zmttsJOn+NBLP7pI+KulhSbf3VZxmZmZm1rzcgt+LIuLkauWSBkZERxebvQ94JCKOTHVvAo6LCCf4ZmZmZpUaaybb3DjBX0OSvkk27v2TwHPAg8AY4IaIuFrSYuBSYF/gfEnrks00+ybgD2STXG0N/AAYKmkucB2wG7CFpOuBrwPfByYB6wAXRMTUfjpFMzMzM2tATvDXgKTxwMHATmSf4WyyBL+z5RGxW9pmg4j4SVr+LnBMRJwn6WRgfER8Pq3bC5gSEe2S2shmtJ0gaR1gpqRbIuKJPj9JMzMzM2tITvDXzG7Ab9IkVEj6bRf1flmxPCYl9qPIJs66uYbj7AvsIOmQ9H4ksBXgBN/MzMxajjwOfk38kO2aqfX/rlcqlqcBn4+I7YFTgSE1HucLETE2vbaIiFuqVpTaJLVLai8WizWGZ2ZmZmbNxgn+mrkb+A9JQySNAD5cwzbrAk9LGkzWd78WNwOfTdsgaWtJw6tVjIhiRIyPiPFtbW017t7MzMysgagOXg3AXXTWQETMSg/BzgP+BLQDS3rY7JvA/an+ArKEvyeXAKOB2ZIEPAscuGZRm5mZmVkrcIK/5s6KiFMkDQNmAD8sP0QLEBGjKytHxEXARZ13EhHTyLrvlN9PqlheBZyYXmZmZmZmPXKCv+aKkrYl60t/eUTMzjsgMzMzs6bmcfBr4gR/DUXEJ/KOoTuenjx/31j833mHYPi7UA98DczM+pcTfDMzMzNrDB4msyZO8JvUmZ/4Zc+VrE989eeHArC8Y1XOkbS2cqvxspW+DnkZOii7Bv4u5Md3T8xak7/5ZmZmZmZNxC34DUTSKcDSiDgr71jMzMzM+p176NTELfhmZmZmZk3ELfj9JM1A+yvgbcBA4DvAH4CzgRHAc8DkiHha0qeBNuBNqc4nI+LVXAI3MzMzqxceJrMmbsHvPx8AnoqIHSNiDHATcB5wSESMAy4FTkt1r42ICRGxI/AwcEwuEZuZmZlZw3ELfv9ZAJwl6QzgBuAFYAxwq7JfowOBp1PdMZK+C4wia92/uaedS2oja/Vn6tSpwMheDt/MzMzMGoET/H4SEY9KGgd8CDgduBVYFBETq1SfBhwYEfMkTQYm1bD/IlAsvz3zDg+TaWZmZk3GfU9q4o+pn0jaFHg1Iq4AzgLeDWwkaWJaP1jSdqn6usDTkgYDh+cSsJmZmZk1JLfg95/tgTMlrQJWAJ8FVgLnShpJdi3OARYB3wTuB/5E1rVn3TwCNjMzM6srfsi2Jk7w+0lE3Ez1vvR7VKl7EXBRlfJTej8yMzMzM2smTvCb1Fd/fmjeIbQ8TxFfH4YO8nXIm78LZmb9ywm+mZmZmTUEuYtOTZzgN6nfPPDnvENoWQfsshkALy1fkXMkrW29IYMBWLayI+dIWtfQQQMBWN6xKudIWpfvnpi1Jif4ZmZmZtYY/Ju1Jv6Y6pCkSyRtm3ccZmZmZtZ43IJfhyLiU3nHYGZmZmaNyS34OZI0WtIjki6XNF/S1ZKGSbpD0vhU5xhJj6ayn0g6P++4zczMzHIh5f9qAE7w81cAihGxA/AScFx5RZr99pvAe4D3A9vkEqGZmZmZNQwn+Pl7MiJmpuUrgN0q1u0C3BkRz0fECuCqrnYiqU1Su6T2YrHYh+GamZmZWT1zH/z8RTfva74PFBFFoJzZh4fJNDMzs6bTIF1k8uYW/PxtJmliWj4MuLti3QPAnpLeLGkQcHC/R2dmZmZmDcUJfv4eBo6UNB9YH7iovCIi/gJ8D7gf+D3wELAkjyDNzMzMcjegDl4NwF108rcqIo7tVDapYvnnEVFMLfjXAbf0W2RmZmZm1nCc4Ne/UyTtAwwhS+5/XctGB+yyWV/GZDVYb8jgvEMwYOiggXmH0PKGDGyQJi8zsybhBD9HEbEYGNNDnSn9E42ZmZlZnfNDtjVxgt+k9td+eYfQsq6PGwB4/NmlOUfS2t6x0QgAlq3syDmS1lW+e7K8Y1XOkbQu3z0xa01O8M3MzMysMbgFvyb+ad+LJI2WtDDvOMzMzMysdTnBrxNplBwzMzMzs7XipLL3DZT0E2BX4C/AAUABuBgYBvwRODoiXpB0B3AP8F7gekl/Br4FdABLImIPSQOB75MNnbkOcEFETO3fUzIzMzOrA26arok/pt63FVkSvh3wItnssz8F/isidgAWkCXxZaMiYs+I+CFwMvDvEbEjsH9afwxZsj8BmAB8WtIW/XMqZmZmZtZo3ILf+56IiLlp+UFgS7Ik/s5UdjlwVUX9X1YszwSmSfoVcG0q2xfYQdIh6f1Ish8RT1QeVFIb0AYwdaob+M3MzKwJ+SHbmjjB732vVSx3AKN6qP9KeSEijpX0buDDwFxJYwEBX4iIm7vbSUQUgWL57Q2fuX41wzYzMzOzZuAuOn1vCfCCpN3T+08Cd1arKGnLiLg/Ik4GngPeDtwMfFbS4FRna0nD+yFuMzMzM2tAbsHvH0cCF0saBjwOHNVFvTMlbUXWaj8dmAfMB0YDsyUJeBY4sK8DNjMzM6s77qJTEyf4vSgiFgNjKt6fVbH6PVXqT+r0/iPVdgucmF5mZmZmZt1ygt+kro8b8g6h5b1joxF5h2DA0EED8w6h5Q0Z6N6gZtZL/M9JTfwxmZmZmZk1EbfgN6lXV3TkHULLGjY4azE+Yfjnc46ktZ3zyvkALFvp70JeyndPlnesyjmS1uW7J2atyQm+mZmZmTUGP2RbE/+072eSTpE0Je84zMzMzKw5uQW/AUkaFBEr847DzMzMrF+5Bb8mbsHvB5JOklSS9HugkMq2lHSTpAcl3SVpm1S+kaRrJM1Kr/em8lMkFSXdAvw0v7MxMzMzs3rmFvw+Jmkc8HFgJ7LPezbwIFAEjo2IxyS9G7gQ2Bv4MfCjiLhb0mZkM9m+K+1uHLBbRCzr59MwMzMzswbhBL/v7Q5cFxGvAki6HhgC7ApcpddvNa2T/u4DbFtRvp6kddPy9V0l95LagDaAqVOn8p9HHdPb52FmZmaWL/c9qYkT/P4Rnd4PAF6MiLFV6g4AJnZO5FPC/0qXB4gokt0VAAgPk2lmZmbWmvw7qO/NAA6SNDS1xP8H8CrwhKSPAiizY6p/C/DPAdQlje3neM3MzMysgTnB72MRMRv4JTAXuAa4K606HDhG0jxgEXBAKj8eGC9pvqSHgGP7N2IzMzOzOiXl/2oA7qLTDyLiNOC0Kqs+UKXuc8ChVcpP6f3IzMzMzKzZOMFvUsMGD8w7hJZ3zivn5x2CAUMH+buQtyEDfbPYzHpJYzSg587/6pqZmZmZNRG34DepZSs9ik5eyi3Gvgb5Kl+Hl5avyDmS1rXekMEALO9YlXMkrct3T8xakxN8MzMzM2sMA9xHpxb+ad9AJE2SdEPecZiZmZlZ/XILvpmZmZk1hgYZpjJvbsFfTZK+Jun4tPwjSbel5fdJukLSvpLulTRb0lWSRqT1J0uaJWmhpKLS1LSSjpf0UBr3/n9T2XBJl6b6cyQd0FU8ZmZmZmaVnOCvvhnA7ml5PDBC0mBgN2AB8A1gn4jYGWgHvpzqnh8REyJiDDAU2C+Vfx3YKSJ24PVJrU4CbouICcBewJmShvfxeZmZmZlZE3CCv/oeBMZJWhd4DbiXLNHfHVgGbAvMlDQXOBLYPG23l6T7JS0A9ga2S+XzgSsl/SewMpXtC3w97eMOYAiwWXdBSWqT1C6pvVgs9sZ5mpmZmdUX1cGrAbgP/mqKiBWSFgNHAfeQJeh7AVsCTwC3RsRhldtIGgJcCIyPiCclnUKWtAN8GNgD2B/4pqTtyP73OTgiSp32s0k3cRWBcmYfHqLRzMzMrDW5BX/NzACmpL93kXWtmQvcB7xX0jsBJA2TtDWvJ/PPpT75h6T1A4C3R8TtwNeAUcAI4GbgCxX99Hfqn9MyMzMzq2MDlP+rATjBXzN3AW8B7o2IvwHLgbsi4llgMvALSfPJEv5tIuJF4CdkffR/DcxK+xkIXJG67cwBfpTqfgcYDMyXtDC9NzMzMzPrkSIi7xis97mLTo48k2198Ey2+fNMtvnzTLbWy3Jvvv7hqb/PPXH9yrf2yf1z6In74JuZmZlZY/A4+DVxgt+kyq2Xlh9fg/pQbkW2/LgV2cysfznBNzMzM7PG4Ab8mjjBb1L3P/Zs3iG0rHdvtRHgfsd5K7caL3Ef/NyMdB/83PnuiVlr8jc/J5JGpxFyaq0/WdKmfRmTmZmZmTU+t+A3jsnAQuCpnOMwMzMzy0eDjEOfN7fg52uQpMslzZd0dZoY62RJsyQtlFRU5hBgPHClpLmShuYduJmZmZnVJyf4+SoAxYjYAXgJOA44PyImRMQYYCiwX0RcDbQDh0fE2IhYll/IZmZmZlbPnODn68mImJmWrwB2A/aSdH+a3XZvYLtadiSpTVK7pPZisdhH4ZqZmZnlSMr/1QDcBz9fnWdjC+BCYHxEPCnpFGBITTuKKALlzD48io6ZmZlZa3ILfr42kzQxLR8G3J2Wn5M0Ajikou7LwLr9GZyZmZlZXVEdvBrAarXgSxoAjIiIl/oonlbzMHCkpKnAY8BFwJuBBcBiYFZF3WnAxZKWARPdD9/MzMzMqukxwZf0c+BYoAN4EBgp6eyIOLOvg2tmEbEY2LbKqm+kV+f61wDX9HFYZmZmZtbgaumis21qsT8Q+H/AZsAn+zIoMzMzM7N/MUD5vxpALV10BksaTJbgnx8RKyR1fjjU6sy7t9oo7xBanqeIrw8jhwzOO4SW5++CmVn/qiXBn0rWH3weMEPS5mRjtpuZmZmZ9Z8GGaYybz0m+BFxLnBuRdGfJO3VdyFZb1i2siPvEFrW0EEDAVjesSrnSFpbudX4mZdfyzmS1rXxuusA/i7kyXdPzFpTlwm+pC/3sO3ZvRyLmZmZmZmtpe5a8D3mupmZmZnVD9+UqkmXCX5EnNqfgTQSSaOBm8gmpnoP2fMJlwGnAhsDhwN/AC4F3gG8CrRFxPw0O+3SiDgr7WshsB/wLPAr4G3AQOA7EfFLSePI7paMAJ4DJkfE0/1zpmZmZmbWaGoZB39rsgmYNomIMZJ2APaPiO/2eXT17Z3AR4E2sgmpPgHsBuwPnAg8CcyJiAMl7Q38FBjbzf4+ADwVER8GkDQyjV50HnBARDwr6VDgNODovjklMzMzszrmh2xrUssoOj8Bvko2mg6pFfrnQKsn+E9ExAIASYuA6RERkhYAo4HNgYMBIuI2SRtIGtnN/hYAZ0k6A7ghIu6SNAYYA9yq7H/ogUDV1ntJbWQ/Npg6dSqfPPqY3jhHMzMzM1tDhULhI8DngPHAeqVSSRXrJgG3d9pkXqlUGltRZwRwPvARYAUwDfhaqVTqdjSVWhL8YRHxgN74i2llDds1u8qhOVZVvF9F9rlW+4wilVf2IBsCEBGPpu44HwJOl3QLcB2wKCIm9hRMRBSBYvmtR9ExMzMzy90w4Dbg98D3uqjzlorlFZ3WXUD242Afsu7aVwBLgG93d9BaEvznJG1Jlpwi6RC6aEW2N5hB1hf/O5ImAc9FxEuSFpP1uUfSzsAWaXlT4PmIuELSUmAy8H1gI0kTI+Le1GVn64hY1N8nY2ZmZpa7BuuiUyqVroB/ttZ3Veev1coLhcKbyXLJ95dKpQdS2TeA0wuFwndLpVKXYxDXkuB/jqxleBtJfwGeSAez7p0CXCZpPtlDtkem8muAIyTNJeu7/2gq3x44U9Iqsl9vn42If6QfVOem7j2DgHMAJ/hmZmZmOSgUCqOAUVVWvVgqlV5cg/0tJuvdcQ9Z95s/p1XjyBrYZ1RUn042oMsWwB+72mctE109DuwjaTgwICJeXt3Am01ELCbrG19+P7mLdQdU2XYZsG+V3S4Gbq5Sfy6wx5pHa2ZmZtYk6mOYzBOAb1UpP5WsgbdWTwPHAA8C6wPfBG4vFApjSqXSMrJE/vlO/e2fTX83Zm0SfEkbkJ3EbkBIuhv4dkT8fTVOwMzMzMysGZxD9rBrZy+uzk5KpVIJKJXfFwqFduDPZF25rwKq9UeKWvZdSxed/yW7NXBwen848Euyzv5Wp4YOGph3CC3PU8TXh43XXSfvEFqevwtm1kxSN5wX+2C/LxcKhT+QjcYI8Ddg/UKhMLCiFX/j9PeZ7vZVS4K/fkR8p+L9dyUduBrxmpmZmZmtvQZ7yHZ1FAqFoWQTpP4pFc0ma8XfHbgjle1Nltw/0d2+aknwb5f0cbJZVgEOAW5cvZCtv738mkcyzcu662RfqzsXebCpPO25XTbq2F9eeDXnSFrXW988DIDlHV0O9GB9zHdPzPJVKBTWBzYjmyCVQqEwNq16iGzExL+SDZ4yiqxL/FLg/wGUSqXnC4XCz4HzCoXCMcBwsnmoLuhuBB3oJsGX9DJZPx8BXyYbdxOyxxuWUv3hgoYiaRrZpFJXdyrfFDg3Ig6RNBkYHxGf74XjnQIsjYiz1nZfZmZmZi2n8Vrw9wcuq3g/J/3dAhgMnA28jWxs+5nA+0ql0tKK+seRjYU/ndcnujqtp4N2meBHxLq1x95cIuIpsjsVZmZmZmZrpFQqTaP6A7mQJe4X9LD9UrKh1o/srl5nNd27k/RmSbtI2qP8Wp2D1AtJR0iaL2mepJ+l4j0k3SPp8TTmPJJGS1pYsemmkm6S9JikH1Tsb2nF8iHpjgCS/kPS/ZLmSPq9pE0q9rWtpDvS8Y6vdjxJU1JrP5I+LWlWivkaScN6+WMxMzMzsybSY4Iv6VNko+jcTDa+582s3hifdUHSdsBJwN4RsSPwxbTqLWRDgO5HNnNsNWOBQ8kmozpU0tt7ONzdwHsiYieyUYi+VrFuG+DfgV2Ab6XZabtzbURMSDE/TDZeqpmZmVnrGVAHrwZQS5hfBCYAf4qIvYCdeH2Q/UayN3B1RDwHEBHPp/JfR8SqiHgI2KSLbadHxJKIWE72UMTmPRzrbcDNkhYAXwW2q1h3Y0S8luJ4pptjlo2RdFfa1+Gd9vVPktoktUtqLxaLPezSzMzMzJpVLaPoLI+I5ZKQtE5EPCKp0OeR9T5RfXKA1zrVqaayTgevf26V+xtSsXwecHZEXC9pEm+841FtXyt544+tyn1NAw6MiHnpgd9J1QKMiCJQzuzDo+iYmZmZtaZaWvD/T9Io4NfArZJ+AzzVl0H1kenAx9LMvEhavxf2+TdJ75I0ADioonwk8Je0XMtDEX8DNpa0gaR1yLoLla0LPJ268hzeCzGbmZmZNSYp/1cD6LEFPyLKiespkm4nS15/16dR9YGIWCTpNOBOSR28PkzR2vg6cAPwJLAQGJHKTwGukvQX4D6yoZC6i22FpG8D95NNXPBIxepvpvI/AQvIEn4zMzMzs6oUUa3XSg8bSX+OiM36IB7rHe6ikyNPdFUfPNFV/jzRVf480ZX1stybr3944b2rn7j2sq8cNzH3z6Ena/rNr/sTMzMzMzNrRbU8ZFtN7r+erHvlVmTLT7kF2fJVbkW2/LgV2cysf3WZBUr6clereL2vuZmZmZlZ/3B7QU26a+bt7mHOH/d2INa7Xli2Iu8QWtabh2Zzlz33yms91LS+tOHwdQBY/NwrOUfSukZvOBxwH/w8+e6JWWvqMsGPiFP7M5C+IukSsjHpH8o7lrUl6cSI+F7ecZiZmZnlQQ0yTGXemv6nfUR8qhmS++TEvAMwMzMzs/rWNAm+pOGSbpQ0T9JCSYem8jskjU/LF0lql7RI0qkV2y6W9D1J96b1O0u6WdIfJR1bUe+rkmZJml/evpvjjpN0p6QH077+5YlLSZtIui5tO0/Srqn812m7RZLaUtn3gaGS5kq6sg8/SjMzMzNrYM001MoHgKci4sMAkkZWqXNSRDwvaSAwXdIOETE/rXsyIiZK+hEwDXgvMARYBFwsaV9gK2AXsgeNr5e0B7BR5+OmWWfPAw6IiGdT0n8acHSneM4F7oyIg1JM5YeXj05xDgVmSbomIr4u6fMRMXZtPygzMzOzhuQuOjXpsQVf0hclrafM/0ianZLderMA2EfSGZJ2j4glVep8TNJssllstwO2rVh3fcV+7o+IlyPiWWC5pFHAvuk1B5gNbEOW8Fc7bgEYA9wqaS7wDeBtVeLZG7gIICI6KmI+XtI8sllw356O0y1JbenuQ3uxWOypupmZmZk1qVpa8I+OiB9L+ney1uqjgMuAW/o0stUUEY9KGgd8CDhd0i0R8e3yeklbAFOACRHxgqRpZC30ZeUhT1ZVLJffDyJrtT89IqZ2Pnbn4wLXAYsiYuLqnoekScA+wMSIeFXSHZ3irCoiikA5sw+PomNmZmbNxg34tamlD375o/wQcFlEzKMOZ7KVtCnwakRcAZwF7NypynrAK8ASSZsAH1zNQ9wMHC1pRDreWyVt3MVxS8BGkiamuoMlbVdln9OBz6Y6AyWtB4wEXkjJ/TbAeyrqr0jdf8zMzMzMqqqlBf/B1Cq9BfDfktYla9WuN9sDZ0paBawgJc5lETFP0hyyPvWPAzNXZ+cRcYukdwH3piGalgL/Cbyz83Ej4h+SDgHOTc8CDALOSceu9EWgKOkYoCPFfBNwrKT5ZD8U7quoXwTmS5odEYevTvxmZmZm1hoUEd1XkAYAY4HHI+JFSesDb6t4ONXqj7vo5MgTXdUHT3SVP090lT9PdGW9LPceHOdc8kD3iWs/OOFTu+T+OfSklm/+RKCUkvv/JHtgtNoDrGZmZmZmlrNauuhcBOwoaUfga8D/AD8F9uzLwGztlFuRLT/lFmTLV7kV2fLjVmQz6zX+56QmtXxMKyPrx3MA8OOI+DGwbt+GZWZmZmZma6KWFvyXJf032QOle6QJmdw8XOdeWdGRdwgta/jggYD7Heet3Gr8+LNLc46kdb1jo2zuPn8X8uO7J2atqZZv/qFk48IfExF/Bd4KnNmnUVlVkk6QNCzvOMzMzMzyICn3VyPoMcGPiL9GxNkRcVd6/+eI+Gnfh9aa0ozBXV2XEwAn+GZmZmbWpR4TfEnvkTRL0lJJ/5DUIcmj6PQiSaMlPSzpQmA28D+S2iUtknRqqnM8sClwu6Tb84zXzMzMLBdS/q8GUEsf/POBjwNXAeOBI4Ct+jKoFlUAjoqI4yStHxHPp+cdpkvaISLOlfRlYK+IeC7nWM3MzMysTtX09E1E/AEYGBEdEXEZMKlPo2pNf4qI8qy1H5M0G5gDbAds29PGktpSq397sVjsyzjNzMzMrI7V0oL/qqQ3AXMl/QB4GvDA0r3vFQBJWwBTgAkR8YKkacCQnjaOiCJQzuzDo+iYmZlZs2mQHjK5q6UF/5PAQODzZEno24GD+zKoFrce2ee8RNImwAcr1r2M5yAwMzMzs2702IIfEX9Ki8uAU/s2HIuIeZLmAIuAx4GZFauLwO8kPR0Re+USoJmZmZnVtS4TfEkLgOhqfUTs0CcRtaCIWAyMqXg/uYt65wHn9U9UZmZmZnXGfXRq0l0L/n79FoWZmZmZmfWK7hL8wcAmEVHZRQRJuwNP9WlUttaGDx6Ydwgtz1PE14d3bDQi7xBanr8LZtZbNMAt+LXo7l/dc8ge6uxsWVpnZmZmZmZ1prsW/NERMb9zYUS0SxrddyFZb/j7K//IO4SWtcHwNwG+BnkrX4f/uemRnCNpXcd8YBsAlnesyjmS1uW7J2atqbtvfndjrw/t7UAahaTxks7N4bijJX2iv49rZmZmVjdUB68G0F2CP0vSpzsXSjoGeLDvQqoPkqre3YiI9og4vr/jAUYDTvDNzMzMrFvdddE5AbhO0uG8ntCPB94EHNTHca211I3oJuB+YCfgUeCIiHhV0jjgbGAE8BwwOSKelnQHcA/wXuB6SX8GvgV0AEsiYg9Jk4ApEbGfpF3InkcYSvZswlERUZI0GdgfGAZsCVwXEV+rEuME4MdkMwO/BrwP2AD4Ga/PFvz5iLgH+D7wLklzgcsj4ke980mZmZmZNQZ5mMyadJngR8TfgF0l7cXrY7TfGBG39UtkvaMAHBMRMyVdChwn6cdkY8kfEBHPSjoUOA04Om0zKiL2hH/OBfDvEfEXSaOq7P8RYI+IWClpH+B7vD7L71iyHxavASVJ50XEk+UNJb0J+CVwaETMkrQe2Y+EZ4D3R8RySVsBvyD7YfV10g+LXvpszMzMzKwJ1TKT7e3A7f0QS194smKYzyuA48la9ccAt6ZfgQOBpyu2+WXF8kxgmqRfAddW2f9I4PKUiAfZ0KJl0yNiCYCkh4DNgScr1heApyNiFkBEvJTqDgfOlzSW7M7B1rWcqKQ2oA1g6tSpHHz45Fo2MzMzM7Mm02OC3+A6z8QbZI9HLIqIiV1s88o/K0ccK+ndwIeBuSnprvQd4PaIOCh1CbqjYt1rFcsd/OtnrSrxAXwJ+BuwI9kzEsu7iPMNIqIIFMtvPYKLmZmZNRv30KlNs4+ftZmkciJ/GHA3UAI2KpdLGixpu2obS9oyIu6PiJPJ+uq/vVOVkcBf0vLk1YztEWDT1A8fSeumB3tHkrXsrwI+SXaHAbI5CdZdzWOYmZmZWYtp9gT/YeBISfOB9YGLIuIfwCHAGZLmAXOBXbvY/kxJCyQtBGYA8zqt/wFwuqSZvJ6I1yTFcShwXorjVrKhSS9MMd9H1j2nfEdhPrBS0jxJX1qdY5mZmZk1BSn/VwNQRLVeIo0vdZm5ISLG9FS3CbmLTo480VV98ERX+fNEV/nzRFfWy3LPbi/437m5J66f+/jY3D+Hnvibb2ZmZmbWRJr2IduIWMzrw3u2nHLrpeXH16A+lFuRLT9uRTaz3uJx8Gvjf3XNzMzMzJpI07bgt7plK93nNS9DB2W/m5f+Y2XOkbS2EW/K/nm75Hfug5+XT33QffDz5rsn1nT8v3RN/DGtAUmj08g6tdZfLGnDXjjuiWu7DzMzMzNrbk7wG4sTfDMzMzPrlhP8NTdI0uWS5ku6WtIwSe+TNCeNnX+ppHUq6n9V0gPp9U4ASZtIui6NbT9P0q6p/NeSHpS0SFJbKvs+MFTSXElX9v/pmpmZmeVLUu6vRuAEf80VgGJE7AC8BHwZmAYcGhHbkz3f8NmK+i9FxC7A+cA5qexc4M6I2BHYGViUyo+OiHHAeOB4SRtExNeBZRExNiIO79tTMzMzM7NG5QR/zT0ZETPT8hXA+4AnIuLRVHY5sEdF/V9U/J2YlvcGLgKIiI6IWJLKj0+z294HvB3YqqdgJLVJapfUXiwW1/SczMzMzKzBeRSdNbe6M6lFF8tvIGkSsA8wMSJelXQHMKTHnUcUgXJmHx5Fx8zMzJpOg3SRyZtb8NfcZpLKLfGHAb8HRpf71wOfBO6sqH9oxd970/J0UjceSQMlrQeMBF5Iyf02wHsq9rFC0uDePxUzMzMzaxZO8Nfcw8CRkuYD6wM/Ao4CrpK0AFgFXFxRfx1J9wNfBL6Uyr4I7JXqPwhsB9xE9gDvfOA7ZN10yorAfD9ka2ZmZq1Iyv/VCNxFZw1ExGJg2yqrpgM7Vak/Oi2e2qn8b8ABVfbzwS6O+1/Af61GqGZmZmbWYtyCb2ZmZmbWRNyC36SGDvJvt7yNeJO/XvXgUx/cJu8QWt6Qgf73yMx6R6OMQ583/6trZmZmZtZE3MTYpJZ3eJjMvJRbK5et7Mg5ktY2dNBAAIrXL+qhpvWVtv23A/zvUZ5898Sajv+Xrok/ppxJOl7Swx4Zx8zMzMx6g1vw83cc8MGIeCLvQMzMzMys8bkFP0eSLgbeAVwv6SuSfi1pvqT7JO0gaZCkWWl2WySdLum0PGM2MzMzy4uk3F+NwAl+jiLiWOApYC9gNDAnInYATgR+GhErgcnARZLeD3yATmPpm5mZmZlVcoJfP3YDfgYQEbcBG0gaGRGLUvlvgaMj4h/VNpbUJqldUnuxWOy3oM3MzMz6Td7T2DZIC7774NePav/HRPq7PfAisElXG0dEEShn9uFRK8zMzMxak1vw68cM4HCA1Of+uYh4SdJHgA2APYBzJY3KK0AzMzMzq39uwa8fpwCXSZoPvAocKWlD4PvA+yLiSUnnAz8GjswvTDMzM7N8NEgPmdw5wc9ZRIyueHtAlSpbV9Q9t88DMjMzM7OG5gTfzMzMzBqDm/Br4gS/SXl68vwNHTQw7xAMaNt/u7xDaHn+98jMrH/5X10zMzMzsybiFvwmtWylh8nMy9BB2e/mF16tOmWB9ZM3D3sTABf+cl7OkbSu4w7dEQAP25sf3z2xZqMB7qJTC3/zq5A0WtLCfjrWZEmb9sexzMzMzKz5uQU/f5OBhcBTtW4gaVBErOyziMzMzMzqkJ+xrY0T/K4NlPQTYFfgL2RDWP4n0Aa8CfgD8MmIeFXSNGAZsA2wOXAU2Vj1E4H7I2KypIHA/wDjyWaovRR4Mr2/UtKyVH9b4GxgBPAcMDkinpZ0B3AP8F7geuCHff0BmJmZmVnjcRedrm0FXBAR2wEvAgcD10bEhIjYEXgYOKai/puBvYEvAb8FfgRsB2wvaSwwFnhrRIyJiO2ByyLiaqAdODwixgIrgfOAQyJiHNmPgNMqjjEqIvaMCCf3ZmZmZlaVE/yuPRERc9Pyg8BoYIykuyQtAA4nS+DLfhsRASwA/hYRCyJiFbAobfs48A5J50n6APBSlWMWgDHArZLmAt8A3lax/pddBSupTVK7pPZisbjaJ2tmZmZW96T8Xw3AXXS69lrFcgcwFJgGHBgR8yRNBiZVqb+q07argEER8YKkHYF/Bz4HfAw4utMxBSyKiIldxPRKV8FGRBEoZ/bhUXTMzMzMWpNb8FfPusDTkgaTteDXTNKGwICIuAb4JrBzWvVy2i9ACdhI0sS0zWBJnqXHzMzMzGrmFvzV803gfuBPZF1x1u2++hu8FbhMUvlH1X+nv9OAiysesj0EOFfSSLLrcw5ZNx8zMzOzlqYG6SKTNyf4VUTEYrK+8OX3Z1WsvqhK/cndbDu5ourOdJJa9K+pKJoL7FGl3qQeAzczMzOzlucE38zMzMwagxvwa+IEv0kNHeTHK/L25mFvyjsEA447dMe8Q2h5Qwb63yMzs/7kf3XNzMzMzJqIW/Cb1PIOD5OZl3JrpYcqzVf5LtYPPnB5zpG0rq/ddCTgf4/y5Lsn1mw0wH10auFvfj+StDgNl9m5fJqkQ6qUj5d0blqeJGnX/ojTzMzMzBqXW/B7mbLxm5RmsV0rEdEOtKe3k4ClwD1ru18zMzOzRuT2+9q4Bb8XSBot6WFJFwKzgf+R1C5pkaRTO1X/qqQH0uudFeX7SLpL0qOS9kv7nSTpBkmjgWOBL0maK2n3fjkxMzMzM2s4bsHvPQXgqIg4TtL6EfG8pIHAdEk7RMT8VO+liNhF0hFkk1jtl8pHA3sCWwK3Vyb/EbFY0sXA0k5j8puZmZmZvYFb8HvPnyLivrT8MUmzgTnAdsC2FfV+UfF3YkX5ryJiVUQ8BjwObLM6B5fUlu4atBeLxTU7AzMzM7M6Jin3VyNwC37veQVA0hbAFGBCRLwgaRowpKJe1LBc7X23IqIIlDP78KgVZmZmZq3JLfi9bz2yZH+JpE2AD3Zaf2jF33sryj8qaYCkLYF3AKVO270MrNsH8ZqZmZk1BCn/VyNwC34vi4h5kuYAi8i62szsVGUdSfeT/bg6rKK8BNwJbAIcGxHLO90G+i1wtaQDgC9ExF19dQ5mZmZm1ric4PeCiFgMjKl4P7mLeqPT4qmdyruqfwdwR1p+FNhh7SI1MzMzs2bnBN/MzMzMGkKjdJHJmxP8JuXpyfM3dJCvQT342k1H5h1Cy/O/R2Zm/csJvpmZmZk1BHku25o4wW9Sy1Z6mMy8lFvuPVRpvsqtxqdt7bnh8nLSo1MAfxfy5LsnZq3J3/x+JOmetdx+tKSFvRWPmZmZmTUft+D3o4jYtXOZpIER0dHVezMzMzPLNNpDtoVC4SPA54DxwHqlUkmd1r8buIBsNMbHga+USqXfVawfAZwPfARYAUwDvlYqlbrNFd2CXwNJwyXdKGmepIWSDpX0AUmPSLpb0rmSbkh1T5E0pWLbhZJGp+Wl6e8kSbdL+jmwoMr7gZLOlDRL0nxJn8nhtM3MzMxs7QwDbgO+33lFoVDYAPgd2ZxJOwM/A64rFApbVVS7AJgA7AN8lGwOpZN6Oqhb8GvzAeCpiPgwgKSRwEJgb+APwC/XYJ+7AGMi4glJkzq9bwOWRMQESesAMyXdAsTan4qZmZlZY2q0FvxSqXQFQKFQmFRl9eHAS8AJpVIpgIcKhcIHgc8AUwqFwptTnfeXSqUH0n6+AZxeKBS+WyqVunzAyQl+bRYAZ0k6A7gBeBl4IiIeA5B0BdC2mvt8ICKe6OL9vsAOkg5J70cCWwGPdrWz9KOgDWDq1Kl88uhPrWY4ZmZmZtaTQqEwChhVZdWLpVLpxdXY1S7A7Sm5L5tO1loPMI6scXdGp/UbA1sAf+xqx07waxARj0oaB3wIOB3orjV9JW/s+jSki3qvdPNewBci4ubKCuWuPl3EWASK5bceRcfMzMysT5wAfKtK+anAKauxn42BOZ3Knk3l5fXPd+pv/2zFOif4a0PSpsDzEXFF6kd/LLCFpC0j4o9k/aHKFgP7pe12JvuFtbpuBj4r6baIWCFpa+Ava3USZmZmZg1O9dFH5xyyh107e3E199PTyVRbX1N3bSf4tdkeOFPSKrInmD8LbAjcKOk54G6yp58BrgGOkDQXmEU33Wq6cQkwGpit7P/kZ4ED1yJ+MzMzM+sFqRvOi72wq7/xemt92UbAMxXr1y8UCgMrWvHL9Z+hG07wa5C6ytxcZdU2kI2KQ0rwI2IZWR/6avsZkf7eAdxRUd75/SrgxPSqtITXf0iYmZmZWeN6APhSp7K9gfvT8myyVvzdeT1P3JssuX+CbjjBNzMzM7OGUBcddFZDoVBYH9gMeGd6Pzategi4EjilUCicA0wF9gfeDXwKoFQqPV8oFH4OnFcoFI4BhgPfBS7obgQdcILfKzq3wNeDoYM8xUHePEV8fTjp0Sk9V7I+5e+CmbWw/YHLKt6XH6rdolQqLS4UCh8iG+v+s2QTXX2kVCo9VlH/uLR+Oq9PdHVaTwdVhIdWb0K+qGZmZtbbcm9A/+XdT+Se4xy62xa5fw49cQt+k/rh2XflHULL+sqXdwdgeYeHKs1TudXY1yE/vgb5890Ts9bU0t98ScdKOmI1t5ks6fzVqD+tYsIqMzMzM7M+1bIt+JIGRcTFecexJtLQmUqj7ZiZmZm1hPoYBr/+NWULvqRvSnpE0q2SfiFpSiq/Q9L3JN0JfFHSKZKmSNpU0tyKV4ekzTuVLZO0Z8Ux1pX0hKTB6f16khaX33eyj6S7JD0qqTwJ1kBJZ0qaJWm+pM+k8hGSpkuaLWmBpANS+WhJD0u6kGzYpLf37adoZmZmZo2o6VrwJY0HDgZ2Iju/2cCDFVVGRcSeqe4pABHxFDA2lX0O2DMi/lRR9h/A14B7SDPTRsTLku4APgz8Gvg4cE1ErKgS1mhgT2BL4HZJ7wSOAJZExARJ6wAzJd0CPAkcFBEvSdoQuE/S9Wk/BeCoiDhuzT8hMzMzs8bkBvzaNF2CD+wG/CZNOIWk33Za/8uuNpT0XrKxR3evKNsKOBPYOyJWdJoi+RKyxP/XwFHAp7vY9a9Sd5rHJD1ONkHWvsAOFf3zRwJbAf8HfE/SHsAq4K3AJqnOnyLivq5P3czMzMxaXTMm+D39uHul6kbSW4D/AfaPiKWpbDjwK+DTqZX/DSJiZuo6sycwMCIWdnHMzkM6RYrzC2mW3Mo4JpNNUzwu/aBYDAzpLva0XRvQBjB16lTgXV1VNTMzM7Mm1ox98O8G/kPSEEkjyLrQdCv1m/8V8F8R8WjFqsuAyyKiuzEnfwr8gjdOYtDZRyUNkLQl8A6gBNwMfLaiD//W6QfFSOCZlNzvBWzeU/wAEVGMiPERMb6tra2WTczMzMwaiqTcX42g6VrwI2JW6rM+D/gT0A4s6WGzXYEJwKmSTk1lBwCHAFtLOjqVfarKtleSTRv8i272XwLuJOtqc2xELJd0CVnf/NlpVJxngQPT/n4rqR2YCzzSQ+xmZmZmZv/UdAl+clZEnCJpGDAD+CFAREyqrBQRp1S8HcK/qnaHo51smuCy3YCrI+LFaoFExOQuylcBJ6ZXZxOrbQOM6aLczMzMrOk1SAN67po1wS9K2pYsab88Imb3xUEknQd8EPhQX+zfzMzMzGx1KaLz85/WBHxRzczMrLfl3n5+zb2Lc89xDp44OvfPoSfN2oJvZmZmZk2mUR5yzZsT/Cb16oqOvENoWcMGDwRgeceqnCNpbUMGZo/QLFvp70Jehg7ydyFv5e+BmbUWf/P7iKTFaSbatd3PtIrJsMzMzMxalurg1Qic4JuZmZmZNREn+L1A0nBJN0qaJ2mhpEMr1g2VdJOkT0uaLOn8inU3SJqUlpdKOi3t4z5Jm1Q5zndSi76vm5mZmZlV5USxd3wAeCoidoyIMcBNqXwE8Fvg5xHxkx72MRy4LyJ2JBu7/9OVKyX9ANgYOCqNoW9mZmbWUqT8X43ACX7vWADsI+kMSbtHRHnm3N8Al0XET2vYxz+AG9Lyg2Sz3JZ9ExgVEZ8Jj2tqZmZmZt1wgt8LIuJRYBxZon+6pJPTqpnAB/X6mE4reeNnXjl77oqK5L2DN45wNAsYJ2n9rmKQ1CapXVJ7sVhci7MxMzMzq0+Scn81Ag+T2QskbQo8HxFXSFoKTE6rTiZrfb8Q+CywGDgu9aF/K7BLjYe4CbgZuFHSvhHxcucKEVEEypl9eJhMMzMzs9bkFvzesT3wgKS5wEnAdyvWnQAMSX3oZwJPkLX0nwXMrvUAEXEV8BPgeklDeydsMzMzM2s2cpfupuQW/Bx5oqv64Imu8ueJrvLnia6sl+XeP+X6B/6ce+K6/y6b5f459MTffDMzMzOzJuI++E2q3Ips+XHLWX0otyJbfvxdMDPrX07wzczMzKwhNMggNrlzgt+knl36Wt4htKyNRqwDwA9PujnnSFrbV077dwBeWr4i50ha13pDBgPug58n3z0xa03+5udI0imSpqTlaZIOScuXSNo23+jMzMzM6kveY+B7HHxbYxHxqbxjMDMzM7PG5Bb8PiDpCEnzJc2T9DNJm0uansqmS9qsh+3vkDQ+Le8r6V5JsyVdJWlE/5yFmZmZmTUiJ/i9TNJ2ZJNd7R0ROwJfBM4HfhoROwBXAufWuK8NgW8A+0TEzkA78OU+CdzMzMyszkn5vxqBu+j0vr2BqyPiOYCIeF7SROAjaf3PgB/UuK/3ANsCM1OfrzcB9/ZuuGZmZmbWTNyC3/sE9DTLWq2zsAm4NSLGpte2EXFM1YpSm6R2Se3FYnF14jUzMzNrCKqD/xqBE/zeNx34mKQNACStD9wDfDytPxy4u8Z93Qe8V9I7076GSdq6WsWIKEbE+IgY39bWtlYnYGZmZmaNy110ellELJJ0GnCnpA5gDnA8cKmkrwLPAkfVuK9nJU0GfiFpnVT8DeDR3o/czMzMzJqBE/w+EBGXA5d3Kt67Sr1TKpYnVyxPqli+DZjQ2zGamZmZNZpGecg1b+6iY2ZmZmbWRBRR6/Oe1kB8Uc3MzKy35d5+ftOc/8s9x/nATm/L/XPoiVvwzczMzMyaiPvgN6m/vPBq3iG0rLe+eRgAy1auyjmS1jZ0UNZ+8fdX/pFzJK1rg+FvAmB5h78LeRky0O14Zq2o4b/5ko6VdESV8tGSFuYRU08kLc07BjMzM7NGMwDl/moEDd+CHxEX98Z+JA2MiI7e2JeZmZmZWV7qrgVf0nBJN0qaJ2mhpENT+WJJZ0h6IL3Kkz+dImlKWh6XtrsX+FzFPgdKOlPSLEnzJX0mlU+SdLuknwMLqsSyNB3zQUm/l7SLpDskPS5p/1RniKTLJC2QNEfSXql8sqRrJd0k6TFJP6iy/w0l3Svpw5I2knRNinGWpPdKGpC23SjVHyDpD5I27O3P3czMzKzeSfm/GkHdJfjAB4CnImLHiBgD3FSx7qWI2AU4HzinyraXAcdHxMRO5ccASyJiAtmY8p+WtEVatwtwUkRsW2V/w4E7ImIc8DLwXeD9wEHAt1OdzwFExPbAYcDlkoakdWOBQ4HtgUMlvb28Y0mbADcCJ0fEjcCPgR+lGA8GLomIVcAVZLPfAuwDzIuI56rEamZmZmZWlwn+AmCf1HK+e0QsqVj3i4q/b0jiJY0ERkXEnanoZxWr9wWOkDQXuB/YANgqrXsgIp7oIpZ/8PoPjAXAnRGxIi2PTuW7lY8VEY8AfwK2TuumR8SSiFgOPARsnsoHA9OBr0XEralsH+D8FOP1wHqS1gUuBcrPGBxN9iPGzMzMzKyqukvwI+JRYBxZEn26pJMrV3exDNnYrF2NjSrgCxExNr22iIhb0rpXuglnRbw+UcAq4LUU4ypef36hu5s1r1Usd1RssxJ4EPj3ivUDgIkVMb41Il6OiCeBv0naG3g38LuqJyi1SWqX1F4sFrsJyczMzKwx5d09x1101pCkTYFXI+IK4Cxg54rVh1b8vbdyu4h4EVgiabdUdHjF6puBz0oanI6xtaThvRTyjPKxJG0NbAaUetgmyFrjt5H09VR2C/D5cgVJYyvqX0LWVedXXT0IHBHFiBgfEePb2trW5DzMzMzMrAnU4yg62wNnSloFrAA+W7FuHUn3k/0wOazKtkcBl0p6lSypL7uErEvNbEkCngUO7KV4LwQulrSArGV+ckS8ph5+4kVEh6SPA7+V9BJwPHCBpPlk12UGcGyqfj1Z1xx3zzEzMzOzbun1Hij1TdJiYHwrPmAqaTzZA7i717hJeKKr/Hiiq/rgia7y54mu8ueJrqyX5d5BZfr8p3JPXN+3w6a5fw49qccWfKuQuvB8ljd2OTIzMzMzq6phWvBttfiimpmZWW/LveX6tjpowd+7AVrwfe/OzMzMzKyJuItOk3p1RdXBdqwfDBs8EIAffvnGnCNpbV85+8MAPPPy8pwjaV0br5vN+ec++PlxH3yz1uRvfg4kLU1/N5V0dd7xmJmZmTUCSbm/GoFb8HMUEU8Bh+Qdh5mZmZk1D7fg50jSaEkL0/IQSZdJWiBpjqS9UvlkSddKuknSY5J+kG/UZmZmZvnIexbbBmnAdwt+HfkcQERsL2kb4JY0My7AWGAn4DWgJOm8iHgynzDNzMzMrJ65Bb9+7Ab8DCAiHgH+BJQT/OkRsSQilgMPAZvnE6KZmZmZ1Tsn+PWju5s+r1Usd1DlzoukNkntktqLxWKvB2dmZmaWt7y757iLjq2uGWSz1d6WuuZsBpSAnWvZOCKKQDmzDw+TaWZmZtaanODXjwuBiyUtAFYCkyPitUYZjsnMzMysryn/yXQbghP8HETEiPR3MTAmLS8HJlepOw2YVvF+v34I0czMzMwalPvgm5mZmZk1EUVE3jFY7/NFNTMzs96We/+Yux76a+45zu7b/lvun0NP3IJvZmZmZtZE3Ae/SS36vxfzDqFlbfe2UQBMm/5YvoG0uMnv2wqAR55aknMkrWubTUcCsLxjVc6RtK4hA92OZ83Fg4/Uxt/8TiSNlrSwl/Y1WdL5vbGvtL9Jkm7orf2ZmZmZWfOpywRfku8sAJIG5h2DmZmZmTWWPkukJX2TbOKmJ4HngAcj4ixJWwIXABsBrwKfjohHJE0Dngd2AmZL2gBYBmwDbA4cBRwJTATuj4jJ6TgXAROAocDVEfGtVL4YuBz4D2Aw8NGIeKRTjNsBlwFvIvuxczCwAhgo6SfArsBfgAMiYlkPsS8HtgM2Ab4cEeWW9k0l3QRsCVwXEV9Lx94XOBVYB/gjcFRELE1xXwrsC5wv6UXgnPQZzl7d62BmZmbWLNxDpzZ90oIvaTxZsrwT8BFgfMXqIvCFiBgHTCGb4Klsa2CfiPhKev9mYG/gS8BvgR+RJdHbSxqb6pwUEeOBHYA9Je1Qsb/nImJn4KJ0rM6OBX4cEWNTjP+XyrcCLoiI7YAX07n0FPtoYE/gw2QTVg1J5WOBQ4HtgUMlvV3ShsA30rnuDLQDX67Y1/KI2A34NfATsh8puwP/VuUczMzMzMz+qa9a8HcDfhMRywAk/Tb9HUHWKn5VxUMS61Rsd1VEdFS8/21ERJrd9W8RsSDtZxFZQj0X+JiktnQubwG2Bean7a9Nfx8k+6HR2b3ASZLeBlwbEY+luJ6IiLkV246uIfZfRcQq4DFJj5PdeQCYHhFLUtwPkd2NGJXinJn29aYUS9kv099tUiyPpe2vANqqnIeZmZlZ0/NDtrXpqwS/q09/APBiajGv5pVO719Lf1dVLJffD5K0BVlL+oSIeCF1lRlSZfsOqpxrRPxc0v1kre43S/oU8HinY3WQdf/pKfbO47KW33fe1yCyz+fWiDisi31Vfg41jfeafuS0AUydOpX3fuhjtWxmZmZmZk2mrx6yvRv4D0lDUsv3hwEi4iXgCUkfBVBmx7U4znpkyfASSZsAH1ydjSW9A3g8Is4Frifr5lNVDbF/VNKA1E//HUCpm0PfB7xX0jvTvoZJ2rpKvUeALdI+Abr6QUBEFCNifESMb2tzI7+ZmZlZq+qTBD8iZpElzPPIusm0A+XBqA8HjpE0D1gEHLAWx5kHzEn7uRSYuZq7OBRYKGkuWXeYn/ZQv7vYS8CdwO+AYyNieTdxPwtMBn4haT5Zwr9NlXrLyVrlb5R0N/Cn2k7LzMzMrPmoDl6NQBF9M+OvpBFpVJhhwAygLSKachSY1DXohoi4Ou9YkvBEV/nxRFf1wRNd5c8TXeXPE11ZL8s9v7239EzfJK6rYWJh49w/h5705XjzRUnbkvWJv7xZk3szMzMzs3rSZy34litfVDMzM+ttubdc3/do/i3479m6/lvwfe/OzMzMzKyJ9GUXHcvRy6+tzDuElrXuOtnXauk/fA3yNOJN2XW4+p7F+QbSwg7ZdTTgPvh5ch98azYeB782/uabmZmZmTURJ/h1TtJiSRum5eMlPSzpyrzjMjMzM7P65C46jeU44IMR8UTegZiZmZn1N/fQqY1b8PuJpK9JOj4t/0jSbWn5fZKukHSYpAWSFko6o8r2F5PNkHu9pC/1b/RmZmZm1iic4PefGcDuaXk8MELSYGA34DHgDGBvYCwwQdKBlRtHxLHAU8BeEfGjforZzMzMrG6oDv5rBE7w+8+DwDhJ6wKvAfeSJfq7Ay8Cd0TEsxGxErgS2GN1di6pTVK7pPZisdi7kZuZmZlZw3Af/H4SESskLQaOAu4B5gN7AVsCfwbGreX+i0A5sw8Pk2lmZmbWmtyC379mAFPS37uAY4G5wH3AnpI2lDQQOAy4M68gzczMzOqRlP+rETjB7193AW8B7o2IvwHLgbsi4mngv4HbgXnA7Ij4TX5hmpmZmVmjchedfhQR04HBFe+3rlj+OfDzKtuMrrZsZmZm1mo8k21tFBF5x2C9zxfVzMzMelvu2fWDj/899xxn3Ds2yP1z6Im76JiZmZmZNRF30WlSyztW5R1CyxoyMPvd7GuQr/J1+OGF9+YcSev6ynETAX8X8lT+Hpg1C/fQqY2/+WZmZmZmTcQJfg0kLc07BgBJoyUtzDsOMzMzszzkPURmo9xBcIJfxyS5C5WZmZmZrRYn+KtJ0lclzZI0X9KpFeVflrQwvU5IZaMlPSLp8lT/aknD0rqT034WSioqjfsk6Q5J35N0J/BFSeMkzZN0L/C5HE7ZzMzMzBqIE/zVIGlfYCtgF2AsME7SHpLGAUcB7wbeA3xa0k5pswJQjIgdgJeA41L5+RExISLGAEOB/SoONSoi9oyIHwKXAcdHxMQ+Pj0zMzOzuqY6+K8ROMFfPfum1xxgNrANWcK/G3BdRLwSEUuBa4Hd0zZPRsTMtHxFqguwl6T7JS0A9ga2qzjOLwEkjSRL9u9M5T/rKjBJbZLaJbUXi8W1PU8zMzMza1Du4716BJweEVPfUJi65HSh84QMIWkIcCEwPiKelHQKMKSizisVx6tpQoeIKALlzD48LJ2ZmZk1m0Z5yDVvbsFfPTcDR0saASDprZI2BmYAB0oaJmk4cBBwV9pmM0nl7jWHAXfzejL/XNrXIdUOFhEvAksklVv9D+/tEzIzMzOz5uIW/NUQEbdIehdwb3omdinwnxExW9I04IFU9ZKImCNpNPAwcKSkqcBjwEUR8aqknwALgMXArG4OexRwqaRXyX5gmJmZmZl1SRE19QCxNZAS/BvSg7T9yV10cuSZbOuDZ7LNn2eyzZ9nsrVelnsHmfl/fiH3xHWHzd6c++fQE7fgNyn/o54/X4P6UE4yLT/+LpiZ9S8n+H0oIhYD/d16b2ZmZmY5KxQK04AjOxV/qVQqnVNR593ABWT54uPAV0ql0u/W9thO8JvUBb+Yk3cILetzh2VTILhbQr7cVSp/5WuwbKWvQV6GDvLdE2suDTiKzq+AL1a8f6m8UCgUNgB+RzYM+hHAAcB1hUJh+1Kp9NjaHNQJvpmZmZlZ31hWKpX+2sW6w8kS/hNKpVIADxUKhQ8CnwGmrM1B6zrB7+ohVUnfBmZExO9zCSyLYRTwiYi4MK8YzMzMzFpJPcwkWygURgGjqqx6sVQqvdipbP9CofAs8DTwC+DMUqm0Mq3bBbg9Jfdl04F91jbGhrx3FxEn55ncJ6OA43KOwczMzMz61wnAE1VeJ3Sq9/+ATwB7A2cDXwa+XbF+Y+CZTts8m8rXSiMk+AMl/UTSIkm3SBoqaZqkQwAkLZZ0hqQH0uudqXyapIsl3SXpUUn7pfIhki6TtEDSHEl7pfLJkn4j6SZJJUnfKgcg6cuSFqbXCan4+8CWkuZKOlPSCEnTJc1O+z4gbTta0iOSLknbXylpH0kzJT0maZdU7xRJUyqOuTBtO1zSjZLmpbJD++EzNzMzM7PqzgG2qPI6p7JSqVT6ValUuqlUKi0olUrTyBL84wuFQvk2RJ/djqjrLjrJVsBhEfFpSb8CDq5S56WI2EXSEWQf7n6pfDSwJ7AlcHtK/j8HEBHbS9oGuEXS1qn+LmRPMb8KzJJ0IxBkk029m+xC3C/pTuDrwJiIGAsgaRBwUES8JGlD4D5J16f9vhP4KNBGNqnVJ4DdgP2BE4EDuzn/DwBPRcSH03FG9viJmZmZmTWhenjINnXDeXENNp0NDAc2JGup/xv/2lq/Ef/aqr/aGqEF/4mImJuWHyRL2jv7RcXfykGvfxURqyLiMbKhh7YhS6x/BhARjwB/AsoJ/q0R8feIWAZcm+ruBlwXEa9ExNJUvnuVGAR8T9J84PfAW4FNKs5hQUSsAhYB0yObYWxBF+dTaQGwT7pLsXtELKlWSVKbpHZJ7cVisYddmpmZmVk/KzciP5fePwBM6lRnb+D+tT1QI7Tgv1ax3AEMrVInalguv+/ut9/q1q90ONmvrnERsULSYmBIWld5Dqsq3q/i9Wuwkjf+4BoCEBGPShoHfAg4XdItEVHZf4tUrwiUM/vwMJlmZmbWbAbUQxN+jQqFwtnAL8la6ncm64c/teKh2iuBUwqFwjnAVLKeHe8GPrW2x26EFvxaHFrxt3Je+o9KGiBpS+AdQAmYQZaMk7rmbJbKAd4vaX1JQ8m6zcxM9Q+UNEzScOAg4C7gZWDdimONBJ5Jyf1ewOareQ6LyS4+knYm68uFpE2BVyPiCuCsch0zMzMzq2vbAjeQ5ZlnkCXx/11eWSqV/k7WgLs7MBeYDHxkbcfAh8Zowa/FOpLuJ/vBclhFeQm4k6yrzLERsVzShcDFkhaQtZpPjojXlP0ivJus+847gZ9HRDtkD+yS3UYBuCQi5qTymZIWkk1ScAbwW0ntZBfpkdU8h2uAIyTNJeun/2gq3x44U9IqYAXw2dXcr5mZmZn1s1Kp9IEa6twHjOvtYyvrCt64UleY8RHxXKfyaWRj6F9d434mp/18vrdjzIG76OTIM9nWB89kmz/PZJs/z2RrvSz3/jGPPLUk98R1m01H5v459KRZWvCtk3KSafkpJzeWL1+H/DnJNDPrXw3fgm9V+aKamZlZb8u95br0dP4t+IW3uAXfcvLDC+7JO4SW9ZXP7Qq4a0je3EUnf6930enIOZLWNXTQwLxDMLMc+L6pmZmZmdn/b+/O4+SqyvyPf74kgWwQNCwvQKQBoRFCSCDhx5IJqzgIM+KwZBDQwPzMKIgoMguIYXHUcYQRF7aWJaAMIots+hMQEvZAks4ONLxkEYWRRRJISACT5/fHPSVlU52ukOqcWr5vXvXq2+eee+9T9ybhqVPPvaeJOMHvhaRpksaswfZTJB1Ry5jMzMzMWpHq4L9G4AS/xiT1+fehKvjamZmZmdl7tFSSKKlN0hOSrpI0T9INkgandQdImi1pvqQrJK1XYfuLJc2UtFDSOWXtz0qaLOkB4MgKhz5Q0v2SnpR0aNqmn6TvSpqRYvnn1D5U0t2SOlMsnyyL/fH0HP9OYMvanyEzMzOz+iXlfzWClkrwk3agIyJGAq8DJ0oaCEwBJkTEzhQ3H1eaUOprETEGGAnsI2lk2brlETEuIn5WYbs2YB/gEIpJtgYC/wQsjoixwFjgc5K2BpYDn4qIXYH9gPOlv/xxageujojREfHcGpwDMzMzM2tSrZjgPx8RD6blnwLjKBLnZyKiNHvsVcD4CtseJakTmA3sRDEFccl1qzjmzyNiZUQ8BTwN7AAcxLsz1z4CDAe2o3gE1bckzQN+A2xBMRMvwHMRMb3SASRNSt8uzOzo6FhFKGZmZmbWzFrxMZndn58aVPFc1zS6fhowNiJeSzPlDizrsvR9HPPkiLij23EmAhsDu0XEO2mm3tJxejxGRHQApcw+/JhMMzMzazZqlBqZzFpxBP/DkvZMy0cDDwBPAG2SPpLajwPu7bbdBhQJ9mJJmwIHr8Yxj5S0jqRtgW2ALuAO4AuSBgBI2l7SEGAY8FJK7vcDtlr9t2hmZmZmraoVR/AfBz4r6VLgKeDiiFgu6Xjgekn9gRnAJeUbRcRcSbOBhRRlNg9SvS6KDwybAp9Px7uMoja/M9XYvwwcBlwD3CZpJjCH4sOHmZmZmVlVWjHBXxkRn+/eGBF3A6MrtO9btjyx0g4joq2ng61im5XAGenV3Z4V2gBG9HQcMzMzs2bnCp3qKKJ7eXjzktQG3B4RzZ4ot85FNTMzs7Ule3r925feyJ7jbLvJ+tnPQ29aagQ/Ip7Fo+BmZmZmDck32VanpRL8VrL0nRW5Q2hZQwYUkxkvX7EycyStbWC/4hkCr735duZIWtcHBq8LwLI/+9+jXAb17/PJ1c2sDrXiU3TMzMzMzJqWR/DNzMzMrCG4QKc6HsGvkgo+X2ZmZmZW15ywroKkNkmPS7oI6AQulzRT0kJJ55T1GyvpIUlzJT0qaX1J/SSdJ2m+pHmSTk59D5A0O7VfIWm9XtqflXSOpM60bocc58LMzMwsN0nZX43ACX7v2oGrI2I08NWIGAOMBPaRNFLSusB1wCkRsQtwILAMmARsDYyOiJHANZIGAlOACRGxM0WJ1Bd6ai+L4ZWI2BW4GDitz9+xmZmZmTUsJ/i9ey4ipqfloyR1ArOBnYAdKT4AvBgRMwAi4vWI+DNFon9JWiYi/pT6PhMRT6b9XQWMX0V7yU3p5yyK2W/fQ9Kk9O3CzI6OjjV9z2ZmZmbWoHyTbe+WAkjammL0fGxEvCZpCjCQ4n6PSpMuVGrv6Xud3r7veSv9XEEP1ywiOoBSZh9+TKaZmZk1mwapkMnOI/jV24Ai2V8saVPg4NT+BLC5pLEAqf6+P3An8Pm0jKQPpr5tkj6Stj0OuHcV7WZmZmZmq8Uj+FWKiLmSZgMLgaeBB1P725ImAD+UNIii/v5A4DJge2CepHeAH0fEjyQdD1yfEv8ZFGU8b1VqX9vv0czMzKyeeQC/OoqoVF1iDc4lOhl5Jtv64Jls8/NMtvl5Jlursez59e9eXZo9cf3w8CHZz0NvXKJjZmZmZtZEXKLTpEqjyJZPaQTZ8iqNIls+HkU2s5rxXbZVcQZiZmZmZtZEPILfpL4/ZWbuEFrWKRPHAPCm74PIarDvhciu9C2W/y7kM9jf5lqT8fh9dZp6BF/SFElHVGjfXNINOWIyMzMzM+tLLTmCHxEvAO9J/M3MzMzMGl1TjeBL+oykeZLmSvpJah4v6SFJT5dG8yW1SVqQlidKulnSbZKekfRFSadKmi1pepqgCknTJF2Q9rVA0u6pfffUNjv9bC/b702Sfi3pKUn/ldr/XtKc9OqS9Exq303SvZJmSbpD0mapfdu0j1mS7pe0w1o9qWZmZmZ1Qsr/agRNk+BL2gn4GrB/ROwCnJJWbQaMAw4F/rOHzUcAnwZ2B74JvBkRo4GHgc+U9RsSEXsBJwJXpLYngPGp/2TgW2X9RwETgJ2BCZK2jIhbI2JURIwC5gLnSRoA/BA4IiJ2S/v+ZtpHB3Byaj8NuGi1ToyZmZmZtZRmKtHZH7ghIl4BiIg/qfiYdXNErAQek7RpD9tOjYg3gDckLQZuS+3zgZFl/a5N+75P0gaSNgTWB66StB0QwICy/ndHxGIASY8BWwHPp9//FVgWERdKGkHxIeOuFHM/4EVJQ4G9KGa4Le1zvUpvQNIkYBLApZdeCuvuusqTZWZmZtZoGmQAPbtmSvBFkWB391a3PpWU91lZ9vtK/vocdd9/AN+g+IDwKUltwLQe9ruitC9JBwBHAuPL4loYEXuW71zSBsCiNNq/ShHRQTHaDxB+io6ZmZlZa2qaEh3gbuAoScMBSrXzNTYh7XscsDiNzg8D/pDWT+xtB5K2oiizOSoilqXmLmBjSXumPgMk7RQRrwPPSDoytUvSLrV8Q2ZmZmbWXJpmBD8iFkr6JnCvpBXA7D44zGuSHgI2AE5Ibf9FUaJzKnBPFfuYCAwHfpHKbl6IiE+kG4B/IGkYxXW5AFgIHANcLOlMivKfn1HU7puZmZm1lka5yzUzRVSqarHuJE0DTouIRqh9cYlORp7oqj54oqv8PNFVfp7oymose3b9h0XLsieuW2w4KPt56E0zleiYmZmZmbU8j+A3J19UMzMzq7XsI9cv1MEI/uYewTczMzMzs7WpaW6ytb+2ePk7uUNoWcMGFlMhvP7WnzNH0to2WK/45801+PmUavBnPf1q5kha127bDM8dgllN+R7b6rTkCL6kNkkLelh3rqQD13ZMZmZmZma10Koj+KKHDzcRMXktxwKApH4R4UdNmJmZmdkaaZkR/DRq/7iki4BOYJCkH0taKOlOSYNSvynpmfRIelbSOZI6Jc2XtENq/6CkmyXNkzRd0sjUPlTSlanvPEmHp/aDJD2c9nO9pKFl+58s6QHgSElHp20XSPpO6tMvxbQgrfvKWj95ZmZmZnVBdfCqfy2T4CftwNXAaGBL4MKI2AlYBBzewzavRMSuwMXAaantHGB2RIwEzkj7BPg6xQy3O6d190jaCDgTODDtZyZwatn+l0fEOOA+4DvA/sAoYKykw9LyFhExIiJ2Bq5cozNgZmZmZk2t1RL85yJielp+JiLmpOVZQFsP29xUoc844CcAEXEPMDzNQHsgcGFpw4h4DdgD2BF4UNIc4LPAVmX7vy79HAtMi4iXI+LPwDXAeOBpYBtJP5T0t8DrlYKUNEnSTEkzOzo6VnUOzMzMzBqSlP/VCFqtBn9p2fJbZcsrgEE9bPNWWZ/S+ap0eSO1d38+q4C7IuLoXmKq+EcmIl6TtAvwceAk4CjghAr9OoBSZh9+io6ZmZlZa2q1EfxauQ84BkDSvhRlPK8DdwJfLHWS9AFgOrC3pI+ktsGStq+wz0eAfSRtJKkfcDRwbyrxWScibqQoAdq1z96VmZmZmTW8VhvBr5WzgSslzQPepCi7AfgP4ML0CM4VwDkRcZOkicC1ktZL/c4EnizfYUS8KOl0YCrFaP6vIuKWNHp/paTSh7HT+/B9mZmZmdWtBqmQyU4R2Wf8tdpziU5GnuiqPniiq/w80VV+nujKaix7fv3H15dnT1w33WBg9vPQG4/gm5mZmVlDaJSbXHPzCH5z8kU1MzOzWsueXr/0Rv4R/E3Wr/8RfN9ka2ZmZmbWRFyi06Qe6nopdwgta6/2TQDXfudWqv/2dcindA2m3P1U5kha18QDtssdglmN1f3geV3wCH5Gkj4v6TNpeYqkI9LyNElj8kZnZmZmZo3II/iZSOofEZfkjsPMzMysUfgm2+o4wV9Dkr5OMenV88ArwCzgN8AlwGDgt8AJaUbaacBDwN7ArZLWB5ZExHmr2P/FwFiKmXZviIiz+vDtmJmZmVmDc4nOGkhlNIcDo4F/AEplNVcD/xYRI4H5QHlSvmFE7BMR51d5mK9FxBhgJMVMtyNrE72ZmZmZNSMn+GtmHHBLRCyLiDeA24AhFEn8vanPVcD4sm2uW81jHCWpE5gN7ATsWKmTpEmSZkqa2dHRsZqHMDMzM6t/qoNXI3CJzpp5P9d5adU7l7YGTgPGphKfKcDASn0jogMoZfbhp+iYmZmZtSaP4K+ZB4C/kzRQ0lDgEIoE/jVJf5P6HAfc29MOerFB2t9iSZsCB69pwGZmZmYNK/fwfYMM4XsEfw1ExAxJtwJzgeeAmcBi4LPAJZIGA08Dx7/P/c+VNBtYmPbzYE0CNzMzM7Om5QR/zZ0XEWenZP4+4PyImAPs0b1jROzb7fezy5YnVupX3m5mZmZm1hsn+GuuQ9KOFLXxV0VEZ+6AzMzMzJqRGqVGJjNFRO4YrPZ8Uc3MzKzWsmfXry59O3uOM3zIutnPQ298k62ZmZmZWRNxiU6TWvD8otwhtKwRW24IwCtL3sobSIvbaOh6ACxfsTJzJK1rYL9iDOnCa2dnjqR1nXT06NwhmNWU6n7svD54BL8Xkp6VtNH73HZamu22FnF8Od3Ia2ZmZmbWIyf4jePLgBN8MzMza1m5H4HfKF8gOMEvI+lYSY9KmiPpUkn9qlkvaYmk8yV1Srpb0sZlmx2ZtnmyNPmVpDZJ96f+nZL2Su37plH/GyQ9IekaFb4EbA5MlTR1LZ0OMzMzM2tATvATSR8FJgB7R8QoYAVwTJXrhwCdEbErxay1Z5Xtun9E7E4xAl9qfwn4WOo/AfhBWf/Rqe+OwDbpeD8AXgD2i4j9avOOzczMzKwZ+Sbbdx0A7AbMUHEHxyCKRLya9SuB69LyT4GbyrYrLc8C2tLyAOBHkkZRfFDYvqz/oxHxewBJc9I2D/QWvKRJwCSASy+9lL0OPqq3TczMzMwai++yrYoT/HeJYqKq0/+qUZq4qvU9KH9Ga+lRKit493x/BfgjsAvFtyjLK/Tvvs2qDxjRAXSUfvVTdMzMzMxak0t03nU3cISkTQAkfVDSVlWuXwc4Ii1/mt5H3IcBL0bESuA4oF8v/QHeANav6p2YmZmZNaHcN9g2yvcHHsFPIuIxSWcCd0paB3gHOKmK9c8BS4GdJM0CFlPU1a/KRcCNko4Epqbte9MB/D9JL7oO38zMzMx6oojsM/42PElLImJo7jjKuEQnI090VR880VV+nugqP090ZTWWfQB70bJ3sieuGw4akP089MYj+GZmZmbWEHyPbXU8gt+cfFHNzMys1rKn14uX5x/BHzbQI/hmZmZmZjVR95l1nXCC36SWvL0idwgta+i6xUORXPudV6n+29chn9I1OH/yXZkjaV1fPfdjuUMwswxa4jGZks6VdOD73HaUpE/UOqYqjrtkbR/TzMzMzBpfS4zgR8TkNdh8FDAG+FVtojEzMzOz98V32ValqUbwJX1d0hOS7pJ0raTTUvsUSUek5cmSZkhaIKlDKv6kSJom6TuSHpX0pKS/kbQucC4wQdIcSRO6HW+ipFsk/VpSl6SzytbdLGmWpIWSJpW1L5H0TUlzJU2XtGlq31rSwym2b5T1HyrpbkmdkuZL+mRfnkMzMzMza2xNk+BLGgMcDowG/oFi1L2SH0XE2IgYAQwCDi1b1z8idge+DJwVEW8Dk4HrImJURFxXYX+7A8dQjPQfmeIAOCEidktxfEnS8NQ+BJgeEbsA9wGfS+3fBy6OiLHA/5btfznwqYjYFdgPOL/0ocTMzMysleSexbZRErCmSfCBccAtEbEsIt4Abuuh336SHpE0H9gf2Kls3U3p5yygrcrj3hURr0bEsrT9uNT+JUlzgenAlsB2qf1t4PYKx9kbuDYt/6Rs/wK+JWke8BtgC2DT7kFImiRppqSZHR0dVYZuZmZmZs2mmWrwe/1QJWkgcBEwJiKel3Q2MLCsS2nq0RVUf266P481JO0LHAjsGRFvSppWdpx34t3JB7ofp9KzXY8BNgZ2i4h3JD3bLeZiw4gOoJTZh5+iY2ZmZtaammkE/wHg7yQNlDQUOKRCn1Ji/Erqc0QV+30DWH8V6z8m6YOSBgGHAQ8Cw4DXUnK/A7BHFcd5EPjHtHxMWfsw4KWU3O8HbFXFvszMzMyajpT/1QiaJsGPiBnArcBcilKZmcDibn0WAT8G5gM3AzOq2PVUYMdKN9kmD1CU1MwBboyImcCvgf6prOYbFGU6vTkFOEnSDIqkvuQaYIykmRSJ/xNV7MvMzMzMWpTerRZpfJKGRsQSSYMpbmCdFBGdfXi8iRTlPl/sq2O8Ty7RycgTXdUHT3SVnye6ys8TXVmNZR+/XvL2n7MnrkPX7Z/9PPSmmWrwATok7UhRinNVXyb3ZmZmZra21X1uXReaagTf/sIX1czMzGote3a95O0V2XOcoev2y34eetNsI/hmZmZm1qQa5SbX3JzgNynXHefj2u/6ULoOvh8ln9L9KOefcUfmSFrXV7/18dwhmFkGTfMUnXokaUNJJ6blzSXdkDsmMzMzM2tuTvD71obAiQAR8UJEVPPc/V5J6leL/ZiZmZk1EtXBqxE4we9b/wlsm56hf72kBQCSBkv6uaR5kq6T9IikMWndQZIeltSZthma2p+VNFnSA8CR+d6SmZmZmdUz1+D3rX8HRkTEKEltwO2p/USKmW5HShpBMUkWkjYCzgQOjIilkv4NOBU4N223PCLGrc03YGZmZlYvfJNtdTyCn8c44GcAEbEAmJfa9wB2BB6UNAf4LLBV2XbX9bRDSZMkzZQ0s6Ojo0+CNjMzM7P65xH8PHr6/Cngrog4uof1S3vaYUR0AKXMPvwEFzMzM7PW5BH8vvUGsH6F9geAowDSzLs7p/bpwN6SPpLWDZa0/doI1MzMzKz+5b7FtjFqhDyC34ci4lVJD6abax8vW3URcJWkecBsihKdxRHxsqSJwLWS1kt9zwSeXJtxm5mZmVlttLe3nw6cTPF0xTuBSV1dXS/15TGd4PexiPh0heblwLERsVzStsDdwHOp/z3A2Ar7aevLOM3MzMzqXaPdZNve3n48cAbwGeAZ4ALgWuCAvjyuE/w8BgNTJQ2g+K7nCxHxduaYzMzMzKy2Tgb+u6ur6xcA7e3tJwC/bW9vH9HV1bWgrw6qiOirfVs+vqhmZmZWa9nHz5evWJk9x9llx49+gKLcprtFXV1di0q/tLe3rwe8Cezf1dV1b1n7M8B/dHV1Xd5XMXoEvzll/wu4piRNSk8Gskx8DfLzNagPvg75+RrUh3q4DgP7rVMPOc7ZwFkV2s9J60qGUzzQpnu9/cvAJn0RWImfomP1alLuAMzXoA74GtQHX4f8fA3qg69D4QJg6wqvC7r1y/ZhxCP4ZmZmZmZVSmU4i6ro+gqwkmK0vvxpihvz3lH9mvIIvpmZmZlZjXV1db0FzAX2K7W1t7dvDbQBj/TlsT2Cb/XKtZb5+Rrk52tQH3wd8vM1qA++DqvvR8AF7e3ts4Fnge8BU/vyCTrgp+iYmZmZmfWZNNHVl4BhwF0UE139sS+P6QTfzMzMzKyJuAbfzMzMzKyJOMG3uiHpCkkvSerTujTrmaQtJU2V9LikhZJOyR1TK5I0UNKjkuam63BO7phalaR+kmZLuj13LK1K0rOS5kuaI2lm7nhakaQNJd0g6Yn0/4c9c8dkq+YSHasbksYDS4CrI2JE7nhakaTNgM0iolPS+sAs4LCIeCxzaC1FkoAhEbFE0gDgAeCUiJieObSWI+lUYAywQUQcmjueViTpWWBMRLySO5ZWJekq4P6IuEzSusDgiFiUOSxbBY/gW92IiPuAP+WOo5VFxIsR0ZmW36B4bu8WeaNqPVFYkn4dkF4ejVnLJH0IOAS4LHcsZrlI2gAYD1wOEBFvO7mvf07wzawiSW3AaPr4Wb1WWSoNmUMxGcpdEeHrsPZdAPwrxUQ1lk8Ad0qaJckzqa592wAvA1emcrXLJA3JHZStmhN8M3sPSUOBG4EvR8TrueNpRRGxIiJGAR8CdpfksrW1SNKhwEsRMSt3LMbeEbErcDBwUirntLWnP7ArcHFEjAaWAv+eNyTrjRN8M/srqeb7RuCaiLgpdzytLn0VPg3427yRtJy9gb9P9d8/A/aX9NO8IbWmiHgh/XwJ+AWwe96IWs7vgd+XfYt4A0XCb3XMCb6Z/UW6ufNy4PGI+O/c8bQqSRtL2jAtDwIOBJ7IGlSLiYjTI+JDEdEG/CNwT0QcmzmsliNpSLrhn1QWchDgJ62tRRHxv8DzktpT0wGAH7xQ5/rnDsCsRNK1wL7ARpJ+D5wVEZfnjarl7A0cB8xP9d8AZ0TEr/KF1JI2A66S1I9iIObnEeHHNFor2hT4RTH2QH/gfyLi13lDakknA9ekJ+g8DRyfOR7rhR+TaWZmZmbWRFyiY2ZmZmbWRJzgm5mZmZk1ESf4ZmZmZmZNxAm+mZmZmVkTcYJvZmZmZtZEnOCbmTUASd+WtK+kwyRVnEVS0tmS/iBpTtlrwz6O64y+3L+Zma0+J/hmZo3h/wCPAPsA96+i3/ciYlTZa1FfBKPCOoATfDOzOuME38ysjkn6rqR5wFjgYeD/AhdLmrwa+zhV0hVpeWdJCyQNTiP+P5F0j6SnJH2ubJt/kTRD0jxJ56S2NkmPS7oI6KSY9XhQ+qbgmjTr6C8lzU3HmFDDU2FmZlXyTLZmZnUsIv5F0vUUMwyfCkyLiL1XsclXJB2bll+LiP2AC4Bpkj4FfA3454h4M80OOhLYAxgCzJb0S2AEsB2wOyDgVknjgd8B7cDxEXEigKQjI2JUWj4ceCEiDkm/D6vRaTAzs9XgBN/MrP6NBuYAOwCP9dL3exFxXnlDRKyUNBGYB1waEQ+Wrb4lIpYByyRNpUjqxwEHAbNTn6EUCf/vgOciYnoPx54PnCfpO8DtEbGqUiIzM+sjTvDNzOqUpFHAFOBDwCvA4KJZc4A9U2Jere2AJcDm3dqjwu8Cvh0Rl3aLpw1Y2tMBIuJJSbsBnwC+LenOiDh3NWI0M7MacA2+mVmdiog5qfzlSWBH4B7g4+nm2aqT+1Qq831gPDBc0hFlqz8paaCk4cC+wAzgDuAESUPT9ltI2qSH3b8jaUDqtznwZkT8FDgP2LX6d2tmZrXiEXwzszomaWOKWvqVknaIiN5KdMpr8AEOAyYDF6UR9n8Cpkq6L61/FPgl8GHgGxHxAvCCpI8CD6c6/SXAscCKCsfrAOZJ6gSuBr4raSXwDvCF9/GWzcxsDSmi+7ezZmbWCiSdDSzpXrNvZmaNzSU6ZmZmZmZNxCP4ZmZmZmZNxCP4ZmZmZmZNxAm+mZmZmVkTcYJvZmZmZtZEnOCbmZmZmTURJ/hmZmZmZk3k/wNNBRqL7EfSDwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x864 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 39;\n",
       "                var nbb_unformatted_code = \"import matplotlib.pyplot as plt\\nimport seaborn as sns\\n\\n# print(y_axis_labels)\\ny_axis_labels = sorted_cols\\nfor idx, label in enumerate(y_axis_labels):\\n    y_axis_labels[idx] = label.replace(\\\"+\\\", \\\" \\\")\\n#     print(idx, label)\\n\\n\\n\\nx_axis_labels = [1, 2, 3, 4, 5, 6]\\n# for i in range(1, 201):\\n#     y_axis_labels.append(i)\\n\\nfig, ax = plt.subplots(figsize=(12, 12))\\nsns.set(font_scale=1.2)\\np = sns.heatmap(\\n    expert_gt[order],\\n    xticklabels=x_axis_labels,\\n    yticklabels=y_axis_labels,\\n    linewidths=0.1,\\n    cmap=\\\"BuPu\\\",\\n)\\np.set_xlabel(\\\"# Experts\\\")\\np.set_ylabel(\\\"Class Labels\\\")\\n# p.set_yticklabels(p.get_yticklabels(), rotation=10)\\nplt.show()\";\n",
       "                var nbb_formatted_code = \"import matplotlib.pyplot as plt\\nimport seaborn as sns\\n\\n# print(y_axis_labels)\\ny_axis_labels = sorted_cols\\nfor idx, label in enumerate(y_axis_labels):\\n    y_axis_labels[idx] = label.replace(\\\"+\\\", \\\" \\\")\\n#     print(idx, label)\\n\\n\\nx_axis_labels = [1, 2, 3, 4, 5, 6]\\n# for i in range(1, 201):\\n#     y_axis_labels.append(i)\\n\\nfig, ax = plt.subplots(figsize=(12, 12))\\nsns.set(font_scale=1.2)\\np = sns.heatmap(\\n    expert_gt[order],\\n    xticklabels=x_axis_labels,\\n    yticklabels=y_axis_labels,\\n    linewidths=0.1,\\n    cmap=\\\"BuPu\\\",\\n)\\np.set_xlabel(\\\"# Experts\\\")\\np.set_ylabel(\\\"Class Labels\\\")\\n# p.set_yticklabels(p.get_yticklabels(), rotation=10)\\nplt.show()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# print(y_axis_labels)\n",
    "y_axis_labels = sorted_cols\n",
    "for idx, label in enumerate(y_axis_labels):\n",
    "    y_axis_labels[idx] = label.replace(\"+\", \" \")\n",
    "#     print(idx, label)\n",
    "\n",
    "\n",
    "x_axis_labels = [1, 2, 3, 4, 5, 6]\n",
    "# for i in range(1, 201):\n",
    "#     y_axis_labels.append(i)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(12, 12))\n",
    "sns.set(font_scale=1.2)\n",
    "p = sns.heatmap(\n",
    "    expert_gt[order],\n",
    "    xticklabels=x_axis_labels,\n",
    "    yticklabels=y_axis_labels,\n",
    "    linewidths=0.1,\n",
    "    cmap=\"BuPu\",\n",
    ")\n",
    "p.set_xlabel(\"# Experts\")\n",
    "p.set_ylabel(\"Class Labels\")\n",
    "# p.set_yticklabels(p.get_yticklabels(), rotation=10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.8434)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1)/test_tensor_y_1.size(0)\";\n",
       "                var nbb_formatted_code = \"torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1) / test_tensor_y_1.size(\\n    0\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(test_tensor_preds_1.argmax(dim=1) == test_tensor_y_1) / test_tensor_y_1.size(\n",
    "    0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python_3_7",
   "language": "python",
   "name": "python_3_7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}