src/codebase/iPython/Cumulative_performance/ISIC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0d2e0138",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\\n\\nimport os\\nimport sys\\n\\nsys.path.append(\\n    os.path.abspath(\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/codebase\\\")\\n)\\nimport torch.backends.cudnn as cudnn\\nimport random\\nimport time\\nfrom collections import OrderedDict\\nimport pickle\\nimport numpy as np\\nimport pandas as pd\\nimport torch\\nimport torch.nn.functional as F\\nfrom torch.utils.data import DataLoader\\nfrom tqdm import tqdm\\nfrom torch.nn.functional import one_hot\\nimport sklearn.metrics as metrics\\nimport utils\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss\\nfrom Explainer.models.Gated_Logic_Net import Gated_Logic_Net\\nfrom Explainer.models.explainer import Explainer\\nfrom Explainer.models.pi import Pi\\nfrom dataset.dataset_cubs import Dataset_cub_for_explainer\\nfrom dataset.utils_dataset import get_dataset_with_image_and_attributes\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss, Selective_Distillation_Loss\\nfrom Explainer.models.concepts import Conceptizator\\n\\nimport matplotlib.pyplot as plt\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\\n\\nimport os\\nimport sys\\n\\nsys.path.append(\\n    os.path.abspath(\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/codebase\\\")\\n)\\nimport torch.backends.cudnn as cudnn\\nimport random\\nimport time\\nfrom collections import OrderedDict\\nimport pickle\\nimport numpy as np\\nimport pandas as pd\\nimport torch\\nimport torch.nn.functional as F\\nfrom torch.utils.data import DataLoader\\nfrom tqdm import tqdm\\nfrom torch.nn.functional import one_hot\\nimport sklearn.metrics as metrics\\nimport utils\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss\\nfrom Explainer.models.Gated_Logic_Net import Gated_Logic_Net\\nfrom Explainer.models.explainer import Explainer\\nfrom Explainer.models.pi import Pi\\nfrom dataset.dataset_cubs import Dataset_cub_for_explainer\\nfrom dataset.utils_dataset import get_dataset_with_image_and_attributes\\nfrom Explainer.loss_F import loss_fn_kd, entropy_loss, Selective_Distillation_Loss\\nfrom Explainer.models.concepts import Conceptizator\\n\\nimport matplotlib.pyplot as plt\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "sys.path.append(\n",
    "    os.path.abspath(\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/codebase\")\n",
    ")\n",
    "import torch.backends.cudnn as cudnn\n",
    "import random\n",
    "import time\n",
    "from collections import OrderedDict\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "from torch.nn.functional import one_hot\n",
    "import sklearn.metrics as metrics\n",
    "import utils\n",
    "from Explainer.loss_F import loss_fn_kd, entropy_loss\n",
    "from Explainer.models.Gated_Logic_Net import Gated_Logic_Net\n",
    "from Explainer.models.explainer import Explainer\n",
    "from Explainer.models.pi import Pi\n",
    "from dataset.dataset_cubs import Dataset_cub_for_explainer\n",
    "from dataset.utils_dataset import get_dataset_with_image_and_attributes\n",
    "from Explainer.loss_F import loss_fn_kd, entropy_loss, Selective_Distillation_Loss\n",
    "from Explainer.models.concepts import Conceptizator\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "234ad112",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"device = utils.get_device()\\nprint(f\\\"Device: {device}\\\")\";\n",
       "                var nbb_formatted_code = \"device = utils.get_device()\\nprint(f\\\"Device: {device}\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "device = utils.get_device()\n",
    "print(f\"Device: {device}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "77d02ffb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.01\n",
      "0.2\n",
      "\n",
      "\n",
      " << Val sizes >>\n",
      "torch.Size([121, 2])\n",
      "torch.Size([121, 2])\n",
      "torch.Size([121])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"iter1\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[0])\\nprint(args.cov[0])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter1\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n\\n# val\\nval_tensor_preds_1 = torch.load(\\n    os.path.join(\\n        root, experiment, iteration, expert_type, output, \\\"val_tensor_preds.pt\\\"\\n    )\\n)\\n\\nval_tensor_preds_bb_1 = torch.load(\\n    os.path.join(\\n        root, experiment, iteration, expert_type, output, \\\"val_tensor_preds_bb.pt\\\"\\n    )\\n)\\n\\nval_tensor_y_1 = torch.load(\\n    os.path.join(root, experiment, iteration, expert_type, output, \\\"val_tensor_y.pt\\\")\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_1.size())\\nprint(val_tensor_preds_bb_1.size())\\nprint(val_tensor_y_1.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"iter1\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[0])\\nprint(args.cov[0])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter1\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n\\n# val\\nval_tensor_preds_1 = torch.load(\\n    os.path.join(\\n        root, experiment, iteration, expert_type, output, \\\"val_tensor_preds.pt\\\"\\n    )\\n)\\n\\nval_tensor_preds_bb_1 = torch.load(\\n    os.path.join(\\n        root, experiment, iteration, expert_type, output, \\\"val_tensor_preds_bb.pt\\\"\\n    )\\n)\\n\\nval_tensor_y_1 = torch.load(\\n    os.path.join(root, experiment, iteration, expert_type, output, \\\"val_tensor_y.pt\\\")\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_1.size())\\nprint(val_tensor_preds_bb_1.size())\\nprint(val_tensor_y_1.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.01\n",
    "base_cov = 0.2\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\",\n",
    "        \"iter1\",\n",
    "        \"explainer/accuracy\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[0])\n",
    "print(args.cov[0])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\"\n",
    "experiment = f\"explainer/{experiment_folder}\"\n",
    "iteration = \"iter1\"\n",
    "expert_type = \"explainer/accuracy\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "\n",
    "# val\n",
    "val_tensor_preds_1 = torch.load(\n",
    "    os.path.join(\n",
    "        root, experiment, iteration, expert_type, output, \"val_tensor_preds.pt\"\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_preds_bb_1 = torch.load(\n",
    "    os.path.join(\n",
    "        root, experiment, iteration, expert_type, output, \"val_tensor_preds_bb.pt\"\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_y_1 = torch.load(\n",
    "    os.path.join(root, experiment, iteration, expert_type, output, \"val_tensor_y.pt\")\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Val sizes >>\")\n",
    "print(val_tensor_preds_1.size())\n",
    "print(val_tensor_preds_bb_1.size())\n",
    "print(val_tensor_y_1.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9ad9ac0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.01\n",
      "0.2\n",
      "\n",
      "\n",
      " << Val sizes >>\n",
      "torch.Size([77, 2])\n",
      "torch.Size([77, 2])\n",
      "torch.Size([77])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter2\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter2\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_2 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_2 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_2 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_2.size())\\nprint(val_tensor_preds_bb_2.size())\\nprint(val_tensor_y_2.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter2\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter2\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_2 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_2 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_2 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_2.size())\\nprint(val_tensor_preds_bb_2.size())\\nprint(val_tensor_y_2.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.01\n",
    "base_cov = 0.2\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\",\n",
    "        \"cov_0.2\",\n",
    "        \"iter2\",\n",
    "        \"explainer/accuracy\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\"\n",
    "experiment = f\"explainer/{experiment_folder}\"\n",
    "iteration = \"iter2\"\n",
    "expert_type = \"explainer/accuracy\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "# val\n",
    "val_tensor_preds_2 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_preds_bb_2 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_y_2 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Val sizes >>\")\n",
    "print(val_tensor_preds_2.size())\n",
    "print(val_tensor_preds_bb_2.size())\n",
    "print(val_tensor_y_2.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e46e177d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.01\n",
      "0.2\n",
      "\n",
      "\n",
      " << Val sizes >>\n",
      "torch.Size([108, 2])\n",
      "torch.Size([108, 2])\n",
      "torch.Size([108])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter3\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter3\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_3 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_3 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_3 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_3.size())\\nprint(val_tensor_preds_bb_3.size())\\nprint(val_tensor_y_3.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter3\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter3\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_3 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_3 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_3 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_3.size())\\nprint(val_tensor_preds_bb_3.size())\\nprint(val_tensor_y_3.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.01\n",
    "base_cov = 0.2\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\",\n",
    "        \"cov_0.2\",\n",
    "        \"iter3\",\n",
    "        \"explainer/accuracy\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\"\n",
    "experiment = f\"explainer/{experiment_folder}\"\n",
    "iteration = \"iter3\"\n",
    "expert_type = \"explainer/accuracy\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "# val\n",
    "val_tensor_preds_3 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_preds_bb_3 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_y_3 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Val sizes >>\")\n",
    "print(val_tensor_preds_3.size())\n",
    "print(val_tensor_preds_bb_3.size())\n",
    "print(val_tensor_y_3.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5da897a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.01\n",
      "0.2\n",
      "\n",
      "\n",
      " << Val sizes >>\n",
      "torch.Size([33, 2])\n",
      "torch.Size([33, 2])\n",
      "torch.Size([33])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter4\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter4\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_4.size())\\nprint(val_tensor_preds_bb_4.size())\\nprint(val_tensor_y_4.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter4\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter4\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_4 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_4.size())\\nprint(val_tensor_preds_bb_4.size())\\nprint(val_tensor_y_4.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.01\n",
    "base_cov = 0.2\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\",\n",
    "        \"cov_0.2\",\n",
    "        \"iter4\",\n",
    "        \"explainer/accuracy\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\"\n",
    "experiment = f\"explainer/{experiment_folder}\"\n",
    "iteration = \"iter4\"\n",
    "expert_type = \"explainer/accuracy\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "# val\n",
    "val_tensor_preds_4 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_preds_bb_4 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_y_4 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Val sizes >>\")\n",
    "print(val_tensor_preds_4.size())\n",
    "print(val_tensor_preds_bb_4.size())\n",
    "print(val_tensor_y_4.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "415e43bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.01\n",
      "0.2\n",
      "\n",
      "\n",
      " << Val sizes >>\n",
      "torch.Size([116, 2])\n",
      "torch.Size([116, 2])\n",
      "torch.Size([116])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter5\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter5\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_5 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_5 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_5 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_5.size())\\nprint(val_tensor_preds_bb_5.size())\\nprint(val_tensor_y_5.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter5\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter5\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_5 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_5 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_5 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_5.size())\\nprint(val_tensor_preds_bb_5.size())\\nprint(val_tensor_y_5.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.01\n",
    "base_cov = 0.2\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\",\n",
    "        \"cov_0.2\",\n",
    "        \"iter5\",\n",
    "        \"explainer/accuracy\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\"\n",
    "experiment = f\"explainer/{experiment_folder}\"\n",
    "iteration = \"iter5\"\n",
    "expert_type = \"explainer/accuracy\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "# val\n",
    "val_tensor_preds_5 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_preds_bb_5 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_y_5 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Val sizes >>\")\n",
    "print(val_tensor_preds_5.size())\n",
    "print(val_tensor_preds_bb_5.size())\n",
    "print(val_tensor_y_5.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "eac65b00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "########################\n",
      "0.01\n",
      "0.2\n",
      "\n",
      "\n",
      " << Val sizes >>\n",
      "torch.Size([16, 2])\n",
      "torch.Size([16, 2])\n",
      "torch.Size([16])\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter6\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter6\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_6 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_6 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_6 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_6.size())\\nprint(val_tensor_preds_bb_6.size())\\nprint(val_tensor_y_6.size())\";\n",
       "                var nbb_formatted_code = \"base_lr = 0.01\\nbase_cov = 0.2\\npickle_in = open(\\n    os.path.join(\\n        f\\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\\\",\\n        \\\"cov_0.2\\\",\\n        \\\"iter6\\\",\\n        \\\"explainer/accuracy\\\",\\n        \\\"test_explainer_configs.pkl\\\",\\n    ),\\n    \\\"rb\\\",\\n)\\nargs = pickle.load(pickle_in)\\nn_classes = len(args.labels)\\nx_to_bool = 0.5\\ntop_k_explanations = 50\\nconcept_names = args.concept_names\\nprint(\\\"########################\\\")\\nprint(args.lr[-1])\\nprint(args.cov[-1])\\n\\nuse_concepts_as_pi_input = True\\nexplainer_init = \\\"none\\\"\\n\\nexperiment_folder = (\\n    f\\\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\\\"\\n    f\\\"_input-size-pi_{args.input_size_pi}\\\"\\n    f\\\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\\\"\\n    f\\\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\\\"\\n    f\\\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\\\"\\n)\\n\\nroot = \\\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\\\"\\nexperiment = f\\\"explainer/{experiment_folder}\\\"\\niteration = \\\"iter6\\\"\\nexpert_type = \\\"explainer/accuracy\\\"\\noutput = \\\"g_outputs\\\"\\n\\n# val\\nval_tensor_preds_6 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds.pt\\\",\\n    )\\n)\\n\\nval_tensor_preds_bb_6 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_preds_bb.pt\\\",\\n    )\\n)\\n\\nval_tensor_y_6 = torch.load(\\n    os.path.join(\\n        root,\\n        experiment,\\n        f\\\"cov_{args.cov[-1]}\\\",\\n        iteration,\\n        expert_type,\\n        output,\\n        \\\"val_tensor_y.pt\\\",\\n    )\\n)\\n\\n\\nprint(\\\"\\\\n\\\\n << Val sizes >>\\\")\\nprint(val_tensor_preds_6.size())\\nprint(val_tensor_preds_bb_6.size())\\nprint(val_tensor_y_6.size())\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_lr = 0.01\n",
    "base_cov = 0.2\n",
    "pickle_in = open(\n",
    "    os.path.join(\n",
    "        f\"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC/explainer/lr_{base_lr}_epochs_500_temperature-lens_0.7_input-size-pi_2048_cov_{base_cov}_alpha_0.5_selection-threshold_0.5_lambda-lens_0.0001_alpha-KD_0.9_temperature-KD_10.0_hidden-layers_1\",\n",
    "        \"cov_0.2\",\n",
    "        \"iter6\",\n",
    "        \"explainer/accuracy\",\n",
    "        \"test_explainer_configs.pkl\",\n",
    "    ),\n",
    "    \"rb\",\n",
    ")\n",
    "args = pickle.load(pickle_in)\n",
    "n_classes = len(args.labels)\n",
    "x_to_bool = 0.5\n",
    "top_k_explanations = 50\n",
    "concept_names = args.concept_names\n",
    "print(\"########################\")\n",
    "print(args.lr[-1])\n",
    "print(args.cov[-1])\n",
    "\n",
    "use_concepts_as_pi_input = True\n",
    "explainer_init = \"none\"\n",
    "\n",
    "experiment_folder = (\n",
    "    f\"lr_{args.lr[0]}_epochs_{args.epochs}_temperature-lens_{args.temperature_lens}\"\n",
    "    f\"_input-size-pi_{args.input_size_pi}\"\n",
    "    f\"_cov_{args.cov[0]}_alpha_{args.alpha}_selection-threshold_{args.selection_threshold}\"\n",
    "    f\"_lambda-lens_{args.lambda_lens}_alpha-KD_{args.alpha_KD}\"\n",
    "    f\"_temperature-KD_{float(args.temperature_KD)}_hidden-layers_{len(args.hidden_nodes)}\"\n",
    ")\n",
    "\n",
    "root = \"/ocean/projects/asc170022p/shg121/PhD/ICLR-2022/out/SIIM-ISIC\"\n",
    "experiment = f\"explainer/{experiment_folder}\"\n",
    "iteration = \"iter6\"\n",
    "expert_type = \"explainer/accuracy\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "# val\n",
    "val_tensor_preds_6 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_preds_bb_6 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_preds_bb.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "val_tensor_y_6 = torch.load(\n",
    "    os.path.join(\n",
    "        root,\n",
    "        experiment,\n",
    "        f\"cov_{args.cov[-1]}\",\n",
    "        iteration,\n",
    "        expert_type,\n",
    "        output,\n",
    "        \"val_tensor_y.pt\",\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "print(\"\\n\\n << Val sizes >>\")\n",
    "print(val_tensor_preds_6.size())\n",
    "print(val_tensor_preds_bb_6.size())\n",
    "print(val_tensor_y_6.size())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5ea3c068",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"expert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\npreds = torch.cat(\\n    (\\n        val_tensor_preds_1, val_tensor_preds_2, val_tensor_preds_3, \\n        val_tensor_preds_4, val_tensor_preds_5, val_tensor_preds_6\\n    ), dim=0\\n)\\ngt = torch.cat(\\n    (\\n        val_tensor_y_1, val_tensor_y_2, val_tensor_y_3 ,val_tensor_y_4,\\n        val_tensor_y_5, val_tensor_y_6\\n    ), dim=0\\n)\\nbb = torch.cat(\\n    (\\n        val_tensor_preds_bb_1.cpu(), val_tensor_preds_bb_2.cpu(), val_tensor_preds_bb_3.cpu(),\\n        val_tensor_preds_bb_4.cpu(), val_tensor_preds_bb_5.cpu(), val_tensor_preds_bb_6.cpu()\\n    ),dim=0\\n)\";\n",
       "                var nbb_formatted_code = \"expert_type = \\\"explainer\\\"\\noutput = \\\"g_outputs\\\"\\n\\npreds = torch.cat(\\n    (\\n        val_tensor_preds_1,\\n        val_tensor_preds_2,\\n        val_tensor_preds_3,\\n        val_tensor_preds_4,\\n        val_tensor_preds_5,\\n        val_tensor_preds_6,\\n    ),\\n    dim=0,\\n)\\ngt = torch.cat(\\n    (\\n        val_tensor_y_1,\\n        val_tensor_y_2,\\n        val_tensor_y_3,\\n        val_tensor_y_4,\\n        val_tensor_y_5,\\n        val_tensor_y_6,\\n    ),\\n    dim=0,\\n)\\nbb = torch.cat(\\n    (\\n        val_tensor_preds_bb_1.cpu(),\\n        val_tensor_preds_bb_2.cpu(),\\n        val_tensor_preds_bb_3.cpu(),\\n        val_tensor_preds_bb_4.cpu(),\\n        val_tensor_preds_bb_5.cpu(),\\n        val_tensor_preds_bb_6.cpu(),\\n    ),\\n    dim=0,\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "expert_type = \"explainer\"\n",
    "output = \"g_outputs\"\n",
    "\n",
    "preds = torch.cat(\n",
    "    (\n",
    "        val_tensor_preds_1, val_tensor_preds_2, val_tensor_preds_3, \n",
    "        val_tensor_preds_4, val_tensor_preds_5, val_tensor_preds_6\n",
    "    ), dim=0\n",
    ")\n",
    "gt = torch.cat(\n",
    "    (\n",
    "        val_tensor_y_1, val_tensor_y_2, val_tensor_y_3 ,val_tensor_y_4,\n",
    "        val_tensor_y_5, val_tensor_y_6\n",
    "    ), dim=0\n",
    ")\n",
    "bb = torch.cat(\n",
    "    (\n",
    "        val_tensor_preds_bb_1.cpu(), val_tensor_preds_bb_2.cpu(), val_tensor_preds_bb_3.cpu(),\n",
    "        val_tensor_preds_bb_4.cpu(), val_tensor_preds_bb_5.cpu(), val_tensor_preds_bb_6.cpu()\n",
    "    ),dim=0\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "40fe06d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "G Auroc: 0.8677986658580958\n",
      "BB Auroc: 0.8800099233695353\n",
      "g_tot_acc: 83.65180467091295 (%)\n",
      "bb_tot_acc: 84.50106157112526 (%)\n",
      "g_tot_f1: 83.65180467091295 (%)\n",
      "g_tot_precision: 83.65180467091295 (%)\n",
      "g_tot_recall: 83.65180467091295 (%)\n",
      "bb_tot_recall: 84.50106157112526 (%)\n",
      "bb_tot_f1: 84.50106157112526 (%)\n",
      "total samples covered by g: 471 (out of 500)\n",
      "total coverage by g: 0.942\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"preds_tot_np = preds.argmax(dim=1).numpy()\\ngt_tot_np = gt.numpy()\\nbb_tot_np = bb.argmax(dim=1).numpy()\\nbb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\\ng_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\\ng_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\nbb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\nbb_tot_f1 = metrics.f1_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\ng_tot_cov = preds.size(0) / 500\\n\\nproba = torch.nn.Softmax(dim=1)(preds)[:, 1]\\ntest_auroc, test_aurpc = utils.compute_AUC(gt, pred=proba)\\nproba_bb = torch.nn.Softmax(dim=1)(bb)[:, 1]\\ntest_auroc_bb, test_aurpc_bb = utils.compute_AUC(gt, pred=proba_bb)\\n\\nprint(f\\\"G Auroc: {test_auroc}\\\")\\nprint(f\\\"BB Auroc: {test_auroc_bb}\\\")\\nprint(f\\\"g_tot_acc: {g_tot_acc * 100} (%)\\\")\\nprint(f\\\"bb_tot_acc: {bb_tot_acc * 100} (%)\\\")\\nprint(f\\\"g_tot_f1: {g_tot_f1 * 100} (%)\\\")\\nprint(f\\\"g_tot_precision: {g_tot_precision * 100} (%)\\\")\\nprint(f\\\"g_tot_recall: {g_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_recall: {bb_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_f1: {bb_tot_f1 * 100} (%)\\\")\\nprint(f\\\"total samples covered by g: {preds.size(0)} (out of {500})\\\")\\nprint(f\\\"total coverage by g: {g_tot_cov}\\\")\";\n",
       "                var nbb_formatted_code = \"preds_tot_np = preds.argmax(dim=1).numpy()\\ngt_tot_np = gt.numpy()\\nbb_tot_np = bb.argmax(dim=1).numpy()\\nbb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\\ng_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\\ng_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\ng_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\\\"micro\\\")\\nbb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\nbb_tot_f1 = metrics.f1_score(gt_tot_np, bb_tot_np, average=\\\"micro\\\")\\ng_tot_cov = preds.size(0) / 500\\n\\nproba = torch.nn.Softmax(dim=1)(preds)[:, 1]\\ntest_auroc, test_aurpc = utils.compute_AUC(gt, pred=proba)\\nproba_bb = torch.nn.Softmax(dim=1)(bb)[:, 1]\\ntest_auroc_bb, test_aurpc_bb = utils.compute_AUC(gt, pred=proba_bb)\\n\\nprint(f\\\"G Auroc: {test_auroc}\\\")\\nprint(f\\\"BB Auroc: {test_auroc_bb}\\\")\\nprint(f\\\"g_tot_acc: {g_tot_acc * 100} (%)\\\")\\nprint(f\\\"bb_tot_acc: {bb_tot_acc * 100} (%)\\\")\\nprint(f\\\"g_tot_f1: {g_tot_f1 * 100} (%)\\\")\\nprint(f\\\"g_tot_precision: {g_tot_precision * 100} (%)\\\")\\nprint(f\\\"g_tot_recall: {g_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_recall: {bb_tot_recall * 100} (%)\\\")\\nprint(f\\\"bb_tot_f1: {bb_tot_f1 * 100} (%)\\\")\\nprint(f\\\"total samples covered by g: {preds.size(0)} (out of {500})\\\")\\nprint(f\\\"total coverage by g: {g_tot_cov}\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds_tot_np = preds.argmax(dim=1).numpy()\n",
    "gt_tot_np = gt.numpy()\n",
    "bb_tot_np = bb.argmax(dim=1).numpy()\n",
    "bb_tot_acc = metrics.accuracy_score(gt_tot_np, bb_tot_np)\n",
    "g_tot_acc = metrics.accuracy_score(gt_tot_np, preds_tot_np)\n",
    "g_tot_f1 = metrics.f1_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "g_tot_precision = metrics.precision_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "g_tot_recall = metrics.recall_score(gt_tot_np, preds_tot_np, average=\"micro\")\n",
    "bb_tot_recall = metrics.recall_score(gt_tot_np, bb_tot_np, average=\"micro\")\n",
    "bb_tot_f1 = metrics.f1_score(gt_tot_np, bb_tot_np, average=\"micro\")\n",
    "g_tot_cov = preds.size(0) / 500\n",
    "\n",
    "proba = torch.nn.Softmax(dim=1)(preds)[:, 1]\n",
    "test_auroc, test_aurpc = utils.compute_AUC(gt, pred=proba)\n",
    "proba_bb = torch.nn.Softmax(dim=1)(bb)[:, 1]\n",
    "test_auroc_bb, test_aurpc_bb = utils.compute_AUC(gt, pred=proba_bb)\n",
    "\n",
    "print(f\"G Auroc: {test_auroc}\")\n",
    "print(f\"BB Auroc: {test_auroc_bb}\")\n",
    "print(f\"g_tot_acc: {g_tot_acc * 100} (%)\")\n",
    "print(f\"bb_tot_acc: {bb_tot_acc * 100} (%)\")\n",
    "print(f\"g_tot_f1: {g_tot_f1 * 100} (%)\")\n",
    "print(f\"g_tot_precision: {g_tot_precision * 100} (%)\")\n",
    "print(f\"g_tot_recall: {g_tot_recall * 100} (%)\")\n",
    "print(f\"bb_tot_recall: {bb_tot_recall * 100} (%)\")\n",
    "print(f\"bb_tot_f1: {bb_tot_f1 * 100} (%)\")\n",
    "print(f\"total samples covered by g: {preds.size(0)} (out of {500})\")\n",
    "print(f\"total coverage by g: {g_tot_cov}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc91b383",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "290fd134",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37499ed1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03f0bf37",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python_3_7_rtx_6000",
   "language": "python",
   "name": "python_3_7_rtx_6000"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}