electra_fine_tune_interpret_captum_ig.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Copy of electra_fine_tune_interpret_captum_ig.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2fa165296c664aa2be133523b2553a39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0cc26044da8484b989d235fd8b5c2ff",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4559b4cba09641498757c2824a3bf71c",
              "IPY_MODEL_7d85c7a5d9d74583a47c8f642e5d4e7c"
            ]
          }
        },
        "b0cc26044da8484b989d235fd8b5c2ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4559b4cba09641498757c2824a3bf71c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0fc01cdc8579498898b204530c0bdd8b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b6286be7b9a948e7aab4ef4cf0f767ec"
          }
        },
        "7d85c7a5d9d74583a47c8f642e5d4e7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1234383d81604073aeaf2b7c0d1c02f6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466/466 [00:02&lt;00:00, 185B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c37621beb7454486b64b13f3bdd4f6e6"
          }
        },
        "0fc01cdc8579498898b204530c0bdd8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b6286be7b9a948e7aab4ef4cf0f767ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1234383d81604073aeaf2b7c0d1c02f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c37621beb7454486b64b13f3bdd4f6e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e2a3011f09024d049ee9b932385202c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_11db628685bd4b1887da0fd99ca8857b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_228756b95d5d436a89e20db08dfe277c",
              "IPY_MODEL_52e02012ccce4188b3467a40197fa5ac"
            ]
          }
        },
        "11db628685bd4b1887da0fd99ca8857b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "228756b95d5d436a89e20db08dfe277c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8c26f27649df44239fef5ea03bf30c1b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 54245363,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 54245363,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_77b27ea9183c4012a065c088cf011a4f"
          }
        },
        "52e02012ccce4188b3467a40197fa5ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_22662dd134744a918f7f367d796ce035",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 54.2M/54.2M [00:01&lt;00:00, 28.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0887573c76a44c2294c964382de6880f"
          }
        },
        "8c26f27649df44239fef5ea03bf30c1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "77b27ea9183c4012a065c088cf011a4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "22662dd134744a918f7f367d796ce035": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0887573c76a44c2294c964382de6880f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5d97b31441c4bd190472d41bc1ff690": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_44fd65bdc89e407982a3f6424f07f5ba",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b1736eaabccd4ca081af80f6eb777605",
              "IPY_MODEL_ef183506d505472a8df3661b0497d545"
            ]
          }
        },
        "44fd65bdc89e407982a3f6424f07f5ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1736eaabccd4ca081af80f6eb777605": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d6ab129643c044eca57b4661d151a7f4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc228450bdd14d2f9d43c37b3525850c"
          }
        },
        "ef183506d505472a8df3661b0497d545": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fd6a7b1e6aaa419690de3cf28249bb0a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:09&lt;00:00, 24.6kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_286f78363ccf4e78b1b402bb0ece8276"
          }
        },
        "d6ab129643c044eca57b4661d151a7f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dc228450bdd14d2f9d43c37b3525850c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd6a7b1e6aaa419690de3cf28249bb0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "286f78363ccf4e78b1b402bb0ece8276": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cae2ed62500b4e628c0b2240c557e8b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_be2576ab0cb34e7ba183a5ed737dd68f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cede0a2d8a314d3eb5190ecba5064f1b",
              "IPY_MODEL_0dd9dec9cebc478ab6d9057fef42bdff"
            ]
          }
        },
        "be2576ab0cb34e7ba183a5ed737dd68f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cede0a2d8a314d3eb5190ecba5064f1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d0db1f55615b401fb03707209cd0d6fe",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 29068,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 29068,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42825a57bcb44d669eb67f4e6139f32f"
          }
        },
        "0dd9dec9cebc478ab6d9057fef42bdff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f3fba90c44b44ee586608204651326ef",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29.1k/29.1k [00:00&lt;00:00, 30.5kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c5dd6acd7e0c47d9a67ee8fa79d67339"
          }
        },
        "d0db1f55615b401fb03707209cd0d6fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42825a57bcb44d669eb67f4e6139f32f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3fba90c44b44ee586608204651326ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c5dd6acd7e0c47d9a67ee8fa79d67339": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9c7bd45d6ed9440f80992b72b8f8c194": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_beeb36fe601c4d4bbf2cf0c822fcd807",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bbc5ef5f7bf743d49ad8023d99a7f6e1",
              "IPY_MODEL_6a97c789f90b463f92a7ca9d4613b9de"
            ]
          }
        },
        "beeb36fe601c4d4bbf2cf0c822fcd807": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbc5ef5f7bf743d49ad8023d99a7f6e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_011eff21098e4378855bad72eba06cb4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 30329,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 30329,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_91767051562c41bcbb1a412369d9cefe"
          }
        },
        "6a97c789f90b463f92a7ca9d4613b9de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_faa5e5e552fb4204b2b7dd794cdfda86",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 30.3k/30.3k [00:00&lt;00:00, 287kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aea8d41be30d4acdb33b063485c03593"
          }
        },
        "011eff21098e4378855bad72eba06cb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "91767051562c41bcbb1a412369d9cefe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "faa5e5e552fb4204b2b7dd794cdfda86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aea8d41be30d4acdb33b063485c03593": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "527000b2ef9f4ed8ac41e313f12ed943": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_41d355c455304b928370280499510c04",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b2f3f33409074b4ca78666bac15a63c7",
              "IPY_MODEL_7e0f7ba49ed741bfa31b0cb347b42f09"
            ]
          }
        },
        "41d355c455304b928370280499510c04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2f3f33409074b4ca78666bac15a63c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bd1318125543430e8db6ac8bf51d7afa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 7439277,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 7439277,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c42a7f5e46db4ab5837b341cc3ea3341"
          }
        },
        "7e0f7ba49ed741bfa31b0cb347b42f09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aed7770446bc47f7b9d8a08bfeb4098e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 7.44M/7.44M [00:00&lt;00:00, 18.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c8e9b7928b74fddb570d0591fd1f462"
          }
        },
        "bd1318125543430e8db6ac8bf51d7afa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c42a7f5e46db4ab5837b341cc3ea3341": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aed7770446bc47f7b9d8a08bfeb4098e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c8e9b7928b74fddb570d0591fd1f462": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fdc91e89d7884641954455e1b28a827e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_62198255f0d9462d92b9cfa15caed712",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1e164ec812584fe8ad655ebc2f162a60",
              "IPY_MODEL_b05c3b1ca054451b8358be45d2438231"
            ]
          }
        },
        "62198255f0d9462d92b9cfa15caed712": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e164ec812584fe8ad655ebc2f162a60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_79cfc2a506ed4207b04b0627737b77f9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_491e70d6964e462193e1e111f28df688"
          }
        },
        "b05c3b1ca054451b8358be45d2438231": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5a533b02ee8d4555b298b258c0687202",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 67349/0 [00:01&lt;00:00, 42905.70 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e2d4197e70294676affa725e5a38e818"
          }
        },
        "79cfc2a506ed4207b04b0627737b77f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "491e70d6964e462193e1e111f28df688": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a533b02ee8d4555b298b258c0687202": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e2d4197e70294676affa725e5a38e818": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6d710654817041a0a7694d62b829c6b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_deecf8705c7b4d9385d373286e7dc64d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6a9d432d0f534c528d58cd279845d3aa",
              "IPY_MODEL_1bcbd81da66a4f6fb9fdbdfb39f9e187"
            ]
          }
        },
        "deecf8705c7b4d9385d373286e7dc64d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6a9d432d0f534c528d58cd279845d3aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0dc71f120f1240318c3d40b5a034e0c2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1e23fd906ea94721b68087d8daf85b38"
          }
        },
        "1bcbd81da66a4f6fb9fdbdfb39f9e187": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe51d57d46b1417886cf40bedbcb24d5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 872/0 [00:00&lt;00:00, 14753.48 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_576ec8e88c7d490b893b0ec02b71c150"
          }
        },
        "0dc71f120f1240318c3d40b5a034e0c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1e23fd906ea94721b68087d8daf85b38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe51d57d46b1417886cf40bedbcb24d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "576ec8e88c7d490b893b0ec02b71c150": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "099914e405c64179a44e6605b68b6023": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6374c92441a943e1acb7153413f45567",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_faca957637334fd0895598db90f74ff0",
              "IPY_MODEL_41cb5b0ac5a0451fa62d83b5270e6c8d"
            ]
          }
        },
        "6374c92441a943e1acb7153413f45567": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "faca957637334fd0895598db90f74ff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3a8cedda9c434680bc71ce33afbbeb0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_20acab854f32489b977bccdb7dd66bde"
          }
        },
        "41cb5b0ac5a0451fa62d83b5270e6c8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9a7e57eab34c444db5b6b96929a90fd3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1821/0 [00:00&lt;00:00, 21614.56 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08246b9dd2614c40a9048ab795951bd1"
          }
        },
        "3a8cedda9c434680bc71ce33afbbeb0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "20acab854f32489b977bccdb7dd66bde": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a7e57eab34c444db5b6b96929a90fd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08246b9dd2614c40a9048ab795951bd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "515e6a8c59ad442dae9f3e4bdbaffd4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d06e2fc8fe4a42138b93583bf0807e2f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4878d398a15249d7926b605d924cc3fb",
              "IPY_MODEL_27180224a0044f1e9171a57ad2a420a6"
            ]
          }
        },
        "d06e2fc8fe4a42138b93583bf0807e2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4878d398a15249d7926b605d924cc3fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_19355e93f085467e83888ea59f82678a",
            "_dom_classes": [],
            "description": "Epoch: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3b6075b83e414f0684c8ca15d770a463"
          }
        },
        "27180224a0044f1e9171a57ad2a420a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b61cac8151984edcb9108bdfbcaaa38b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [15:57&lt;00:00, 319.30s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3d69e37f0f044dcbba5411f2d7b5dea1"
          }
        },
        "19355e93f085467e83888ea59f82678a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3b6075b83e414f0684c8ca15d770a463": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b61cac8151984edcb9108bdfbcaaa38b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3d69e37f0f044dcbba5411f2d7b5dea1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a7d4104c50a4485820ec3e6b757e4e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b13ce1f6400343db917f6d96ab3586de",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9abefd88f3bf471bbca170931764c1cc",
              "IPY_MODEL_2657dce6830c4eaaa5164be6b2a53904"
            ]
          }
        },
        "b13ce1f6400343db917f6d96ab3586de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9abefd88f3bf471bbca170931764c1cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1838094d3aad4a43bdac85f8d3d0345e",
            "_dom_classes": [],
            "description": "Iteration: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2104,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2104,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_620e2b2480204591a0989d45370c8d52"
          }
        },
        "2657dce6830c4eaaa5164be6b2a53904": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e0ab87123ffb4b018efecc36c00e665a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2104/2104 [15:57&lt;00:00,  2.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_547143a6e0c340cda9fbedebb465ac9d"
          }
        },
        "1838094d3aad4a43bdac85f8d3d0345e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "620e2b2480204591a0989d45370c8d52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0ab87123ffb4b018efecc36c00e665a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "547143a6e0c340cda9fbedebb465ac9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "17e4e106b3c5402eb6c028c28525700a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2f4913c6857b4ee0ace51a4a88a63012",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4f8af6fc0b294ae8b8169a84020402f5",
              "IPY_MODEL_a1888f9d76694ad0a0d9df7b7677107b"
            ]
          }
        },
        "2f4913c6857b4ee0ace51a4a88a63012": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f8af6fc0b294ae8b8169a84020402f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_af7ce86aaaac46acb13cc65537b25571",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1eeff683391d42ee95b5328968787028"
          }
        },
        "a1888f9d76694ad0a0d9df7b7677107b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d2216ed153cc4ea59540fff4d7bc8061",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:01&lt;00:00, 79.67it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52bfd2f135224341b44119c97a6bbd52"
          }
        },
        "af7ce86aaaac46acb13cc65537b25571": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1eeff683391d42ee95b5328968787028": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d2216ed153cc4ea59540fff4d7bc8061": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52bfd2f135224341b44119c97a6bbd52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1305437e0af94ac8b77153547f95486d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1e2c4074816848c2a9d4e3132b34e1c1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2a483a8cbd454f098327ea75b8bd530c",
              "IPY_MODEL_44ec186472694ecbb300d3521998e190"
            ]
          }
        },
        "1e2c4074816848c2a9d4e3132b34e1c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a483a8cbd454f098327ea75b8bd530c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b973157c678048f7848387e037ebfa66",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f2551355d1b4acd9f49e901a35e81e0"
          }
        },
        "44ec186472694ecbb300d3521998e190": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_baadcd7ef9794660bc831449a78d45a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:01&lt;00:00, 78.81it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8945b7975ef04bd887153fa3ccb82383"
          }
        },
        "b973157c678048f7848387e037ebfa66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f2551355d1b4acd9f49e901a35e81e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "baadcd7ef9794660bc831449a78d45a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8945b7975ef04bd887153fa3ccb82383": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3142c36e00cd4c299159559e91f4474e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a5ef7ab6899442089b05f5eb6ccdb240",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_28095903c54b44e8b255d41474eca0f4",
              "IPY_MODEL_57085d2c040a42c6b4e4c6229216ca9c"
            ]
          }
        },
        "a5ef7ab6899442089b05f5eb6ccdb240": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "28095903c54b44e8b255d41474eca0f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc61beb8cd6d43f9908fe35b93f26768",
            "_dom_classes": [],
            "description": "Iteration: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2104,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2104,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_63e5bb476e8f4a9ab327c40607d570fe"
          }
        },
        "57085d2c040a42c6b4e4c6229216ca9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ab6f9568b1b443e89f510b62cf6aa555",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2104/2104 [12:59&lt;00:00,  2.70it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_63f76a44c096428b838bbe6c83a93bf1"
          }
        },
        "dc61beb8cd6d43f9908fe35b93f26768": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "63e5bb476e8f4a9ab327c40607d570fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab6f9568b1b443e89f510b62cf6aa555": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "63f76a44c096428b838bbe6c83a93bf1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc365a89f45747db83b0e275ba5b4bdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e075b266745a48e98913b4e2017ad3c8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_201f9ddf25864392a762991d50083164",
              "IPY_MODEL_f8169e04316a4eed8e1e04482f42c973"
            ]
          }
        },
        "e075b266745a48e98913b4e2017ad3c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "201f9ddf25864392a762991d50083164": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7e6c7098abf84264bbbb5d986262fca9",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9c4f1c99ebc4367878f65fb64398fc3"
          }
        },
        "f8169e04316a4eed8e1e04482f42c973": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1be470549574436ba6f5dc97590dd1fd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:01&lt;00:00, 80.55it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_af14d8633d924be0895a97be52f7ce15"
          }
        },
        "7e6c7098abf84264bbbb5d986262fca9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9c4f1c99ebc4367878f65fb64398fc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1be470549574436ba6f5dc97590dd1fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "af14d8633d924be0895a97be52f7ce15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d1b01439fc34f9bafdb1cc512902e0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f5f0dc0196f14bd48e6209a995b656e8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e82326872c5848e6a6d01efc0e3b912e",
              "IPY_MODEL_ee8aeb5aab3f4dea816414d63cece5d9"
            ]
          }
        },
        "f5f0dc0196f14bd48e6209a995b656e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e82326872c5848e6a6d01efc0e3b912e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_85dc2499531346e0b9f45696f4780f39",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0db240967fd643af95169010b64fcdee"
          }
        },
        "ee8aeb5aab3f4dea816414d63cece5d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f4523ee835e04ed7a38bc242242ebc21",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:01&lt;00:00, 79.09it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e4cd34b3624b49238654b10af98ba457"
          }
        },
        "85dc2499531346e0b9f45696f4780f39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0db240967fd643af95169010b64fcdee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f4523ee835e04ed7a38bc242242ebc21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e4cd34b3624b49238654b10af98ba457": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1b92e9e37aea4c38828ce06fa8be7608": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_65188d29b3d1453f8dde9f7c9c66f763",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5d4099d823cf4b958c6ff25815d53250",
              "IPY_MODEL_59870b85248246e2b55cd282b7ad7e51"
            ]
          }
        },
        "65188d29b3d1453f8dde9f7c9c66f763": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d4099d823cf4b958c6ff25815d53250": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_31ce28582b274de394e98f16397b4a75",
            "_dom_classes": [],
            "description": "Iteration: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2104,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2104,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c0a492c6fa964723ae90ca8f9b517fc2"
          }
        },
        "59870b85248246e2b55cd282b7ad7e51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d71bc4ffabe343f1a8e39f90b54129b9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2104/2104 [10:00&lt;00:00,  3.51it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8a6b070bcc434cd6ab44cebaf69a1efb"
          }
        },
        "31ce28582b274de394e98f16397b4a75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c0a492c6fa964723ae90ca8f9b517fc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d71bc4ffabe343f1a8e39f90b54129b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8a6b070bcc434cd6ab44cebaf69a1efb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37016f426f244bfb8ebb9db490604665": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c7b36421f6614a6f953709b0db204e49",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6438b1a59efa4fea895f6548aefdc97d",
              "IPY_MODEL_c250c8ad3b1a4b13b8316f2bfce05561"
            ]
          }
        },
        "c7b36421f6614a6f953709b0db204e49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6438b1a59efa4fea895f6548aefdc97d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_33bfdcd711044097a82c561fe8ab6d2e",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fe5c32e7f5cb4ae6825011c59bc8b6d5"
          }
        },
        "c250c8ad3b1a4b13b8316f2bfce05561": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dc5d5323067549b5b097e98dce676b65",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:01&lt;00:00, 79.22it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4f2ff614cecc463d806b8195588bfa40"
          }
        },
        "33bfdcd711044097a82c561fe8ab6d2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fe5c32e7f5cb4ae6825011c59bc8b6d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc5d5323067549b5b097e98dce676b65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4f2ff614cecc463d806b8195588bfa40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "130553b53eb24e59a84ecc730bfbe4f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_68899e22a5f04e028d7434815d77255b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cba118ce90554967b59d9600fc762065",
              "IPY_MODEL_59781d9ab0a7416291254d1bd258bc5a"
            ]
          }
        },
        "68899e22a5f04e028d7434815d77255b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cba118ce90554967b59d9600fc762065": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5c66861f32614460a054a278bb247e3e",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cec0e1a60bdd48ab876763ff23e2070c"
          }
        },
        "59781d9ab0a7416291254d1bd258bc5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8291258e0f364fd7bfd1b871503127a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:22&lt;00:00,  4.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_40a4912051b34dc0aa5892760f545548"
          }
        },
        "5c66861f32614460a054a278bb247e3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cec0e1a60bdd48ab876763ff23e2070c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8291258e0f364fd7bfd1b871503127a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "40a4912051b34dc0aa5892760f545548": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2adf1fd0a79d40948fbeb90711b46944": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d62df1fc3ab347c7bb5575282a92ed69",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_956846094c2f4b8a9e5556dbefb31ff0",
              "IPY_MODEL_87d93832eff244ccb9415c0bff4a5582"
            ]
          }
        },
        "d62df1fc3ab347c7bb5575282a92ed69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "956846094c2f4b8a9e5556dbefb31ff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ab5129556c3f409ca25d82a7a60fe674",
            "_dom_classes": [],
            "description": "Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3d51ef24a3ce41ecb068d098e570b98c"
          }
        },
        "87d93832eff244ccb9415c0bff4a5582": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_18705bfe3aed46f0a2f6abbc80ea6ea6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [02:12&lt;00:00,  1.22s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d57209b086e84388b77da65aaa556ed3"
          }
        },
        "ab5129556c3f409ca25d82a7a60fe674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3d51ef24a3ce41ecb068d098e570b98c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "18705bfe3aed46f0a2f6abbc80ea6ea6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d57209b086e84388b77da65aaa556ed3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5cadbc0eee10468cadb03d67882649a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_aabd5e96eee646fbbcb1750f63393605",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aa1d749cbef34e3e84da730b68a59968",
              "IPY_MODEL_2bc3aa5e05b84aa0b82abe812b98323d"
            ]
          }
        },
        "aabd5e96eee646fbbcb1750f63393605": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aa1d749cbef34e3e84da730b68a59968": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2a660a7941ad4ba0abebf0c83571472b",
            "_dom_classes": [],
            "description": "Prediction: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ce3c5628d3ae4d6cb7a62891209144df"
          }
        },
        "2bc3aa5e05b84aa0b82abe812b98323d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_925e3cd2eb424bbd8dafb4e5e869acc5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 109/109 [00:01&lt;00:00, 77.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_38fad618758b44e987bff1488ebe9f10"
          }
        },
        "2a660a7941ad4ba0abebf0c83571472b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ce3c5628d3ae4d6cb7a62891209144df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "925e3cd2eb424bbd8dafb4e5e869acc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "38fad618758b44e987bff1488ebe9f10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aswingiridhar96/Tried_jupyter_notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "raZx6FZUx1-I",
        "colab_type": "text"
      },
      "source": [
        "[GitHub](https://github.com/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb)\n",
        "# Fine-tuning Electra on SST-2 and interpreting with Integrated Gradients\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "di_oai-BvG5k",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "This notebook contains an example of [fine-tuning](https://huggingface.co/transformers/training.html) an [Electra](https://huggingface.co/transformers/model_doc/electra.html) model on the [GLUE SST-2](https://nlp.stanford.edu/sentiment/index.html) dataset. After fine-tuning, the [Integrated Gradients](https://arxiv.org/pdf/1703.01365.pdf) **interpretability** method is applied to compute tokens' attributions for each target class. \n",
        "* We will instantiate a pre-trained Electra model from the [Transformers](https://huggingface.co/transformers/) library. \n",
        "* The data is downloaded from the [nlp](https://huggingface.co/nlp/) library. The input text is tokenized with [ElectraTokenizerFast](https://huggingface.co/transformers/model_doc/electra.html#electratokenizerfast) tokenizer backed by HF [tokenizers](https://huggingface.co/transformers/main_classes/tokenizer.html) library.\n",
        "* **Fine-tuning** for sentiment analysis is handled by the [Trainer](https://huggingface.co/transformers/main_classes/trainer.html) class. \n",
        "* After fine-tuning, the [Integrated Gradients](https://captum.ai/api/integrated_gradients.html) interpretability algorithm will assign importance scores to\n",
        "input tokens. We will use a **PyTorch** implementation from the [Captum](https://captum.ai/) library. \n",
        "  - The algorithm requires providing a reference sample (a baseline) since importance attribution is performed based on the model's output, as inputs change from reference values to the actual sample. \n",
        "  - The Integrated Gradients method satisfies the **completeness** property. We will look at the sum of attributions for a sample and show that the sum approximates (explains) prediction's shift from the baseline value. \n",
        "* The final sections of the notebook contain a color-coded **visualization** of attribution results made with *captum.attr.visualization* library.\n",
        "\n",
        "The notebook is based on the [Hugging Face documentation](https://huggingface.co/) and the implementation of Integrated Gradients attribution methods is adapted from the Captum.ai\n",
        "[Interpreting BERT Models (Part 1)](https://captum.ai/tutorials/Bert_SQUAD_Interpret)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qi5nKLqupSt_",
        "colab_type": "text"
      },
      "source": [
        "## Installation & imports\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n23Rko2YVSLQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "eebc1755-3d8f-4442-85c7-d42faa647094"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install pyarrow\n",
        "!pip install nlp\n",
        "!pip install captum"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
            "\r\u001b[K     |▍                               | 10kB 23.3MB/s eta 0:00:01\r\u001b[K     |▉                               | 20kB 6.8MB/s eta 0:00:01\r\u001b[K     |█▎                              | 30kB 8.0MB/s eta 0:00:01\r\u001b[K     |█▊                              | 40kB 8.8MB/s eta 0:00:01\r\u001b[K     |██▏                             | 51kB 7.4MB/s eta 0:00:01\r\u001b[K     |██▋                             | 61kB 8.0MB/s eta 0:00:01\r\u001b[K     |███                             | 71kB 8.5MB/s eta 0:00:01\r\u001b[K     |███▍                            | 81kB 9.2MB/s eta 0:00:01\r\u001b[K     |███▉                            | 92kB 8.6MB/s eta 0:00:01\r\u001b[K     |████▎                           | 102kB 8.8MB/s eta 0:00:01\r\u001b[K     |████▊                           | 112kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 122kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 133kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████                          | 143kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 153kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 163kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 174kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 184kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████                        | 194kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 204kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████                       | 215kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 225kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 235kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 245kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 256kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████                     | 266kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 276kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████                    | 286kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 296kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 307kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 317kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 327kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 337kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 348kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 358kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 368kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 378kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 389kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 399kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 409kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 419kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 430kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 440kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 450kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 460kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 471kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 481kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 491kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 501kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 512kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 522kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 532kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 542kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 552kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 563kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 573kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 583kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 593kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 604kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 614kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 624kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 634kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 645kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 655kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 665kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 675kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 686kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 696kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 706kB 8.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 716kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 727kB 8.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 737kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 747kB 8.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 757kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 768kB 8.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 778kB 8.8MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\r\u001b[K     |▍                               | 10kB 25.4MB/s eta 0:00:01\r\u001b[K     |▊                               | 20kB 32.1MB/s eta 0:00:01\r\u001b[K     |█▏                              | 30kB 39.1MB/s eta 0:00:01\r\u001b[K     |█▌                              | 40kB 43.0MB/s eta 0:00:01\r\u001b[K     |█▉                              | 51kB 37.2MB/s eta 0:00:01\r\u001b[K     |██▎                             | 61kB 39.0MB/s eta 0:00:01\r\u001b[K     |██▋                             | 71kB 31.1MB/s eta 0:00:01\r\u001b[K     |███                             | 81kB 32.7MB/s eta 0:00:01\r\u001b[K     |███▍                            | 92kB 31.5MB/s eta 0:00:01\r\u001b[K     |███▊                            | 102kB 26.1MB/s eta 0:00:01\r\u001b[K     |████                            | 112kB 26.1MB/s eta 0:00:01\r\u001b[K     |████▌                           | 122kB 26.1MB/s eta 0:00:01\r\u001b[K     |████▉                           | 133kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 143kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 153kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████                          | 163kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 174kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 184kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████                         | 194kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 204kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 215kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 225kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 235kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████                       | 245kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 256kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 266kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████                      | 276kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 286kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 296kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 307kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 317kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 327kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 337kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 348kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 358kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 368kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 378kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 389kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 399kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 409kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 419kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 430kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████                | 440kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 450kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 460kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 471kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 481kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 491kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 501kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 512kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 522kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 532kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 542kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 552kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 563kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 573kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 583kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 593kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 604kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 614kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 624kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 634kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 645kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 655kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 665kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 675kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 686kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 696kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 706kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 716kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 727kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 737kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 747kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 757kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 768kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 778kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 788kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 798kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 808kB 26.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 819kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 829kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 839kB 26.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 849kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 860kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 870kB 26.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 880kB 26.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 890kB 26.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Collecting tokenizers==0.8.1.rc1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 48.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 48.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=36385c698f3bd72473d1c73170b3d8e16bcdc7bc9afc9ec0bd893e4f3b88f44b\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n",
            "Collecting pyarrow==0.16.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/00/d2/695bab1e1e7a4554b6dbd287d55cca096214bd441037058a432afd724bb1/pyarrow-0.16.0-cp36-cp36m-manylinux2014_x86_64.whl (63.1MB)\n",
            "\u001b[K     |████████████████████████████████| 63.2MB 44kB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pyarrow==0.16.0) (1.15.0)\n",
            "Requirement already satisfied: numpy>=1.14 in /usr/local/lib/python3.6/dist-packages (from pyarrow==0.16.0) (1.18.5)\n",
            "Installing collected packages: pyarrow\n",
            "  Found existing installation: pyarrow 0.14.1\n",
            "    Uninstalling pyarrow-0.14.1:\n",
            "      Successfully uninstalled pyarrow-0.14.1\n",
            "Successfully installed pyarrow-0.16.0\n",
            "Collecting nlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/09/e3/bcdc59f3434b224040c1047769c47b82705feca2b89ebbc28311e3764782/nlp-0.4.0-py3-none-any.whl (1.7MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7MB 9.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from nlp) (0.3.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from nlp) (1.18.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from nlp) (4.41.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from nlp) (0.7)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from nlp) (3.0.12)\n",
            "Collecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f7/73/826b19f3594756cb1c6c23d2fbd8ca6a77a9cd3b650c9dec5acc85004c38/xxhash-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl (242kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 51.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=0.16.0 in /usr/local/lib/python3.6/dist-packages (from nlp) (0.16.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.6/dist-packages (from nlp) (2.23.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from nlp) (1.0.5)\n",
            "Requirement already satisfied: six>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pyarrow>=0.16.0->nlp) (1.15.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.19.0->nlp) (1.24.3)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->nlp) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->nlp) (2018.9)\n",
            "Installing collected packages: xxhash, nlp\n",
            "Successfully installed nlp-0.4.0 xxhash-2.0.0\n",
            "Collecting captum\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/42/de/c018e206d463d9975444c28b0a4f103c9ca4b2faedf943df727e402a1a1e/captum-0.2.0-py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 8.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.2 in /usr/local/lib/python3.6/dist-packages (from captum) (1.6.0+cu101)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from captum) (3.2.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from captum) (1.18.5)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.2->captum) (0.16.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->captum) (2.8.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib->captum) (1.15.0)\n",
            "Installing collected packages: captum\n",
            "Successfully installed captum-0.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FctqAnsgVcQP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "714878b4-9ffb-4038-f691-1ae2d377d2b4"
      },
      "source": [
        "from typing import Dict\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import nlp\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import transformers\n",
        "from captum.attr import (IntegratedGradients, LayerIntegratedGradients,\n",
        "                         configure_interpretable_embedding_layer,\n",
        "                         remove_interpretable_embedding_layer)\n",
        "from captum.attr import visualization as viz\n",
        "from torch.utils.data import Dataset\n",
        "from transformers import (ElectraForSequenceClassification,\n",
        "                          ElectraTokenizerFast, EvalPrediction, InputFeatures,\n",
        "                          Trainer, TrainingArguments, glue_compute_metrics)\n",
        "\n",
        "transformers.__version__"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'3.0.2'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wHa2VomPVf9r",
        "colab_type": "text"
      },
      "source": [
        "## Model\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j3po0cgEvypP",
        "colab_type": "text"
      },
      "source": [
        "Sentiment analysis is a classification task that requires assigning a label to an entire sentence (sequence). We will use a PyTorch implementation of [`ElectraForSequenceClassification`](https://huggingface.co/transformers/model_doc/electra.html#electraforsequenceclassification) from the Hugging Face library. A matching tokenizer implemented in the [`ElectraTokenizerFast`](https://huggingface.co/transformers/model_doc/electra.html#electratokenizerfast) class will handle tokenization."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TioBrt5VhIr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281,
          "referenced_widgets": [
            "2fa165296c664aa2be133523b2553a39",
            "b0cc26044da8484b989d235fd8b5c2ff",
            "4559b4cba09641498757c2824a3bf71c",
            "7d85c7a5d9d74583a47c8f642e5d4e7c",
            "0fc01cdc8579498898b204530c0bdd8b",
            "b6286be7b9a948e7aab4ef4cf0f767ec",
            "1234383d81604073aeaf2b7c0d1c02f6",
            "c37621beb7454486b64b13f3bdd4f6e6",
            "e2a3011f09024d049ee9b932385202c4",
            "11db628685bd4b1887da0fd99ca8857b",
            "228756b95d5d436a89e20db08dfe277c",
            "52e02012ccce4188b3467a40197fa5ac",
            "8c26f27649df44239fef5ea03bf30c1b",
            "77b27ea9183c4012a065c088cf011a4f",
            "22662dd134744a918f7f367d796ce035",
            "0887573c76a44c2294c964382de6880f",
            "e5d97b31441c4bd190472d41bc1ff690",
            "44fd65bdc89e407982a3f6424f07f5ba",
            "b1736eaabccd4ca081af80f6eb777605",
            "ef183506d505472a8df3661b0497d545",
            "d6ab129643c044eca57b4661d151a7f4",
            "dc228450bdd14d2f9d43c37b3525850c",
            "fd6a7b1e6aaa419690de3cf28249bb0a",
            "286f78363ccf4e78b1b402bb0ece8276"
          ]
        },
        "outputId": "a002be59-bd18-41ef-d059-acbf8139cc9a"
      },
      "source": [
        "model = ElectraForSequenceClassification.from_pretrained(\n",
        "    \"google/electra-small-discriminator\", num_labels = 2)\n",
        "\n",
        "tokenizer = ElectraTokenizerFast.from_pretrained(\n",
        "    \"google/electra-small-discriminator\", do_lower_case=True)                      "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2fa165296c664aa2be133523b2553a39",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e2a3011f09024d049ee9b932385202c4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=54245363.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at google/electra-small-discriminator were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense.weight', 'discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense_prediction.bias']\n",
            "- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-small-discriminator and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e5d97b31441c4bd190472d41bc1ff690",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7SdUQg2ntJYj",
        "colab_type": "text"
      },
      "source": [
        "## Data\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--54_ugqalQb",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "**Download**\n",
        "\n",
        "Let's now download the SST-2 dataset from the nlp library and take a brief look at it. It contains short movie reviews labelled for sentiment: 0 for negative and 1 for a positive review. The data is split into training, validation and test set. The labels for the test set are kept undisclosed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SEn6pshAVkmw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470,
          "referenced_widgets": [
            "cae2ed62500b4e628c0b2240c557e8b1",
            "be2576ab0cb34e7ba183a5ed737dd68f",
            "cede0a2d8a314d3eb5190ecba5064f1b",
            "0dd9dec9cebc478ab6d9057fef42bdff",
            "d0db1f55615b401fb03707209cd0d6fe",
            "42825a57bcb44d669eb67f4e6139f32f",
            "f3fba90c44b44ee586608204651326ef",
            "c5dd6acd7e0c47d9a67ee8fa79d67339",
            "9c7bd45d6ed9440f80992b72b8f8c194",
            "beeb36fe601c4d4bbf2cf0c822fcd807",
            "bbc5ef5f7bf743d49ad8023d99a7f6e1",
            "6a97c789f90b463f92a7ca9d4613b9de",
            "011eff21098e4378855bad72eba06cb4",
            "91767051562c41bcbb1a412369d9cefe",
            "faa5e5e552fb4204b2b7dd794cdfda86",
            "aea8d41be30d4acdb33b063485c03593",
            "527000b2ef9f4ed8ac41e313f12ed943",
            "41d355c455304b928370280499510c04",
            "b2f3f33409074b4ca78666bac15a63c7",
            "7e0f7ba49ed741bfa31b0cb347b42f09",
            "bd1318125543430e8db6ac8bf51d7afa",
            "c42a7f5e46db4ab5837b341cc3ea3341",
            "aed7770446bc47f7b9d8a08bfeb4098e",
            "2c8e9b7928b74fddb570d0591fd1f462",
            "fdc91e89d7884641954455e1b28a827e",
            "62198255f0d9462d92b9cfa15caed712",
            "1e164ec812584fe8ad655ebc2f162a60",
            "b05c3b1ca054451b8358be45d2438231",
            "79cfc2a506ed4207b04b0627737b77f9",
            "491e70d6964e462193e1e111f28df688",
            "5a533b02ee8d4555b298b258c0687202",
            "e2d4197e70294676affa725e5a38e818",
            "6d710654817041a0a7694d62b829c6b0",
            "deecf8705c7b4d9385d373286e7dc64d",
            "6a9d432d0f534c528d58cd279845d3aa",
            "1bcbd81da66a4f6fb9fdbdfb39f9e187",
            "0dc71f120f1240318c3d40b5a034e0c2",
            "1e23fd906ea94721b68087d8daf85b38",
            "fe51d57d46b1417886cf40bedbcb24d5",
            "576ec8e88c7d490b893b0ec02b71c150",
            "099914e405c64179a44e6605b68b6023",
            "6374c92441a943e1acb7153413f45567",
            "faca957637334fd0895598db90f74ff0",
            "41cb5b0ac5a0451fa62d83b5270e6c8d",
            "3a8cedda9c434680bc71ce33afbbeb0f",
            "20acab854f32489b977bccdb7dd66bde",
            "9a7e57eab34c444db5b6b96929a90fd3",
            "08246b9dd2614c40a9048ab795951bd1"
          ]
        },
        "outputId": "37d5c595-a71e-4e1c-d40d-226aad8dacff"
      },
      "source": [
        "# Load the SST2 dataset from the nlp library\n",
        "dataset = nlp.load_dataset(\"glue\", \"sst2\")\n",
        "\n",
        "# Look at the labels\n",
        "print(\"Training set labels: {}\".format(set(dataset[\"train\"][\"label\"])))\n",
        "print(\"Validation set labels: {}\".format(set(dataset[\"validation\"][\"label\"])))\n",
        "print(\"Test set labels: {}\".format(set(dataset[\"test\"][\"label\"])))\n",
        "\n",
        "# Explore the dataset\n",
        "df = pd.DataFrame({\"senence\": dataset[\"train\"][\"sentence\"],\n",
        "                   \"label\": dataset[\"train\"][\"label\"]})\n",
        "pd.options.display.max_colwidth = 0\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cae2ed62500b4e628c0b2240c557e8b1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=29068.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9c7bd45d6ed9440f80992b72b8f8c194",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=30329.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Downloading and preparing dataset glue/sst2 (download: 7.09 MiB, generated: 4.81 MiB, post-processed: Unknown sizetotal: 11.90 MiB) to /root/.cache/huggingface/datasets/glue/sst2/1.0.0/005857b1e5a6280d8f1a9b9537d44a08ba30cb6be958e81fac98e625a0d487a7...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "527000b2ef9f4ed8ac41e313f12ed943",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=7439277.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fdc91e89d7884641954455e1b28a827e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6d710654817041a0a7694d62b829c6b0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "099914e405c64179a44e6605b68b6023",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\rDataset glue downloaded and prepared to /root/.cache/huggingface/datasets/glue/sst2/1.0.0/005857b1e5a6280d8f1a9b9537d44a08ba30cb6be958e81fac98e625a0d487a7. Subsequent calls will reuse this data.\n",
            "Training set labels: {0, 1}\n",
            "Validation set labels: {0, 1}\n",
            "Test set labels: {-1}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>senence</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>hide new secretions from the parental units</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>contains no wit , only labored gags</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>that loves its characters and communicates something rather beautiful about human nature</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>remains utterly satisfied to remain the same throughout</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>on the worst revenge-of-the-nerds clichés the filmmakers could dredge up</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                                                     senence  label\n",
              "0  hide new secretions from the parental units                                                0    \n",
              "1  contains no wit , only labored gags                                                        0    \n",
              "2  that loves its characters and communicates something rather beautiful about human nature   1    \n",
              "3  remains utterly satisfied to remain the same throughout                                    0    \n",
              "4  on the worst revenge-of-the-nerds clichés the filmmakers could dredge up                   0    "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8E-1tg1jyAjJ",
        "colab_type": "text"
      },
      "source": [
        "**Create dataset**\n",
        "\n",
        "We will now create a custom [map-style PyTorch dataset](https://pytorch.org/docs/stable/data.html#map-style-datasets) to serve model's key-value parameters in a seamless manner. \n",
        "\n",
        "The `TrainerDataset` class is derived from `torch.utils.data.Dataset`. The overridden `__getitem__` method yields an instance of the `InputFeatures` class. \n",
        "\n",
        "Conversion to torch tensors and placing on cuda/cpu is handled by the `trainer` object used for fine-tuning."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hg9gClv2VtGI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TrainerDataset(Dataset):\n",
        "    def __init__(self, inputs, targets, tokenizer):\n",
        "        self.inputs = inputs\n",
        "        self.targets = targets\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "        # Tokenize the input\n",
        "        self.tokenized_inputs = tokenizer(inputs, padding=True)   \n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.inputs)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return InputFeatures(\n",
        "            input_ids=self.tokenized_inputs['input_ids'][idx],\n",
        "            token_type_ids=self.tokenized_inputs['token_type_ids'][idx],\n",
        "            attention_mask=self.tokenized_inputs['attention_mask'][idx],\n",
        "            label=self.targets[idx])         "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Ncdro4DEcf4",
        "colab_type": "text"
      },
      "source": [
        "We need to create the training and validation datasets. As GLUE SST-2 dataset does not disclose labels for the test set, so we will be using validation data for testing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BIjHu-eMVu3e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataset = TrainerDataset(dataset[\"train\"][\"sentence\"],\n",
        "                               dataset[\"train\"][\"label\"], tokenizer)\n",
        "eval_dataset = TrainerDataset(dataset[\"validation\"][\"sentence\"],\n",
        "                              dataset[\"validation\"][\"label\"], tokenizer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hki37gxJVxTX",
        "colab_type": "text"
      },
      "source": [
        "## Fine-tuning\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CK6lfc6Hidmt",
        "colab_type": "text"
      },
      "source": [
        "Fine-tuning with a `Trainer` class instance requires setting training arguments and creating a `trainer` object. The model, as well as training and validation datasets, are passed to the trainer's constructor, along with training arguments. We will pass one more parameter, the `compute_metrics` function, to have the `trainer` calculate the *Accuracy* metric after fine-tuning. The `Trainer` class takes care of conversion to tensor format and placement on a cpu/gpu device."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lRZJsqMRwixQ",
        "colab_type": "text"
      },
      "source": [
        "### Set parameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y0F-0xBBwoSO",
        "colab_type": "text"
      },
      "source": [
        "Training parameters have been taken from the [Electra Github](https://github.com/google-research/electra/blob/master/configure_finetuning.py) repository or are default values. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o2aR1A8LVzLH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Set seed for reproducibility\n",
        "np.random.seed(123)\n",
        "torch.manual_seed(123)\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"./models/model_electra\",\n",
        "    num_train_epochs=3,  # 1 (1 epoch gives slightly lower accuracy)\n",
        "    overwrite_output_dir=True,\n",
        "    do_train=True,\n",
        "    do_eval=True,\n",
        "    per_device_train_batch_size=32,\n",
        "    evaluate_during_training=True,     \n",
        "    dataloader_drop_last=True,  # Make sure all batches are of equal size\n",
        ")\n",
        "\n",
        "\n",
        "def compute_metrics(p: EvalPrediction) -> Dict:\n",
        "    preds = np.argmax(p.predictions, axis=1)\n",
        "    # The choice of a dataset (task_name) implies metric\n",
        "    return glue_compute_metrics(\n",
        "        task_name=\"sst-2\",\n",
        "        preds=preds,\n",
        "        labels=p.label_ids)\n",
        "\n",
        "\n",
        "# Instantiate the Trainer class\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=eval_dataset,\n",
        "    compute_metrics=compute_metrics)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AXTyBstWV1LM",
        "colab_type": "text"
      },
      "source": [
        "### Run fine-tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xr9UZ4PqiHo0",
        "colab_type": "text"
      },
      "source": [
        "Run the `train` method of the `trainer` object to fine-tune the model on the SST-2 dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7lVhSg4IV23Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 598,
          "referenced_widgets": [
            "515e6a8c59ad442dae9f3e4bdbaffd4b",
            "d06e2fc8fe4a42138b93583bf0807e2f",
            "4878d398a15249d7926b605d924cc3fb",
            "27180224a0044f1e9171a57ad2a420a6",
            "19355e93f085467e83888ea59f82678a",
            "3b6075b83e414f0684c8ca15d770a463",
            "b61cac8151984edcb9108bdfbcaaa38b",
            "3d69e37f0f044dcbba5411f2d7b5dea1",
            "7a7d4104c50a4485820ec3e6b757e4e3",
            "b13ce1f6400343db917f6d96ab3586de",
            "9abefd88f3bf471bbca170931764c1cc",
            "2657dce6830c4eaaa5164be6b2a53904",
            "1838094d3aad4a43bdac85f8d3d0345e",
            "620e2b2480204591a0989d45370c8d52",
            "e0ab87123ffb4b018efecc36c00e665a",
            "547143a6e0c340cda9fbedebb465ac9d",
            "17e4e106b3c5402eb6c028c28525700a",
            "2f4913c6857b4ee0ace51a4a88a63012",
            "4f8af6fc0b294ae8b8169a84020402f5",
            "a1888f9d76694ad0a0d9df7b7677107b",
            "af7ce86aaaac46acb13cc65537b25571",
            "1eeff683391d42ee95b5328968787028",
            "d2216ed153cc4ea59540fff4d7bc8061",
            "52bfd2f135224341b44119c97a6bbd52",
            "1305437e0af94ac8b77153547f95486d",
            "1e2c4074816848c2a9d4e3132b34e1c1",
            "2a483a8cbd454f098327ea75b8bd530c",
            "44ec186472694ecbb300d3521998e190",
            "b973157c678048f7848387e037ebfa66",
            "3f2551355d1b4acd9f49e901a35e81e0",
            "baadcd7ef9794660bc831449a78d45a5",
            "8945b7975ef04bd887153fa3ccb82383",
            "3142c36e00cd4c299159559e91f4474e",
            "a5ef7ab6899442089b05f5eb6ccdb240",
            "28095903c54b44e8b255d41474eca0f4",
            "57085d2c040a42c6b4e4c6229216ca9c",
            "dc61beb8cd6d43f9908fe35b93f26768",
            "63e5bb476e8f4a9ab327c40607d570fe",
            "ab6f9568b1b443e89f510b62cf6aa555",
            "63f76a44c096428b838bbe6c83a93bf1",
            "fc365a89f45747db83b0e275ba5b4bdc",
            "e075b266745a48e98913b4e2017ad3c8",
            "201f9ddf25864392a762991d50083164",
            "f8169e04316a4eed8e1e04482f42c973",
            "7e6c7098abf84264bbbb5d986262fca9",
            "d9c4f1c99ebc4367878f65fb64398fc3",
            "1be470549574436ba6f5dc97590dd1fd",
            "af14d8633d924be0895a97be52f7ce15",
            "4d1b01439fc34f9bafdb1cc512902e0c",
            "f5f0dc0196f14bd48e6209a995b656e8",
            "e82326872c5848e6a6d01efc0e3b912e",
            "ee8aeb5aab3f4dea816414d63cece5d9",
            "85dc2499531346e0b9f45696f4780f39",
            "0db240967fd643af95169010b64fcdee",
            "f4523ee835e04ed7a38bc242242ebc21",
            "e4cd34b3624b49238654b10af98ba457",
            "1b92e9e37aea4c38828ce06fa8be7608",
            "65188d29b3d1453f8dde9f7c9c66f763",
            "5d4099d823cf4b958c6ff25815d53250",
            "59870b85248246e2b55cd282b7ad7e51",
            "31ce28582b274de394e98f16397b4a75",
            "c0a492c6fa964723ae90ca8f9b517fc2",
            "d71bc4ffabe343f1a8e39f90b54129b9",
            "8a6b070bcc434cd6ab44cebaf69a1efb",
            "37016f426f244bfb8ebb9db490604665",
            "c7b36421f6614a6f953709b0db204e49",
            "6438b1a59efa4fea895f6548aefdc97d",
            "c250c8ad3b1a4b13b8316f2bfce05561",
            "33bfdcd711044097a82c561fe8ab6d2e",
            "fe5c32e7f5cb4ae6825011c59bc8b6d5",
            "dc5d5323067549b5b097e98dce676b65",
            "4f2ff614cecc463d806b8195588bfa40",
            "130553b53eb24e59a84ecc730bfbe4f5",
            "68899e22a5f04e028d7434815d77255b",
            "cba118ce90554967b59d9600fc762065",
            "59781d9ab0a7416291254d1bd258bc5a",
            "5c66861f32614460a054a278bb247e3e",
            "cec0e1a60bdd48ab876763ff23e2070c",
            "8291258e0f364fd7bfd1b871503127a5",
            "40a4912051b34dc0aa5892760f545548"
          ]
        },
        "outputId": "a7e2d494-d65c-4ff3-95b4-6a9ed225e5fd"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "515e6a8c59ad442dae9f3e4bdbaffd4b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=3.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7a7d4104c50a4485820ec3e6b757e4e3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=2104.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:200: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.\n",
            "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "17e4e106b3c5402eb6c028c28525700a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1305437e0af94ac8b77153547f95486d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3142c36e00cd4c299159559e91f4474e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=2104.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc365a89f45747db83b0e275ba5b4bdc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4d1b01439fc34f9bafdb1cc512902e0c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1b92e9e37aea4c38828ce06fa8be7608",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Iteration', max=2104.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "37016f426f244bfb8ebb9db490604665",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "130553b53eb24e59a84ecc730bfbe4f5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=6312, training_loss=0.18606270686819198)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3-3V-0yv_r_L",
        "colab_type": "text"
      },
      "source": [
        "### Evaluate "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75MYAZtG1K-N",
        "colab_type": "text"
      },
      "source": [
        "The metric used for evaluation of the Stanford Sentiment Treebank (SST) data is *Accuracy*. The result is returned by the `Trainer` class object used for fine-tuning. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGz07rX7_lsJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "2adf1fd0a79d40948fbeb90711b46944",
            "d62df1fc3ab347c7bb5575282a92ed69",
            "956846094c2f4b8a9e5556dbefb31ff0",
            "87d93832eff244ccb9415c0bff4a5582",
            "ab5129556c3f409ca25d82a7a60fe674",
            "3d51ef24a3ce41ecb068d098e570b98c",
            "18705bfe3aed46f0a2f6abbc80ea6ea6",
            "d57209b086e84388b77da65aaa556ed3"
          ]
        },
        "outputId": "acaf394a-6a5e-4217-d505-40a1ba3876ff"
      },
      "source": [
        "model_result = trainer.evaluate()\n",
        "print(\"Accuracy: {}\".format(model_result[\"eval_acc\"]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2adf1fd0a79d40948fbeb90711b46944",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Evaluation', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Accuracy: 0.9174311926605505\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M-V2TAFppOuz",
        "colab_type": "text"
      },
      "source": [
        "## Interpretability with Captum\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwaRQy2LrnQd",
        "colab_type": "text"
      },
      "source": [
        "The examples below use two attribution methods from the Captum library:\n",
        "- **Integrated Gradients** - the method requires configuring interpretation hooks to perform attribution for all three embedding layers in one step, and\n",
        "- **Layer Integrated Gradients**, computed separately with respect to each of the three layers:\n",
        "    - `model.electra.embeddings.word_embeddings`,\n",
        "    - `model.electra.embeddings.token_type_embeddings`,\n",
        "    - `model.electra.embeddings.position_embeddings`.\n",
        "\n",
        "We will try to find out to what extent, according to these methods, each token has contributed to the model's prediction, or, more precisely, to its shift from the baseline output. \n",
        "Each method requires setting a target class index: 0 for negative or 1 for a positive sentiment. Attribution is performed for each target class separately. Scores will be assigned with regard to the model's output for the selected class.\n",
        "\n",
        "The shape of attributions is the same as the shape of the `inputs` parameter of the `attribute` method."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ly7k0ryjp1C-",
        "colab_type": "text"
      },
      "source": [
        "Let's pick an example:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c0uMvVUnzR4m",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 93
        },
        "outputId": "a442e3f5-8f89-402d-efa5-fe1d9f2cb871"
      },
      "source": [
        "text = \"visually imaginative , thematically instructive and thoroughly \\\n",
        "delightful , it takes us on a roller-coaster ride from innocence to experience \\\n",
        "without even a hint of that typical kiddie-flick sentimentality . \"\n",
        "true_label = 1\n",
        "\n",
        "[x for x in dataset[\"validation\"] if x[\"sentence\"] == text]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'idx': 857,\n",
              "  'label': 1,\n",
              "  'sentence': 'visually imaginative , thematically instructive and thoroughly delightful , it takes us on a roller-coaster ride from innocence to experience without even a hint of that typical kiddie-flick sentimentality . '}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CHJo7nNBiPjV",
        "colab_type": "text"
      },
      "source": [
        "Set a cpu/gpu device according to availability. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uw0e2qtajCrG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SOQOBd8Hitie",
        "colab_type": "text"
      },
      "source": [
        "### Helper functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5rlkMmJmiQg8",
        "colab_type": "text"
      },
      "source": [
        "**Helper functions**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zxZg4TvKqPAH",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "The functions below **construct input tensors** for our sample and for a sequence of [PAD] tokens serving as a baseline. We also need to define a **forward function** running inference on the model. The function will be passed on to objects handling attribution. \n",
        "\n",
        "Computation with `IntegratedGradients` requires altering the model by **configuring additional layers**. For this purpose, the Captum library provides the `configure_interpretable_embedding_layer` and `remove_interpretable_embedding_layer` methods. Configuring an interpretable embedding layer modifies the model. A model with interpretable layers requires input of a different shape. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J89L8HhDB1JM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def configure_interpretable_embeddings():\n",
        "    \"\"\"Configure interpretable embedding layer\"\"\"\n",
        "    interpretable_embedding1 = configure_interpretable_embedding_layer(\n",
        "        model, \"electra.embeddings.word_embeddings\")\n",
        "    interpretable_embedding2 = configure_interpretable_embedding_layer(\n",
        "        model, \"electra.embeddings.token_type_embeddings\")\n",
        "    interpretable_embedding3 = configure_interpretable_embedding_layer(\n",
        "        model,\"electra.embeddings.position_embeddings\")\n",
        "    return (interpretable_embedding1,\n",
        "            interpretable_embedding2,\n",
        "            interpretable_embedding3)\n",
        "\n",
        "\n",
        "def remove_interpretable_embeddings(interpretable_embedding1, \n",
        "                                    interpretable_embedding2, \n",
        "                                    interpretable_embedding3):\n",
        "    \"\"\"Remove interpretable layer to restore the original model structure\"\"\"\n",
        "    if not \\\n",
        "    type(model.get_input_embeddings()).__name__ == \"InterpretableEmbeddingBase\":\n",
        "        return\n",
        "    remove_interpretable_embedding_layer(model, interpretable_embedding1)\n",
        "    remove_interpretable_embedding_layer(model, interpretable_embedding2)\n",
        "    remove_interpretable_embedding_layer(model, interpretable_embedding3)  \n",
        "\n",
        "\n",
        "def predict_forward_func(input_ids, token_type_ids=None, \n",
        "                         position_ids=None, attention_mask=None):\n",
        "    \"\"\"Function passed to ig constructors\"\"\"\n",
        "    return model(input_ids=input_ids, \n",
        "                 token_type_ids=token_type_ids, \n",
        "                 position_ids=position_ids, \n",
        "                 attention_mask=attention_mask)[0]  \n",
        "\n",
        "\n",
        "def prepare_input(text):\n",
        "    \"\"\"Prepare ig attribution input: tokenize sample and baseline text.\"\"\"\n",
        "    tokenized_text = tokenizer(text, return_tensors=\"pt\", \n",
        "                               return_attention_mask=True)\n",
        "    seq_len = tokenized_text[\"input_ids\"].shape[1]\n",
        "    position_ids = torch.arange(seq_len, dtype=torch.long).unsqueeze(0)\n",
        "\n",
        "    # Construct the baseline (a reference sample).\n",
        "    # A sequence of [PAD] tokens of length equal to that of the processed sample\n",
        "    ref_text = tokenizer.pad_token * (seq_len - 2) # special tokens\n",
        "    tokenized_ref_text = tokenizer(ref_text, return_tensors=\"pt\") \n",
        "    ref_position_ids = torch.arange(seq_len, dtype=torch.long).unsqueeze(0)\n",
        "\n",
        "    return (tokenized_text[\"input_ids\"],\n",
        "            tokenized_text[\"token_type_ids\"], \n",
        "            position_ids,\n",
        "            tokenized_ref_text[\"input_ids\"],\n",
        "            tokenized_ref_text[\"token_type_ids\"], \n",
        "            ref_position_ids,\n",
        "            tokenized_text[\"attention_mask\"])   \n",
        "\n",
        "\n",
        "def prepare_input_embed(input_ids, token_type_ids, position_ids,\n",
        "                        ref_input_ids, ref_token_type_ids, ref_position_ids,\n",
        "                        attention_mask):\n",
        "    \"\"\"Construct input for the modified model\"\"\"\n",
        "    input_ids_embed = interpretable_embedding1.indices_to_embeddings(input_ids)\n",
        "    ref_input_ids_embed = interpretable_embedding1.indices_to_embeddings(\n",
        "        ref_input_ids)\n",
        "    token_type_ids_embed = interpretable_embedding2.indices_to_embeddings(\n",
        "        token_type_ids)\n",
        "    ref_token_type_ids_embed = interpretable_embedding2.indices_to_embeddings(\n",
        "        ref_token_type_ids)\n",
        "    position_ids_embed = interpretable_embedding3.indices_to_embeddings(\n",
        "        position_ids)\n",
        "    ref_position_ids_embed = interpretable_embedding3.indices_to_embeddings(\n",
        "        ref_position_ids)\n",
        "    \n",
        "    return (input_ids_embed, token_type_ids_embed, position_ids_embed, \n",
        "            ref_input_ids_embed, ref_token_type_ids_embed, \n",
        "            ref_position_ids_embed, attention_mask)\n",
        "\n",
        "\n",
        "def get_input_data(text):\n",
        "    input_data = place_on_device(*prepare_input(text))\n",
        "    input_data_embed = prepare_input_embed(*input_data)   \n",
        "    return input_data, input_data_embed \n",
        "\n",
        "\n",
        "def place_on_device(*tensors):\n",
        "    tensors_device = []\n",
        "    for t in tensors:\n",
        "        tensors_device.append(t.to(device))\n",
        "    return tuple(tensors_device)  \n",
        "\n",
        "\n",
        "def ig_attribute(ig, class_index, input_data_embed):\n",
        "    return ig.attribute(inputs=input_data_embed[0:3],\n",
        "                        baselines=input_data_embed[3:6],\n",
        "                        additional_forward_args=(input_data_embed[6]),\n",
        "                        target = class_index,\n",
        "                        return_convergence_delta=True,\n",
        "                        n_steps=200)\n",
        "    \n",
        "\n",
        "def lig_attribute(lig, class_index, input_data):\n",
        "    return lig.attribute(\n",
        "        inputs=input_data[0], baselines=input_data[3],\n",
        "        additional_forward_args=(input_data[1], input_data[2], input_data[6]),\n",
        "        return_convergence_delta=True, target=class_index, n_steps=200)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4BN3x0ISIazV",
        "colab_type": "text"
      },
      "source": [
        "### Integrated Gradients"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "asVO1IpJyt8r",
        "colab_type": "text"
      },
      "source": [
        "To compute attributions with Integrated Gradients we will:\n",
        "- instantiate the `IntegratedGradients` class passing the `predict_forward_func` function as parameter,\n",
        "- configure interpretable embedding layer,\n",
        "- prepare input tensors,\n",
        "- compute attributions,\n",
        "- remove interpratable embedding layer.\n",
        "\n",
        "Calling the `get_input_embeddings` method of the model helps to find out whether extra layers have been configured."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMQKjnurz9qz",
        "colab_type": "text"
      },
      "source": [
        "#### Compute attributions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g8ZNDXGZIhW_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Instantiate the IntegratedGradients class\n",
        "ig = IntegratedGradients(predict_forward_func)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1l14-IXz2QnN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 352
        },
        "outputId": "226ae211-d108-4ae5-e97e-29c18f9806d9"
      },
      "source": [
        "# Configure interpretable embedding layer \n",
        "print(\"Original model's input embeddings:\\n {}\\n\".\n",
        "      format(model.get_input_embeddings()))\n",
        "if not \\\n",
        "type(model.get_input_embeddings()).__name__ == \"InterpretableEmbeddingBase\":\n",
        "    interpretable_embedding1, interpretable_embedding2,\\\n",
        "    interpretable_embedding3 = configure_interpretable_embeddings()\n",
        "print(\"Input embeddings with interpretable layer:\\n {}\\n\".\n",
        "      format(model.get_input_embeddings()))\n",
        "\n",
        "# Prepare input\n",
        "input_data, input_data_embed = get_input_data(text)  \n",
        "\n",
        "# Compute attributions for both target classes\n",
        "# class 0 (negative)\n",
        "attributions_0, approximation_error_0 = ig_attribute(ig, 0, input_data_embed)\n",
        "# class 1 (positive)\n",
        "attributions_1, approximation_error_1 = ig_attribute(ig, 1, input_data_embed)\n",
        "\n",
        "# Remove interpratable embedding layer used by ig attribution\n",
        "remove_interpretable_embeddings(interpretable_embedding1, \n",
        "                                interpretable_embedding2, \n",
        "                                interpretable_embedding3)\n",
        "print(\"\\nInput embeddings with interpretable layer removed:\\n {}\\n\"\n",
        ".format(model.get_input_embeddings()))\n",
        "\n",
        "print(\"\\nThe reference sample:\\n{}\".format(tokenizer.convert_ids_to_tokens(\n",
        "    input_data[3].clone().detach().to('cpu').numpy().squeeze())))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original model's input embeddings:\n",
            " Embedding(30522, 128, padding_idx=0)\n",
            "\n",
            "Input embeddings with interpretable layer:\n",
            " InterpretableEmbeddingBase(\n",
            "  (embedding): Embedding(30522, 128, padding_idx=0)\n",
            ")\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/attr/_models/base.py:189: UserWarning: In order to make embedding layers more interpretable they will be replaced with an interpretable embedding layer which wraps the original embedding layer and takes word embedding vectors as inputs of the forward function. This allows us to generate baselines for word embeddings and compute attributions for each embedding dimension. The original embedding layer must be set back by calling `remove_interpretable_embedding_layer` function after model interpretation is finished. \n",
            "  \"In order to make embedding layers more interpretable they will \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Input embeddings with interpretable layer removed:\n",
            " Embedding(30522, 128, padding_idx=0)\n",
            "\n",
            "\n",
            "The reference sample:\n",
            "['[CLS]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[PAD]', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "turkFMM8Ng6n",
        "colab_type": "text"
      },
      "source": [
        "#### Completeness"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kpltx2dbvNfa",
        "colab_type": "text"
      },
      "source": [
        "The Integrated Gradients method satisfies the completeness property. The sum of attributions should be equal, with certain accuracy, to the difference between the model's output for the sample and its output for the selected baseline (in this case a sequence of [PAD] tokens). Increase the `n_steps` parameter of the `attribute` method to obtain better accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "raXBOcBN2up6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 480
        },
        "outputId": "297a2bc2-dd2d-47ed-abdc-43f33b72e2cc"
      },
      "source": [
        "def check_completeness(attributions_0, attributions_1):\n",
        "    input_ids, token_type_ids, position_ids, ref_input_ids,\\\n",
        "    ref_token_type_ids, ref_position_ids, attention_mask = input_data\n",
        "\n",
        "    # Prediction for the sample\n",
        "    scores = predict_forward_func(input_ids, token_type_ids,\n",
        "                                position_ids, attention_mask) \n",
        "    # Prediction for the baseline\n",
        "    ref_scores = predict_forward_func(ref_input_ids, ref_token_type_ids,\n",
        "                                    ref_position_ids, attention_mask)\n",
        "\n",
        "    # Put on cpu\n",
        "    if torch.is_tensor(attributions_0[0]):\n",
        "        attributions_0 = [x.clone().detach().to('cpu').numpy() \n",
        "        for x in attributions_0]\n",
        "    if torch.is_tensor(attributions_1[0]):\n",
        "        attributions_1 = [x.clone().detach().to('cpu').numpy() \n",
        "        for x in attributions_1]  \n",
        "    scores = scores.clone().detach().to('cpu').numpy().squeeze()\n",
        "    ref_scores = ref_scores.clone().detach().to('cpu').numpy().squeeze()    \n",
        "\n",
        "    # How prediction for the sample differs from baseline prediction  \n",
        "    diff_from_baseline = scores - ref_scores\n",
        "\n",
        "    # Sum of attributions\n",
        "    attributions_sum0 = [x.sum() for x in attributions_0]\n",
        "    attributions_sum1 = [x.sum() for x in attributions_1]\n",
        "    attributions_sum = [sum(attributions_sum0), sum(attributions_sum1)]\n",
        "\n",
        "    # Difference from the baseline output for both classes\n",
        "    diff = diff_from_baseline - attributions_sum\n",
        "\n",
        "    # Find out which layers contribute to the score \n",
        "    print(\"Class 0: input tokens attr. sum: {}\".format(attributions_sum0[0]))\n",
        "    print(\"Classs 0: token type attr. sum: {}\".format(attributions_0[1].sum()))\n",
        "    print(\"Class 0: position ids attr. sum: {}\".format(attributions_0[2].sum()))\n",
        "    print(\"Class 1: input tokens attr. sum: {}\".format(attributions_1[0].sum()))\n",
        "    print(\"Classs 1: token type attr. sum: {}\".format(attributions_1[1].sum()))\n",
        "    print(\"Class 1: position ids attr. sum: {}\".format(attributions_1[2].sum()))\n",
        "\n",
        "    # Compare sum of attributions with the difference from baseline prediction\n",
        "    print(\"\\nPrediction for sample: {}\".format(scores))\n",
        "    print(\"Prediction for baseline: {}\".format(ref_scores))\n",
        "    print(\"Difference from baseline: {}\".format(diff_from_baseline))\n",
        "    print(\"Sum of attributions: {}\".format(attributions_sum))\n",
        "    print(\"\\nClass 0:\\n score: {}\\n reference score: {}\\\n",
        "    \\n difference from ref.: {}\\n sum of attributions:  {}\\\n",
        "    \\n difference from reference - attributions: {}\".\\\n",
        "    format(scores[0], ref_scores[0], diff_from_baseline[0], \n",
        "            attributions_sum[0], diff[0]))\n",
        "    print(\"\\nClass 1:\\n score: {}\\n reference score: {}\\\n",
        "    \\n difference from ref.: {}\\n sum of attributions:  {}\\\n",
        "    \\n difference from reference - attributions: {}\".\\\n",
        "    format(scores[1], ref_scores[1], diff_from_baseline[1], \n",
        "            attributions_sum[1], diff[1]))\n",
        "    \n",
        "    return attributions_0, attributions_1\n",
        "    \n",
        "    \n",
        "attributions_0, attributions_1 = check_completeness(attributions_0,\n",
        "                                                    attributions_1)    "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Class 0: input tokens attr. sum: -2.3934506633545016\n",
            "Classs 0: token type attr. sum: 0.0\n",
            "Class 0: position ids attr. sum: 0.0\n",
            "Class 1: input tokens attr. sum: 2.7104060043399527\n",
            "Classs 1: token type attr. sum: 0.0\n",
            "Class 1: position ids attr. sum: 0.0\n",
            "\n",
            "Prediction for sample: [-3.3136892  3.5635498]\n",
            "Prediction for baseline: [-0.9202382   0.85314345]\n",
            "Difference from baseline: [-2.393451   2.7104063]\n",
            "Sum of attributions: [-2.3934506633545016, 2.7104060043399527]\n",
            "\n",
            "Class 0:\n",
            " score: -3.3136892318725586\n",
            " reference score: -0.920238196849823    \n",
            " difference from ref.: -2.393450975418091\n",
            " sum of attributions:  -2.3934506633545016    \n",
            " difference from reference - attributions: -3.1206358919178e-07\n",
            "\n",
            "Class 1:\n",
            " score: 3.563549757003784\n",
            " reference score: 0.8531434535980225    \n",
            " difference from ref.: 2.7104063034057617\n",
            " sum of attributions:  2.7104060043399527    \n",
            " difference from reference - attributions: 2.9906580900629365e-07\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GGv8Z_rPWpxs",
        "colab_type": "text"
      },
      "source": [
        "### Layer Integrated Gradients"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-g85-8mizbf4",
        "colab_type": "text"
      },
      "source": [
        "With Layer Integrated Gradients, attributions are computed with respect to a certain layer. We'll run the algorithm for three [layers](https://github.com/huggingface/transformers/blob/d5b0a0e235cc6fccba4f9013cdb54cee01e90a91/src/transformers/modeling_electra.py#L131) separately: \n",
        "- `model.electra.embeddings.word_embeddings`\n",
        "- `model.electra.embeddings.token_type_embeddings`\n",
        "- `model.electra.embeddings.position_embeddings`\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VtkHVIz80pmG",
        "colab_type": "text"
      },
      "source": [
        "#### Compute attributions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g37CEAkY7GAc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "b632284b-1378-40e2-9825-e7d5b0d08eaa"
      },
      "source": [
        "# Input for lig attributions (model with no special layers configured)\n",
        "input_data = place_on_device(*prepare_input(text))\n",
        "\n",
        "# 1. Layer: model.electra.embeddings.word_embeddings\n",
        "lig_we = LayerIntegratedGradients(\n",
        "    predict_forward_func, \n",
        "    model.electra.embeddings.word_embeddings)\n",
        "layer_attributions_we_0, _ = lig_attribute(lig_we, 0, input_data)\n",
        "layer_attributions_we_1, _ = lig_attribute(lig_we, 1, input_data)\n",
        "\n",
        "# 2. Layer: model.electra.embeddings.token_type_embeddings\n",
        "lig_tte = LayerIntegratedGradients(\n",
        "    predict_forward_func,\n",
        "    model.electra.embeddings.token_type_embeddings)\n",
        "layer_attributions_tte_0, _ = lig_attribute(lig_tte, 0, input_data)\n",
        "layer_attributions_tte_1, _ = lig_attribute(lig_tte, 1, input_data)\n",
        "\n",
        "# 3. Layer: model.electra.embeddings.position_embeddings\n",
        "lig_pe = LayerIntegratedGradients(\n",
        "    predict_forward_func, \n",
        "    model.electra.embeddings.position_embeddings)\n",
        "layer_attributions_pe_0, _ = lig_attribute(lig_pe, 0, input_data)\n",
        "layer_attributions_pe_1, _ = lig_attribute(lig_pe, 1, input_data)\n",
        "\n",
        "print(\"Shape of attributions:\")\n",
        "print(layer_attributions_we_0.shape, layer_attributions_we_1.shape)\n",
        "print(layer_attributions_tte_0.shape, layer_attributions_tte_1.shape)\n",
        "print(layer_attributions_pe_0.shape, layer_attributions_pe_1.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of attributions:\n",
            "torch.Size([1, 41, 128]) torch.Size([1, 41, 128])\n",
            "torch.Size([1, 41, 128]) torch.Size([1, 41, 128])\n",
            "torch.Size([1, 41, 128]) torch.Size([1, 41, 128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ofIDYnUr994Y",
        "colab_type": "text"
      },
      "source": [
        "#### Completeness"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9uTWg6O-NuF",
        "colab_type": "text"
      },
      "source": [
        "Completeness for attributions found for each layer separately"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15G8-Qg69_0O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 480
        },
        "outputId": "c9caf6d2-14d9-406a-9c72-1ef15e18597c"
      },
      "source": [
        "layer_attributions_0, layer_attributions_1 = check_completeness(\n",
        "    (layer_attributions_we_0, layer_attributions_tte_0, layer_attributions_pe_0),\n",
        "    (layer_attributions_we_1, layer_attributions_tte_1, layer_attributions_pe_1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Class 0: input tokens attr. sum: -2.3934506633545016\n",
            "Classs 0: token type attr. sum: 0.0\n",
            "Class 0: position ids attr. sum: 0.0\n",
            "Class 1: input tokens attr. sum: 2.7104060043399527\n",
            "Classs 1: token type attr. sum: 0.0\n",
            "Class 1: position ids attr. sum: 0.0\n",
            "\n",
            "Prediction for sample: [-3.3136892  3.5635498]\n",
            "Prediction for baseline: [-0.9202382   0.85314345]\n",
            "Difference from baseline: [-2.393451   2.7104063]\n",
            "Sum of attributions: [-2.3934506633545016, 2.7104060043399527]\n",
            "\n",
            "Class 0:\n",
            " score: -3.3136892318725586\n",
            " reference score: -0.920238196849823    \n",
            " difference from ref.: -2.393450975418091\n",
            " sum of attributions:  -2.3934506633545016    \n",
            " difference from reference - attributions: -3.1206358919178e-07\n",
            "\n",
            "Class 1:\n",
            " score: 3.563549757003784\n",
            " reference score: 0.8531434535980225    \n",
            " difference from ref.: 2.7104063034057617\n",
            " sum of attributions:  2.7104060043399527    \n",
            " difference from reference - attributions: 2.9906580900629365e-07\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0HnTBKcFAjy5",
        "colab_type": "text"
      },
      "source": [
        "#### Plot"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uASNIh0nA5H7",
        "colab_type": "text"
      },
      "source": [
        "**Compare with IG**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5iurfLHIpJ0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 454
        },
        "outputId": "89b55c94-2be0-4daa-b854-48ad97595f39"
      },
      "source": [
        "# Attributions for input_ids computed with IG and LIG\n",
        "# word_embeddings: index 0\n",
        "ig_1 = attributions_1[0].squeeze().sum(1)\n",
        "lig_1 = layer_attributions_1[0].squeeze().sum(1)\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(tokenizer(text)[\"input_ids\"])\n",
        "\n",
        "range_ig = [x + 0.5 for x in np.arange(len(ig_1))]\n",
        "range_lig = [x + 0.5 for x in range_ig]\n",
        " \n",
        "plt.rcParams[\"figure.figsize\"] = [12, 6] \n",
        "plt.bar(range_ig, ig_1, width=0.5, label='ig')\n",
        "plt.bar(range_lig, lig_1, width=0.5, label='lig')\n",
        "plt.xlabel('Token', fontweight='bold')\n",
        "plt.xticks(list(range(len(lig_1))), tokens, rotation='vertical')\n",
        "plt.legend()\n",
        "plt.title(\"Attributions with IG and LIG for the positive target class.\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsIAAAG1CAYAAADz3ZI9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd7hkRZn48e9LclSChDEgUURxBEVEQDGAYgBdEEUFE7CuqKyBNeKaMOeEP/MqKgZETKi4iCggihIMJNFFRBlABQQEkfz+/qjqmZ47t7tP9+2ee++c7+d57jPT3adOVXef7n5PnbeqIjORJEmS2maV2W6AJEmSNBsMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsrQAR8ayI+EHX7YyIe49x/9dHxL3Gtb9xiYhPRMQb+jx+WER8cUW2aZzG/T4OUW9ExBERcXVEnD6hOg6IiFMnse9xiIhHRMTv+jy+Sf1crLoi2zWXRcQuEbF4ttshzSUGwtI0IuKkGmTcYcr9F0fEbl23N6vB0Gr99peZX8rMx42xbf8xZf9rZuZF49j/OGXmCzPzrTCeH+GpgWdEbBkRR0XEFRHxj4j4v4j4SERsNNO2z9R071O9f7ljJiK2j4jv1mPumog4PyLeHhHr9tj9w4HHAhtl5g5jaGuj43guycyfZOZ9O7enfjYz88/1c3HbOOud7ROE2a5fWtkYCEtTRMRmwCOABPYcw/7mTXAxn9SA+BfAZcCDMnNtYGfgD5RAcV6IiIcBJwE/BbbKzLsATwBuBR7Yo9imwMWZ+c8R6vN4nEW+/tIck5n++edf1x/wRkpQ8gHgu133HwncDvwLuB54NfBnSsB8ff17KHBALf9B4CrgbfW+U7v2lcBLgYuAK4H3AqvUxw4Dvti17WZ1+9WAtwO3ATfW+v5f1/7uXf+/DvAF4ArgT8Dru/Z9AHAq8D7gauCPwO5ddR1Q23RdfexZ07w+C+prsEG9/TpK0LZ2vf1W4EP1/5+rz//OtcztXa/VhvW5Hl3bex1wHrB9n/em+3l+EfjOkO/tFsCP6vtyJfAl4C5dj18MvBI4G7gW+CqwoOvxVwGXU4Lvf+9uzzR1nQT8xzT3L3k/6+1TgY8M8RyeV9//2+rr+OZ6//OBC4G/A8cCG0553f4T+D/gj9Pss9dx3O9YWQf4TH09Lq3v86o92nwYcEx9Pa8Dfgk8sOvx+9XX65p6DOzZ9dgewPm13KXAK+v9uwCL+3w2l7zOwDOAM6e06b+AY+v/71Cf55+BvwKfAO44zfO435TX/pp6/xOBXwH/AC4BDpvm/X5e3f8pwKrA+ynH4B+BF085JqZ9bXvVP0071wOOoBynVwPfmvqa1duHUk4cr6uv8d5dj90bOJnyObgS+Gq9PyjfbX+rz/ccYOtJfif7598k/2a9Af75N9f+KMHEwcCDgVuAu3U9djGwW9ftJT+2XfcdQAkMX1J/hO/I9IHwj+sP1ibA76lBE30C4Xr7JKYEWCwbIH4B+DawVi37e+B5XW27hRI0rQq8qP5YBiVY/Qdw37rtPYD793iNTgGeWv//g/pjunvXY3vX/38OeFv9/zI/wl3P9UZKsLMq8E7g533em+7n+RfggCHf23tTUgruACysbf3QlPf3dEqQvh7wW+CF9bEnUIKkretr9WVmGAjX/dwG7DLk85h6PD2aEqxsV5/bR4BTprxuJ9TnNF2At8wxNuhYqY9/E/hkfQ53ra/bC3q097C6r32A1SknG3+s/1+d8pn7b2CN+lyu6zoOLwceUf+/LrDddMcTfT6bwJ3qPrfsevwMYN/6/w9STh7Wo3xuvgO8s8lr39WWbShXWR9Qj5MnT2nHF+prdUfghZTAc6P6nH7Isp/xnq/tdPVP08bvUU461q2v76N6vGZPoxzrq1BOFv4J3KM+9hXKSe4qlJPfh9f7Hw+cBdyF8r1xv04Z//ybj3+mRkhdIuLhlMvOR2fmWZQA75kj7OqyzPxIZt6amf/qsc27M/Pvmfln4EPAfqO1eqk6MGhf4LWZeV1mXkzpeXpO12Z/ysxPZ8md/Dwl4L1bfex2YOuIuGNmXp6Z5/Wo6mTgUfUy7wOAw+vtBcBDKAFmU6dm5nG1PUfSOx1gqg0owTAAEfHiml97fUR8eroCmXlhZp6QmTdl5hWUXv9HTdns8My8LDP/TgmItq33Px04IjPPzZKScFjjZ9jbupRAo/t5vKc+j39GxOsb7udZwGcz85eZeRPwWuChNc2n4531eOt1PE5n2mMlIu5GOXk5JDP/mZl/owST+/bZ11mZeUxm3kJ53RcAO9W/NYF3ZebNmfkj4Lss/TzcAiyKiLUz8+rM/OUQ7QcgM2+gnBzuByW3HNgKODYiAjgI+K/6+lwHvGPAc5m6/5My85zMvD0zz6YEkVOPq8Pqa/UvyrH04cxcnJlXA+/qbDTia7tERNwD2J1yAnd1Zt6SmSf3aPfX6rF+e2Z+lXLFoJNzfgvlu3DDzLwxM0/tun8tyusXmfnbzLy8SdukuchAWFrW/sAPMvPKevvL9b5hXTLkNn+i9MzM1AaUHqA/Tdn3PbtuLwm6aoAAsGYN7p5B6a26PCK+FxFb9ajnZErv0naUS6MnUH74dwIuzMyrhmjzX7r+fwOwoGEe5VWUwKzzXP5flvzaD1Feg+VExN3q4LpLI+IflPSKDQa0Z836/w1Z/j2bqaspJx/dz+PV9Xl8k9Kb2cSG3e3JzOspr0/3+97kmJxq2mOFEiCtTjlOromIayg9mHfts68l9Wfm7cDi2u4NgUvqfR3dx+xTKYHhnyLi5Ih46AjPA8pnuRNcP5OSLnAD5crAnYCzup7L/9b7G4mIHSPix3XQ5rWUz9DU46r79Z96LHX/f5TXttvGwN9rgD2o3c+NiF931bN1V7tfTenxPT0izouIfweoJyr/D/go8LeI+FRErN2wbdKcYyAsVRFxR0pPzaMi4i8R8RdKHuEDI6LTS5lTik29Pej+bht3/X8TymVnKJcn79T12N2H2PeVLO3J6d73pQ3aQ2Yen5mPpQRmFwDT9qwCPwPuC+wNnJyZ59d69qAEydPuvkkbhnAi8JQhy7yjtmObLIPrnk35sW/icpZ/z2aknnz8guGfx1SX0fWeR8SdgfVZ9n3v9/oP+95cAtxEyRO/S/1bOzPv36fMktcuIlahpAVcVv82rvd1LDlmM/OMzNyLEgh+i5JTPspzOAFYGBHbUgLiL9f7r6TkFt+/67msk5lr9tjPdPV8mZJasXFmrkPJMZ56XHWXu5zy/Du6j6tBr+2g53kJsF5E3KXfRhGxKeXz/WJg/XrydW6n3Zn5l8x8fmZuCLwA+FhnxpbMPDwzHwwsAu5DyZ2X5iUDYWmpJ1PyNRdRLodvS8l/+wnw3LrNX4Hu+XqvoPTojTKH76siYt2I2Bh4GSWnD+DXwCPrPKjrUC5zd5vahiXqJeyjgbdHxFr1x+7llJ7Pvmpv6V41iLqJMhjn9um2rT1pZ1EGYHUC359ResJ6BcJ/Bdavz2kcDgMeEREfiIh71uewAeU962UtyvO6tpYZ5gf8aOCAiFgUEXcC3tSgzGoRsaDrb7qe6lcD/x4Rh0bEXevz2AjYfIi2fQU4MCK2jTLl3zuAX9TUmCaGOo7rpfAfAO+PiLUjYpWI2CIipqYDdHtwRDyl9vYfQjnGfk45EbgBeHVErB4RuwD/BhwVEWtEmYN7nZpS8Q96HJP0+VzUNt8CfI0yMHU9SmDc6Z3+NPDBrtf/nhHx+D71bBQRa3TdtxalF/bGiNiBwelURwMvq/XcBXhNVzsHvbbT1c+U8t+nBK7r1tf0kdNsemdKUH1Ffc4HUnqEqbefFkunIby6bnt7RDyk9oCvTjlpv5He74k05xkIS0vtT8kB/XPtDflLZv6FchnwWfUH/J3A6+ulxFfWgPDtwE/rfTsNUd+3KcHkrymDWz4DkJknUILis+vj351S7sPAPlHmnD18mv2+hPIDdRFl1P+Xgc82aM8qlKD5MsrMA4+iDJDq5WTKJdzTu26vRY/84My8gBKwXVRfqxmlgmTm74EdKT1rv4mI6yizdVwG9FrE482UdI5rKa/5N4ao7/uUtIsfUQZ3/ahBsY9Tehs7f0dMs99TKQPEHgn8vuvS/EmUQW9N2vZDynP+OqW3cQuGy3Ed5Th+LmVw2/mUQOkYulI8pvFtSurN1ZSc9afU/NWbKYHv7pTe2Y8Bz63HC3Xbi2sqywsp+dDTWeaz2WObLwO7AV/LzFu77n8N5T39ea3nh5QrHtP5EWVmi79ERCeF6mDgLfUYfCO9e607Pk0Jds+mzDZxHGWAbWfO436v7XT1T/UcypWhCyizOxwydYN6Fef9wGmU4Hobyuen4yHALyLiekpv98uyzFW+dm3/1ZQUlqsoJxdExH9HxPcHPHdpTumM/pUkaSIi4jDK7BrPnu22zEURsTvwiczcdODGksbKHmFJklagiLhjROwREavVFJ03UQZHSlrBDIQlSVqxgpKmczUlNeK3lJQKSSuYqRGSJElqJXuEJUmS1EoGwpIkSWqlpqsWjd0GG2yQm2222WxVL0mSpJY466yzrszM5VaMnLVAeLPNNuPMM8+creolSZLUEhHxp+nuNzVCkiRJrWQgLEmSpFYyEJYkSVIrzVqOsCRJkuaeW265hcWLF3PjjTfOdlOGtmDBAjbaaCNWX331RtsbCEuSJGmJxYsXs9Zaa7HZZpsREbPdnMYyk6uuuorFixez+eabNypjaoQkSZKWuPHGG1l//fXnVRAMEBGsv/76Q/VkGwhLkiRpGfMtCO4Ytt0GwpIkSZpTHvawh62QeswRliRJUk+bHfq9se7v4nc9ceA2P/vZz8ZaZy/2CEuSJGlOWXPNNQG4/fbbOfjgg9lqq6147GMfyx577MExxxwztnoMhCVJkjQnfeMb3+Diiy/m/PPP58gjj+S0004b6/4NhCVJkjQnnXrqqTztaU9jlVVW4e53vzu77rrrWPdvICxJkqRWcrCcNIv6DUC4eMEz+xc+7Noxt0aSpLll55135vOf/zz7778/V1xxBSeddBLPfOaA38chGAhLkiRpTnrqU5/KiSeeyKJFi9h4443ZbrvtWGeddca2fwNhSZIk9dRkurNxu/766wFYZZVVeN/73seaa67JVVddxQ477MA222wztnoMhCVJkjRnPelJT+Kaa67h5ptv5g1veAN3v/vdx7bvgYFwRHwWeBLwt8zceprHA/gwsAdwA3BAZv5ybC2UJElSa5100kkT23eTWSM+Bzyhz+O7A1vWv4OAj8+8WZIkSdJkDQyEM/MU4O99NtkL+EIWPwfuEhH3GFcDJUmSpEkYxzzC9wQu6bq9uN4nSZIkzVkrdEGNiDgoIs6MiDOvuOKKFVm1JEmStIxxBMKXAht33d6o3reczPxUZm6fmdsvXLhwDFVLkiRpZbPmmmsCcNlll7HPPvtMrJ5xTJ92LPDiiDgK2BG4NjMvH8N+JUmSNNsOG98CFmV/zVdG3XDDDTnmmGPGW3+XgT3CEfEV4DTgvhGxOCKeFxEvjIgX1k2OAy4CLgQ+DRw8sdZKkiSpNS6++GK23rrM3nvDDTfw9Kc/nUWLFrH33nuz4447cuaZZ85o/wN7hDNzvwGPJ/CfM2qFJEmS1MfHPvYx1l13Xc4//3zOPfdctt122xnvc4UOlpMkSZJGceqpp7LvvvsCsPXWW/OABzxgxvs0EJYkSVIrGQhLkiRpztt55505+uijATj//PM555xzZrzPccwaIUmSJE3UwQcfzP7778+iRYvYaqutuP/9788668xsRgsDYUmSJPU2xHRn43L99dcDsNlmm3HuuecCsGDBAr74xS+yYMEC/vCHP7Dbbrux6aabzqgeA2FJkiTNeTfccAO77rort9xyC5nJxz72MdZYY40Z7dNAWJIkSXPeWmutNeN5g6dysJwkSZJayUBYkiRJyyjrpc0/w7bbQFiSJElLLFiwgKuuumreBcOZyVVXXcWCBQsalzFHWJIkSUtstNFGLF68mCuuuGK2mzK0BQsWsNFGGzXe3kBYkiRJS6y++upsvvnms92MFcLUCEmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVDIQlSZLUSgbCkiRJaiUDYUmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EqNAuGIeEJE/C4iLoyIQ6d5fJOI+HFE/Coizo6IPcbfVEmSJGl8BgbCEbEq8FFgd2ARsF9ELJqy2euBozPzQcC+wMfG3VBJkiRpnJr0CO8AXJiZF2XmzcBRwF5Ttklg7fr/dYDLxtdESZIkafxWa7DNPYFLum4vBnacss1hwA8i4iXAnYHdxtI6SZIkaULGNVhuP+BzmbkRsAdwZEQst++IOCgizoyIM6+44ooxVS1JkiQNr0kgfCmwcdftjep93Z4HHA2QmacBC4ANpu4oMz+Vmdtn5vYLFy4crcWSJEnSGDQJhM8AtoyIzSNiDcpguGOnbPNn4DEAEXE/SiBsl68kSZLmrIGBcGbeCrwYOB74LWV2iPMi4i0RsWfd7BXA8yPiN8BXgAMyMyfVaEmSJGmmmgyWIzOPA46bct8bu/5/PrDzeJsmSZIkTY4ry0mSJKmVDIQlSZLUSgbCkiRJaiUDYUmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVDIQlSZLUSgbCkiRJaiUDYUmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqpUaBcEQ8ISJ+FxEXRsShPbZ5ekScHxHnRcSXx9tMSZIkabxWG7RBRKwKfBR4LLAYOCMijs3M87u22RJ4LbBzZl4dEXedVIMlSZKkcWjSI7wDcGFmXpSZNwNHAXtN2eb5wEcz82qAzPzbeJspSZIkjVeTQPiewCVdtxfX+7rdB7hPRPw0In4eEU8YVwMlSZKkSRiYGjHEfrYEdgE2Ak6JiG0y85rujSLiIOAggE022WRMVUuSJEnDa9IjfCmwcdftjep93RYDx2bmLZn5R+D3lMB4GZn5qczcPjO3X7hw4ahtliRJkmasSSB8BrBlRGweEWsA+wLHTtnmW5TeYCJiA0qqxEVjbKckSZI0VgMD4cy8FXgxcDzwW+DozDwvIt4SEXvWzY4HroqI84EfA6/KzKsm1WhJkiRpphrlCGfmccBxU+57Y9f/E3h5/ZMkSZLmPFeWkyRJUisZCEuSJKmVDIQlSZLUSgbCkiRJaiUDYUmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVDIQlSZLUSgbCkiRJaiUDYUmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKjQLhiHhCRPwuIi6MiEP7bPfUiMiI2H58TZQkSZLGb2AgHBGrAh8FdgcWAftFxKJptlsLeBnwi3E3UpIkSRq3Jj3COwAXZuZFmXkzcBSw1zTbvRV4N3DjGNsnSZIkTUSTQPiewCVdtxfX+5aIiO2AjTPze2NsmyRJkjQxMx4sFxGrAB8AXtFg24Mi4syIOPOKK66YadWSJEnSyJoEwpcCG3fd3qje17EWsDVwUkRcDOwEHDvdgLnM/FRmbp+Z2y9cuHD0VkuSJEkz1CQQPgPYMiI2j4g1gH2BYzsPZua1mblBZm6WmZsBPwf2zMwzJ9JiSZIkaQwGBsKZeSvwYuB44LfA0Zl5XkS8JSL2nHQDJUmSpElYrclGmXkccNyU+97YY9tdZt4sSZIkabJcWU6SJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVDIQlSZLUSgbCkiRJaiUDYUmSJLWSgbAkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWql1Wa7AW2x2aHf6/v4xQue2fvBw64dc2skSZJkj7AkSZJayUBYkiRJrWQgLEmSpFYyEJYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWsmV5aQZ6rdqYN8VAwH48ngbI0mSGrNHWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVVpvtBkhqh80O/V7fxy9e8MzeDx527ZhbI0mSPcKSJElqKQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVmoUCEfEEyLidxFxYUQcOs3jL4+I8yPi7Ig4MSI2HX9TJUmSpPEZGAhHxKrAR4HdgUXAfhGxaMpmvwK2z8wHAMcA7xl3QyVJkqRxatIjvANwYWZelJk3A0cBe3VvkJk/zswb6s2fAxuNt5mSJEnSeDUJhO8JXNJ1e3G9r5fnAd+fSaMkSZKkSRvrynIR8Wxge+BRPR4/CDgIYJNNNhln1ZIkSdJQmvQIXwps3HV7o3rfMiJiN+B1wJ6ZedN0O8rMT2Xm9pm5/cKFC0dpryRJkjQWTQLhM4AtI2LziFgD2Bc4tnuDiHgQ8ElKEPy38TdTkiRJGq+BgXBm3gq8GDge+C1wdGaeFxFviYg962bvBdYEvhYRv46IY3vsTpIkSZoTGuUIZ+ZxwHFT7ntj1/93G3O7JEmSpIlyZTlJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVVpvtBkiSltrs0O/1ffziBc/s/eBh1465NZK0crNHWJIkSa1kICxJkqRWMjVCklYSplVI0nDsEZYkSVIrGQhLkiSplUyNGMKMLjvy5fE2pqF+be7fXubdpVIvC0uSpGHYIyxJkqRWskd4HphRr+5864m2Z1aSJK0g9ghLkiSplQyEJUmS1EqmRkgtZOqKJEn2CEuSJKmlDIQlSZLUSgbCkiRJaiVzhKV5aj5OqzfftGlBGklqI3uEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWsnBcpIkrcQc9Cn1ZiCsOaXfFzYM+tJ2JgQtz9k1JEm9mBohSZKkVjIQliRJUiuZGiFJEzB6SsbspGPMJC1psxt7t9kcVElzmT3CkiRJaiV7hCVJs8YZDSTNJgNhSVLrjByAG3xLKxVTIyRJktRKjXqEI+IJwIeBVYH/ycx3TXn8DsAXgAcDVwHPyMyLx9tUSZLmrxnNk25P9ErL9KDZNbBHOCJWBT4K7A4sAvaLiEVTNnsecHVm3hv4IPDucTdUkiRJGqcmqRE7ABdm5kWZeTNwFLDXlG32Aj5f/38M8JiIiPE1U5IkSRqvJqkR9wQu6bq9GNix1zaZeWtEXAusD1w5jkZKkiRNkqkr7RSZ2X+DiH2AJ2Tmf9TbzwF2zMwXd21zbt1mcb39h7rNlVP2dRBwEMAmm2zy4D/96U/jfC4aszblLfkF2MxsvU5tOhbV3GwcF237DMyk3pEXWplQe2erXt+fuSEizsrM7afe36RH+FJg467bG9X7pttmcUSsBqxDGTS3jMz8FPApgO23375/BC5J1cXvemKfR+fnl7IkafY1CYTPALaMiM0pAe++wNRThWOB/YHTgH2AH+WgrmZJkiSNbEadBAN6z9tiYCBcc35fDBxPmT7ts5l5XkS8BTgzM48FPgMcGREXAn+nBMuSJEkrTP/AECZ1BcmrVvNXo3mEM/M44Lgp972x6/83Ak8bb9MkSZKkyXFlOUmSJLVSox5haWU3W5fTJEnS7LFHWJIkSa1kICxJkqRWMhCWJElSK5kjLEmal5yyavKcp1YrOwNh9eSPjCRJWpmZGiFJkqRWskdYkiSNnVcVNR/YIyxJkqRWMhCWJElSK5kaIakxV+CTJK1MDIQlSWrIk0Fp5WIgLEmS1DIOZizMEZYkSVIrGQhLkiSplQyEJUmS1EoGwpIkSWolA2FJkiS1koGwJEmSWslAWJIkSa1kICxJkqRWMhCWJElSK7mynCRJc5yrgEmTYY+wJEmSWslAWJIkSa1kICxJkqRWMhCWJElSKxkIS5IkqZUMhCVJktRKBsKSJElqJQNhSZIktZKBsCRJklrJQFiSJEmtZCAsSZKkVjIQliRJUisZCEuSJKmVDIQlSZLUSpGZs1NxxBXAn1ZQdRsAV67gsrNRp2Xndp2Wndt1tq3sfGtv28rOt/bOx7Lzrb3ztexcsWlmLlzu3sxc6f+AM1d02dmo07Jzu07Lzu0621Z2vrW3bWXnW3vnY9n51t75Wnau/5kaIUmSpFYyEJYkSVIrtSUQ/tQslJ2NOi07t+u07Nyus21l51t721Z2vrV3Ppadb+2dr2XntFkbLCdJkiTNprb0CEuSJEnLMBCWJElSK3vChJoAACAASURBVK022w3Q9CLiHsDfM/Om2W6L5g6Pi94iYl1gS2BB577MPGUF1X33zPzLiqhLc99sHotqzs+twB7h5UTENyLiiREx9GsTxbMj4o319iYRscOITTkSuCAi3jdi+UYiYv1J7r9HnTs3uW/APu40vhYNrOv9EXH/FVXfAI2Pi4h4WZP7Buxj3Yh4wDBlusquEhFrj1J2hLr+AzgFOB54c/33sAblVo2IH4+hCcc13bDWecEY6hxJRDwgIvaMiKd0/lZAnXeLiCfVv7tOur5a54y/Z0asd6RjsZYdus0RcWT9d6jPdlf5OzS5b66JiDtGxH1nuJvGn9ta5+bT3PeQGbahSb2rTrqONlspB8tFxLENNvt7Zh4wTdndgAOBnYCvAUdk5u8a1vtx4Hbg0Zl5v9or8IPMHOmDEhEBLMrM8/ps8w7gPZl5Tb29LvCKzHx9wzr+D/g1cATw/RzigIiI9wBvA/4F/C/wAOC/MvOLA8r9MjO3G3Rfj7IPA/4HWDMzN4mIBwIvyMyD+5S5Duj5vDKzb7BWf9gOpFxBOQL4SmZe26Ct5/SoN0q1OWqAOfC4qNtN9zr/KjMfNKDcScCelOd7FvA34KeZ+fIGbfsy8ELgNuAMYG3gw5n53gZl7wA8FdiMrqtVmfmWBmXPAR4C/Dwzt42IrYB3ZObAIC8iTgSe0uQ97bOPga/rlO2/DbwkM/88Ql33AT4O3C0zt64nKntm5tsalP0s5XN6HuW7Csqx+O8Nyt4JeAWwSWY+PyK2BO6bmd8dUO7pwHuBkyjH/iOAV2XmMQ3qvBvwDmDDzNw9IhYBD83MzzQoO5PvmYXA81n+WGzyOs3kWBy6zRFxPrAb8H1gF8prvERm/n0Cdfb9LsjMDwyoc6bl/w14H7BGZm4eEdsCb8nMPfuVm2Y/w35ufwn8W2ZeWm8/Cvh/mblNnzIfof/vz0sb1HsR8HVKPHJ+0/b22V/fnvCIOLzBbv7RNM6Y61bW1Ij7Af/R5/EAPjrdA5n5Q+CHEbEOsF/9/yXAp4EvZuYtffa7Y2ZuFxG/qvu6OiLWGOkZlPJJ+cHqZ/fM/O+uMldHxB5A0wP0PpQv0X8HDo+Io4HPZebvG5R9XGa+OiL2Bi4GnkLpCZk2EI6IhwIPAxZO+SJcG2h6xvtB4PHAsQCZ+ZuIeGS/Apm5Vq3/rcDllF7VAJ4F3GNQhZn5P8D/1N6HA4GzI+KnwKczs19v4pMGP53hDTouImI/4JnA5lNOCtcC+v4oVutk5j/qCcAXMvNNEXF2w+YtqmWfRflhPpQSTA8MhIFvA9fW7YdN/bgxM2+MCCLiDpl5wRC9RdcD50TECcA/O3c2+YHq8ulhGgusC5wXEadPqbPJD/mngVcBn6xlzq4nIAMDYWCnzFw0ZFs7jqC8Nw+tty+ldBb0DYSB1wEPycy/wZIg84fAwEAY+Fyt93X19u+BrwI9A+Exfc98G/hJbedtDct0DH0szrDNnwBOBO5FeX+W7JYSgN2rR513B+4J3DEiHsTSAHptYNAVt7UGPD5Ip/x9KScNne+pfwNOb1D+MGAHyskVmfnr6XprGxj2c/sC4Fs1EN8OeCewx4AyZ47QrqkeCOxL+R1aBfgscFRm/mPE/X0GeGKfx/cC3jhgH4fSPM6Y01bWQPh1mXlyvw0i4s19HlsfeDbwHOBXwJeAhwP7U864e7mlXsLIup+FLO11mZRV65ftTbXOOwKNL2vVoOoE4ISI2JUSxB4cEb8BDs3M0/oU7xw/TwS+lpnXls7KntYA1qzlur9I/wHsM0SbL5lST9Mfqj0z84Fdtz9en+egD3zn0tRW9e9K4DfAyyPiBZm5b492/qlhu8btZ5SAfwPg/V33Xwc0CWhXi5KL/HSWBiBNrR4RqwNPpvSU3DLgmOi2UWY+Ycj6OhZHxF2Ab1GO5auBpq//N+rfyDLzY0MWecMMqrtTZp4+5XW9tWHZ0yJi0Yi9Sltk5jPqiRaZeUM0e3NX6QTB1VU0T8vbIDOPjojX1jpvjYhBn/dxfM/cKTNf03DbqUY5Fkduc2YeTunE+DglKO50DJySmb/pU/TxwAHARkB3D+x1wH9PV6Crzp6/n010ykfEKcB2mXldvX0Y8L0Gu7hlmt+boS9vD/u5zcwzIuKlwA+AG4HdMvOKAWU+P2y7ptnHdZSg/dO1F/rLwAcj4hjgrZl54ZD76xcEA3xwULvr1eeVwkoZCGfm0VPvq2/aNZ1L/9NtU7f7JuUs9UjKJZDL60NfjYhBZ3aHA98E7hoRb6d8gU36jOlLwIkRcUS9fSDQ+IM3Jej/K/ASytn5tpTenn5n2d+Nkuv4L+BFNfC/sdfG9eTk5Ij43AyCxEuipEdkDbheBvy2Ydl/1p7KoyhfmvvR1RvXS0R8kNJTcSLlEmenx+LdETEwbSaWTc1YA1gd+OeglIxR1df2TyztuRvWWyh5jT+tX/z3Av6vYdlPAH+kBNynRMSmlF7eJn4WEdtk5jnDNjgz967/PSxKzu86lHSdJmU/X08gN8mGaVAzlZkn19dmy8z8YU07aNpbeWVEbMHSE+59KCc+TXyBEgz/hdLrPkyazs31derUuwXNeu6/HxHHA1+pt59B89zMf9bvqE6dOzHgeBrT98x3I2KPzBwqh7TWP/SxOKY2X0DpyPgG5X09MiI+nZkf6VHn54HPR8RTM/Pro1QYEQuA5wH3Z9mBgQNTSKq7ATd33b653jfIeRHxTEpH0JbASykdABMREd9h2UD7TpTj8DMR0ehKTv19fA2wiGVfq0c3KLsqpcPpQEq6zvspv/2PoHyW7tP0uTSRmR+q9W6QmVf222ZlsLLmCL8ROLpekroD5UvogZRek2fW9IdeZXcdcLl7UN1bAY+hfBGdmJlNg7SRRcTutU6AEzLz+CHK/p4S9B+RmYunPPaazHz3gPLrAddm5m31x3ztfrlHtcwJwNNy2bzmozLz8Q3auwHwYUo6R1DOzF+WmVc1KLtZLbsz5Uvtp8AhmXnxgHIHUo6n5YLmiFgnh8gtrT1oe1EuUR/atNwwIuLUzHx4LJ8b3Ql6JjaALSLe1HUzKT1/q2bmwB7QKLmOWwIXMXyQNrIYU77hkHU+HzgIWC8zt6g/5p/IzMcMKEo9MfkU5VL61ZQTj2cPOo5r2QuBlwPn0HW1qknwFRGPpZzYL6J87nYGDsjMkwaUezfwC8pVNSgpBzs16XGNiO2AjwBbA+cCC4F9MnPglY0ahC73A9cw8LgOuDMlMLuZFfDZqfXeB3gly+cmN2nz2ZT86X/W23cGTmvy+YmIJ7J8MNskN/9rlAD8mZQT6GcBv83MRgP3IuJ1lCtP36x3PRn4ama+c0C5O1GuVj2O8t4cT+kZ7dkRMxO1F7anQVeg6z5+QEnreSVlHMX+wBUNPwcXAT8GPpOZP5vy2OFDpnENVL8TP0uJmW4Dnj613pXJyhoInwdsnZkZEQdRev52o5w1fT4zl5vJIQaMnM7MgZdOoySYHzWfDpiIiJzBQVB7Zzdj2S/tLwwos9wAhenu61F24aBLUeNSf4R7ysxfzmDfQw3SWJFiZoOxXtF1cwElT/q3TXqIag/pupReDij55tdMOsUkIs4CHg2c1HlPIuLczNx6gnX+mpLj+IuuOs/JPoNuptnHnSlpB9cNUea0zBz1SkHnCtJOlODj5716i6aUmW4w1tlNT3AiYjXKVboAfpf9x2l0l3tw180FlIGYt2bmq5uUnw1RUrU+Qcn1XZICkpln9Sy0tOw5lFzsG+vtBcAZg46piPgEpYdzV8pA5H2A0zPzeQ3q/FVmPqjzftardD/JzJ0Gle3ax3Z0feYz81dNy65oUfKQL+96je9I+Z68uEHZszLzwd3HfkSckQ0G00fEwzPz1Cn37ZyZPx3piQyu72xK8HtBROxIGZDf92RgPlspUyOAm7uCu8dTgtPbgN/WL9Xp/Fuf/SXNcgjPAl4fZWDEN2u940iUX85Me/26L/XENGl+DS/1HAlsQZl1ovOlnZTLr/3cHhGbZB0xXwOgpsH4TyPiYsqZ9dc7vcpNxPAjwd/f434o7R3YS1Pr7T7JWgXYnj4pJHPAyIOxMnOZ1yzKNG9Nr1A8mTLIdcml3dqWaS/tjtF0+YaTzu2/KTNv7tRZv5f6fgaix0j7zj5ywEj76lf1vfwOXWkNDU/09wZ+lJnfq7fvEhFPzsxv9dj+RcDBwL1i2cGWa1GuxgwUEf8JfCnrDClRpvPbLxvkdk4TPP40yuDEJvV2BtNunplvjYiNgXvk0rSoSbk1Mz8+YtkjgF9ESe+D8nkaOLsG8LAaxJ6dmW+OiPdTBro20TkpuSYitgb+AgycHq9eSey4uP4teSx7zHQRER/KzENi+VQFoPFg05n4GuVKTMdt9b4mM0N1XqvLaw/8ZcB6fbbvdjhlcF63j0xz37jcmpkXAGTmLyJipoMj57SVNRC+qX4o/0o5y31l12N3nq5AZh4400pzac7VepTeh3fXgG/Lme57mroeXv8d9QAdx/zE21NmCRi2R/l1wKkRcTIsmU7poCYFM/M+UeZm3hd4Xb2cflQOmLKtGmokeGbu2qRNDXSfZN1K+dLfa0z7noSZDMZabl+UwThNPI9yybxzaffdwGlMPhBeofmG1ckR8d+UEfuPpQSM3xlQZqYj7QHuSAmAH9d1X9MT/TdlZifIIjOvqakw0wbClAE936eMrO9OA7quV6Azjedn5pIZfrLMivN8YGAgPCXYWgV4MCVft4mPUafCBN5KmVnkozQLeGbiOxFxMKUjpftEZeDrlZkfiDL1YScF5cCGvav/qv/eEBEbUgYzDpxNp/pUlNS211OOxzVpMPiY0mmUlO//TSgpPgHcBfgzvcemHFn/nej8+n2slplLcprryWzTmaHeFmU2qldQvtPWBg7pVyDGMwPKKO46pb5lbjc86Z43VtZA+GWUqXkWUkY//hEgyrRi017O7tXb0jHkG39vyuwCm9J8INdIIuLIzHzOoPumapLT1MC5wN1pPlCnU/f/1sthnctnhzS5xNpV/nTg9ChzKH+AMjiwSSA88kjwEVNA3l3r+372GJw5R408GCuWnTt5VcpncGCuYac4y56g3Fbvm7SXUE7ObqIM6DqeEvxM0qGUwP8cypRMx2Vm36mccuYj7Wd6wj/dTA89f0Oy5M5fS0lNG9Wq3elbUQYNNQ08uoOtWym51AMv91djnQpzCPvXf1/VdV/PKdCmqulaw6ZsfTfKLBfvrWWTkiLRpL7Odqc0bWMttzlARHwa+GbWQYlRxrs8uU+5s+q/4/j9GsUVEbFnZh4LEBF7UWYRauLqrs/ErrX8oAVexjLT0gg+PaW+qbdXKitljnA/0WOEbCw7yGc52WC6mCgLTOwN/IFy6f6bw1y6H8XU/Lt6ifXsHDBXaPRe7AGAJvl7UQajbEvpjeruveh7eSp6zPubDZYgjbJS2d6UHuEtKD0nRzfMoXsb8LMcciR4rxSQHDBAob7GDwDOmpojOZfF9IOxntUkV7emuXTcCvw1Mxv1JteT0f1ZduDM53IlGp3cEREvy8wPD7qvR9nfAQ/IpVMm3oHymR84b3JEbETpjer8AP+EMth0ce9SS8p+FriGpXOw/ydlsN8Bg8qOKiLeS+lQ+GS96wXAJZn5it6lxlLvLyjH/xk1IF5IWRxpTub1j0s9lhZkwwHAMfMFnZbLi5/uvmnKnZ51rE9EPC0zv9akvpmqHQRfAjaknGBdAjw3G0xfNvW3utd9Pcpu2uT7V6NpYyD858zcZEL7fgElb7Vx7+YM6notZa7HOwI3dO6mjHD+VGa+dkD5Tfs93jDomTZ5ftDZes3v6lhAGTR0VjYbGf1HyqXYo7P/HMfTle2MBL+Jkq/VNJ/6t4yQAlJ/xJ9POaO/ofuhJvXOloh4cGaeFV2DsSLiSTlgBbEx1b0dXbMLNLy0O2pd0+YZdkwy37DHj2LTAaMjjbSvZU+gpCx0LjE/m3KS89gGZe9Mmf94t3rXCcDbcprZVMYlyuIBL6BrVhzgf7KM+RhUdnXgRSydV/ck4JPZYLBdlGkWn0HJwfw8dSrMSQVcEfHozPxR9Bi0nQ1yuGdY/9BXvGq56QY+Nwru6rbHU07GOlf0ngU8MnvMIBQRP6NcRXk88ATKtI5nrOiOhohYEyAzr2+wbSe94RDKglAdawN757Jz208tOys50RFxdGY+vf6/c2Wz89gPMvNxvUvPP20MhC/JzI37PD70vIgRsVWW0ZXTfhhzBrMLDBIR7xwU9M51UQaifCgzn9pg25nOcrEeZYqu7vd2UOD+NeCluXRO6WHr/HZmzuWc4GVEWUb0uZl5br29L2Xp7B1nt2Xj1etErmMSl19j6ap/D6cEAB1rA7dlg+nT6n5GGmkfEb/OzG0H3bcyiIj/oczZ3ZlX/TmU17jfqqPd5VfYVJgR8eYsKzh25oPvfMd1Tpqbzss7St0jXfGqZc+mzFTRvaDTmZl5/4Z1rwe8ia5FQIA3Z+/BcgFsQ0kFOoHyXb4NZaaNkzOz6SC/oUTEszPzi71SKLNP6mT9ntmFMmXaJ7oeug74Tmb2nKO9q1NipE6nUXWf4Exz1XnOzng0qpU1R7ifQUHUkZR5ER9P17yIA8q8nDLYa7pZBhrPLjCKzHxtvRw1NbgbmGYAEGWC+o9QlqVeg5LX2Xexhxj/PLWLa/392vmhzDwEODYiRjozjrJk8Msog7d+TclR/hlLe5umbt85C18LOD/KiPPGKSBdLphm38ucZc8x+wDHRBlA9gjguSw7uGqlMKkfkQFGXvUvItbOsnz1egwx0n6KqyLi2Sxd3GI/yuCogWIGc9yOquZQHkZJj1iNpd8xTfJRHzKlt+1HUaYna1JvZyrMjw7ceAwys5Oa9yLKQOvNWPoaT7q3atRBzzDDBZ3qMdtozuHqs5Rg+R+dk4P6nn6f8l01kUCYpYPsh86TzRkslpKzlxPd71hY6XpPV8oe4T75rwHcJzN7LkEcM5gXMSIW5JQJvae7b5x6BHenNf1xirJa3r6UKWC2pwQ995lkL3NEfISl788qlDzjizPz2X3KzPjMuB4XD6HMf7pt7fF5R2ZOezlyXD2GPS6DN55HdTbUoOdblBHce2fmvwYUmXcGfE/kJN+fmmbwr8y8vb7WW1EGVfa8bB8R36XMEHEbXUEwQwSHNSXqI5RVB5MSmL8061SGA8qOPMftqKKsXPlf09TZZAGdX1IW7vlDvX0v4Jgml9EjYn9KasTEp8KcUu//UvKwf8myvbMTG6U/hiteQy/oNOol//pZeQTwHkoHw02UBV5eBJyaK2iO+VHM5EQyVnBOdP3c7Uf5ff4i5SpW1L8vZmbfjqv5ZmUNhEfOf+0ccFFGZh9MmRfx9IY/MiMnw49q2OBumvJnZub2sewk3xO99FF/ZDpupQTBE5kYfEq9Z2TmQ6IsZrBjZt4UEec1vYw3Qn2deVS3ALoHU6xFWb64Z+A/G6YJDO9KGeF8EzQbQDmfzOR7Ygx1n0X5QV+XMqfuGZT5z5/VoOxIi31EmXHhC03q6FH+rMx88OAtxycifjFqSk5EPIYyt+5FlB/wTSlTijVeOTSWToW5L2UJ7rFPhTmlvoku5DKlru4rXkMPep5h3Z2OjVdSjv1ua+WA8QhdHVZ3An5FmdXgEZNKQatXCHpqmEYy9IlkzFJOdJRp+PqNnxjX1KJzwsqaGrE6ZbWXZYKrepmt7/K/LJ0X8Q00nBcxIu4O3JMyJ+iDYMmUT2tT5lGdpBsz88aIICLukCVXeeDo8S43RJkW6NdRZr24nOmnSRqL+mP8uBn8GM/kzHhxlGmCvgWcEBFXA00GBU5NAYESIJ5JGSF9UY+i45hHdUV60mw3YEXqDnQj4m4snSP29Mz824Srj8y8ISKeB3wsM99TT9CaOCsiHpKZUwOIvrIsg75pRKyRXXOhDmHkOW5n4MdRBp1+Y0qdA8ddZOaJUeaF7nwf/i5rLusQVthUmNXPImKbzDxnBdT1Pspv1btZdsqyzn09zTQ9riv4eybwv7l0PMJ+lEFlgwbmvqTu54aIuCAz38dk5xZ+CmWKxXUpM+mMYpTFUnam5EDvAbyakgJ5r4h4FxPMic7MXSax37lqZe0R/i7w2qlfJhGxDaW3tN8qcqPUtz9wACW1oPvy2XWU6Z8mNuI3yipCB1K+PB5N+ZCunpl7NCy/KWXhkTUolyDXofwwD5wOZlQRcSrw6GF+jMd9ZlzTHtahfAn3bUdEvJWSx/xlyhd9Z+q2XwIvavKlUU8A7sayl8QGXo6eTRFxV5bNO5/T7R1VRDydMofqSbBkgZdXZeYxE6zzV5SrBR8EnpeZ50XDJZbrZct7U07i/skQqRwR8QVKPv6xtSzQbJ70KDO2TNUoJWNUUaZonK7Opqlfo86G8B5KcHgRcBTwrZzgVJhdV2NWowQ7F1EC/xWRpjNrqVuddBVKQNwZj/CkHDB9W0S8PuuS77UDaNgTnGHbeT5ltpTvUwa+LTO/eZOTwSjzff+NIU4ko+RenwK8snPlsvYsv5TSAz5wtc9RRMRDKNMU/qXefi7lysifgMPmaEfOyFbWQLjn+t2DfmwiYtre38wcuChA9JijeEUZJribTaP8GEfMzmjhWvdvcsoUN1FH2k/32DTlX0wZ8PNXli7dO9Eft5mIiD0pA7k2pHxxbwr8dlIpJLOt/rA8ttMLHGXO2B8Oel9nWOcjKfmCP83Md9eA4JCGl1inTekYkPJ1ZGY+JyKuYdkpnDplB86TPt/EzGZDOJiymtxmmfmWiNgEuHtOaInl2UjT6Urduhdl7vuOoVK3YumUh0nJ0x1qysMYYjxCRLyGEhh+POtMJ5NOP6x1vJSSh3wv4NLuh2ienz/0iWTMUk50lPz63TLz7/W76ihKL/y2wP0yc5KLeaxwK2tqxF36PHbHAWW758RcQLlc3OiSWGZ+Pcoa4lOnXmu6stZIairHxpQe6OuArWm4utAMUw1G9Yf6twpLR+EOOiObrdHCUNJHnk7puYAyq0JnAGSTM8lDgPtmgwE+c8RbKYMuf1jz8HalzDe7slplSirEVUwwPQiWzOpyStftiyi9PE3KjhIUPTjK8rl/ZsQlq2s+5sspubIHddIOBuVzzkRNWXkHsGFm7h4Ri4CHZuZnGhSfyWwI27B0ieW3UL5Xv86EllieRKDbwIxTt2rH0dNYukT35yLia4N6KmP58QjrUWYs+kVE9BuPcEGt714R8ZN6e/2IuG9m/q5Jm0eRmYcDh0fExzPzRSPuo9ey0f3K/B74fUS8ODN37sqJ3oJyFXpS03Ku2nUMPIOyNsHXga8PkcI1b6ysgfCZEfH8nLJkaZQZFvqOcM7MZaZAi4j3UZZcHSgiPkHJCd6VskTlPpQBCBNTL9sfQLmUtqS3kQFTtnWlGtw1ygC7/wNeS5k9YtLOnxpwR8TTBpR5JyXgvXtE/JRyZnw3yhfoJ/sVHINnAR8GPkZ5bX8OPDvKnJkvblD+EkpO8XxxS2ZeFRGrRMQqmfnjiFjpVneDJVcazogysX9nSrFnAEOtPjhCvQspOX9TT5onNRXZJ4ATgc1ZNn0raL6E7xGU78+H1duXUr4vJrnQyudqva+rt39PWbWzSSA80hLw1WwtsbzC5HiWwH4W8MCsMyPV3NVfA4Mu2Y86HuEaykJSu9S/+1Gmdjy0BsMP61105kYNgmHGJ5IrOid61YhYLcuqoI+hTA/bsdLFjSvdE6oOAb4ZZXWgTuC7PSUPdu8h93UnytRkTTwsy5RrZ2fmmyPi/Uy2pxLKClNbjJAKMStJ+NV0AXffIHwWz4w7vXW98spP7VUulk6+fhFwUkR8j2VzwyY2JdIMXRNl5aRTgC9FxN8ol4lXOpmZEbEDZUBsZ0W7T2XmN/sUG4cvUQK6J1Em2t8fmNjUT+Po0aJ8zzwjyoCmzo9yDCo0Qxtk5tFRVtIkM2+NiL6rysV45v++peb1Z93nQpZ2NGipyygncp0rZHdg2dSBac2gB/zxlM/qFsAHKHNv/zMzDxxxfyvSTE4kd2Hpb83Tx96y5X2FMvfxlcC/qIv/RMS9mV+dOo2slIFwZv4VeFi9pNuZiuZ7mfmjQWWnXLJZFVhIuTTWRCe36YZ6GfIq4B6NGz6acympIMOOcl/hqQZR5pvcA7hnLDsdzdqUadSaWNFnxp0BC9PNdTlotadO2sef698a9W+u+w1lSej/ovT4rEOZPWVldRZlYMi0q0ZNyPqZ+ZmIeFkunXB/qFkgRjGTHi3g5noVpBMcbkFXgDkh/4yI9bvq3InBP8Tj+D44nDKo6a4R8XbqEstj2O/K5lrgvChLdyfwWOD0zvd7k3zsYWTmf8OS36ojKUtgL4wyAPvqHPNA+DEb+kSyKyd6H5b2sp9Ged4Tk5lvj4gTKfHLD7rSi1ah/gavTFbKQLiTPJ9lvshp54zsk2DffcnmVuCv9fJAE9+NMj3Xeyk5uklJkZikdwK/iohzGa7XYzZSDS6jXJbdk2VTVK6jBF1N7MKKPTOGZc/YF1CuKlw2qNA8HoC0a2beTukB+zyUUeSz26SJ2hF4VkR0ZmEAJj5vcmfhjMvruILLKJ+9uexNwP8CG0fElyhXlQ6YcJ0vpwyq3aJ+Ty2kBAU91RMLYprVGyPi3cDAhXAy80tR5nruLLH85JzgEsvz2DfrX8dJK6je47MscHJmRLwoy1RuG6ygukc1yonkrORE94qP6pXZvtvMRyvrrBH/ouS89twEWCczN5mm7BbA4iyLLewCPIAyCf1QU+dExB2ABTlgGpiZiojzKIHrOXRdusvmq56t0InJa52rZ58VtHqUmZXRwj3asgplxG6jfLSYfvWkzjzEn8wJrjw4jJhnC4CMS4wwC8MY6nwS5XLjxpTBa2sDb87MYydV5zjU3tmdKN+hP8/MK1dAnatR5gIOylzAjb47UcE1mgAAD1BJREFUpvt+iDm+ouN8ExH/RrnaOmtpIxHxwMxstHT2bIqIx1KuKiwCfkA9kczMk/qUeRTwC8oKkA+h5ER/D/gRJb94IjnRM4mh5qOVNRDuOxVNdVtmLp6m7K8p+cSbUQbMfBu4fzafl3ekeStHFX2mimtY/uGZeWr9/7cnGQB31Tn0TBURsRfwKOA/KJfuL6AMknjcJM+Me7TlvpQv/3s33P7DlJ6s7sFY/6AEx2tn5nMm0tAhRcQ6lAnj58sCIFqBImJv4Eedk/t69WuXzPzWBOp6dGb+KCKmXSEz+8zNHmOaFkyDRcQXKct1fx34bGZeMMtNmtOGPZGMiHdQrlhtTxk4ejZlEadFE27nyDHUfLRSBsIz0elFiIhXA//KzI9EwyWHYwbzVs6gvR+gXF45liFXXqrlV9jE5FFmqjiXEsAOtSjGbJ0Z17o7qyd1Rtj/hbJgS6M5o6c7WYmlyz1PbIlnzV0RsRGlJ7gz/+pPgJfN5R+WqHNnT7lvIsuxR8SbM/NNNT9/quyXn+8J3YoVEWtTZp44kHIsHwF8JTOvm9WGzUER8QCW7ygbuOBWzYl+HiU3+O3A75j7OdHzxkqZIzxDt9Rk9ueydKaA1RuWncm8laPq/Ajt1HVfk+nTZiMJvzNTxe4MP1PFrI0Wzsy1Bm/V15oRsUnWldmiTM7fGXw2Zxc+0UQdQZnHtTNt4LPrfY+dtRYNNt3cyhP5DalB8CrA9zPz6OGL58UR8Z9TH4iI9QyGxysz/xERx1Dm6D+EMobiVRFxeGaONGf1yigiPktJtTyPZac6bbLy7HzMiZ43DISXdyBlOqO3Z+YfI2JzyujUJmYyb+VIMnPXEYvORhL+yDNVzPZo4SirrT2y3jwph1tE4BXAqRHxB0qv8ubAwRFxZ+pgNLXOwszs7u38XEQcMmutaebMegXqo/X2fzJgXvaZyMzb65W5YQPhL1MGPZ/F0is5S3ZLszmT1UBNWTuAsuT3F4AdMvNvdczJ+Yy4eMtKaqdRUxoy89VdNw+o9008P78tTI0Yo4j4MWUJwlHmrRy1zpFWXpqNVIMYw3KREfGezpdC10C/DSb5pVB7rB9CmfsVymXAMzrBecN93AHYqt783VwZIKfZUacmOoKleeP7AQdm5mNmr1X91RO3NwC71btOAN6Wmf/sXWrGdb4LuJIy53L3jB4De3Vr/urJwE/MXZ2MiPgq8NEsKyV27nt3Zr4mIh6TmSfOYvPmlIj4DPD+zDx/ttuiZRkIVxFxdGY+PZZf+rGzlvjAkcY1uFxO0xkcRhER36euvJSZD6wjrH+VmdsMKDcrSfi17rHMVLGiRgtHmTps287I6Pj/7d17rFxVFcfx749HELDQAuUhgoEGBaVgWh4SeTSgxEIlxLZpBGkx4BsMryYSQSoiUkQMYni0IIoBKTUSqNDQP3g/pFgKoQoJjzZRRKjSQAPIc/nH3tM7vb23M/fOOXNm7vw+yWR6z5xzzybczKzZe+21UqH95Y3+JlrZ8GMjW96MciVpo1GQvpCeHhH/qHRgTZA0ivSeWHqTFUkrBzgcEdFwVlepjvxh+TGOVNLywYi4othR9i5X5mhejg/uIO0xeYchxBZWLgfCmaRdIuJlSWeTWuiut2klqukF31Ddpqt1m1YG2tSykevbnoRfRaWKVuRAeFJtFkrSdqT0iEaB8LA3/NjIJul3wBkRsSb/vB1wWSf/TUgaT1r+rtU7/g8wKyJWVDeqjctfWg8ktb3/NmkD9N4bv8oacWWOoZP0PKkudv9Spx0ZW/QS5whnEVHL6/0oMA94jbQctzBSp7pBSXooJ6/Xqguseyn96timjDFnw+m8VK+KJPxJtL8pRisuJjUtuZf0//Rw1t+NPqCIuCA/d0P7T2uv/WpBMKSlfkmFV18o2LXAWZEaFaFUZ30efS1jC5dXjc4Cdo+Ib0rai5S61TBHP6efbE3aBPwgcGBEDLUDpw3sZtKeDlfmaN7q6PA64b3KM8KDyGVOZgBTSQ02vtDgkkpImkBaYt2XtFlvLDAtIobcCazsVAN1UFOMZuWd69PIH6T58NKI+HcT1260ZW9EXN76CK0b5ZWYSf1mhO9vlNJUJUlPRcT+jY4VfM8FpE1vMyNi3xwYP9LMipekXwITScvQD5Peex6NiLfLGq/ZYCRdBYwGFrH+HiKnyFXMM8KDe5WUy/NfYMdmLsgfZv2tjSF2URuKiHgi5x4NufPSAL+r7HzbStpFtqK2cz2XcBrqt/lWy67ZyPUL4FFJtWYy00mpSZ3sRUnn01dF52vAiyXfc1xEzMglLYmItySp0UX53DNhXU7zyaS9FDsDW5Q0VrON2ZIUAB9dd6zZ8mlWIs8I9yPpu6Tl+rHAQuDWZnd5SlpFapm6hhSUjiYF068A34iIUkoNqc3d7IarikoVRWhl57rZYHKFl1q973s6fTe5pDHAj0lNQCCtksypT/Eo4Z6PAEeR8k4nSBpHatZwUBPXnkbaKDcRWJXH+2BE3FPWeM2s+zgQ7kfSz4AFEfHkMK6dD/wxIu7OPx9NSq24AbgiIg4udLBU081uuKqsVNGKVnau5+s/CVwN7JSXd/cDjovc0c/MBibpi8B5pDKLS0hNeU6OiPuauPYcUvC7LCLeL3OcZoPJK4qXSrqS9fcQAdCJn9W9xoFwgSQ93T/Hr1ZKZiiVHIZ4z2dofze7llRRqaJKku4HZgPX1lX2WBER+1Y7MrPm5S9057Dh6tNGu1gWcN/tSZ0zBfwl3EjAuoikL0fEIkmzBno9ItxUqWLOES7Wy3lD2C355xnAK7mEz4eDX9aStnezK0BXtYuUtDmp6ce6znKkoLbZXOytImJpv9RGz1BZt1kIXANcR9/qUzscQUrHCFK7+9vaeG+zlkTEovzPtyJiYf1rkqYPcIm1mWeEC5SDuQvoy6F7mJRT9zqp/M/zBd5rEemDYRRt7mZXpHY1xWiFpOtIH8C1b+4nAR9ExKlNXr8YOI1Uim+CpGnAKRExuZQBm5VA0rKImNjme15Fat9b68A3A3ghIr7XznGYtWqQ5iMdXTWpVzgQ7lJ545mAuUB9H3IBc8vIR+5VrZaNkrQnffVW1wArgRNdSN26iaQ5pGo6t7H+l+7SNo1KehbYp5b6lcsZ/i0i9inrnmZFkjQZOIa0CX9B3UvbkNIaG278tHI5NaJAksaSgtLPAB+pHS8jhy5y22ZJm0e/Fs6Stiz6fj3uA0njIuIFWBfYNlwa7ldH+C7gXmATUuWJqYDrCFs3qeU4zq47FqTuYmV5HtgdqH1p3C0fM+sW/wL+ChxHqoldsxY4s5IR2XocCBfrJtI3vimkdp6zgNVl3Ki+xWVuAVwzipSSYcWZDdwr6UXSjPsngGa6xdXqCH+KVC7u9nz9SaRUFrOuERF7VHDbUcAzkpaSgu6DSHsL7shj6ooUMOtdOfXvKUk3l9lTwIbPqREFquXQ1SpF5GOPR8SBja4dxr22BcbgFpdtIWkLUkALqWnJOxs7v9+1DwDHRsTa/PMo4M6IOHzjV5p1lnbXLM8pYIPqvxpm1qkkfR6YQ5pI2Yw0KdJ0GU4rj2eEi1X7tveypGNJSyIDdZtrWUS8TtqE99Uyfr9tYCJ9AcBnJQ0lANgJeLfu53fzMbOuMVjNcqDM5j2r+zcakTSpmTrCZh3melIqxDLaW3XFGnAgXKyL8kzt2cCVpGR45wB1uQICgBuBpZJqZZ+OJzUUMesmB9D+muW3SroR+Dlp38WleRyHtHEMZkV4PSIWVz0I25BTI8waKKJpiaQJpHavAA9ExPJCBmfWJpIWAt+PiLbVLJe0NakyzkRSvvBNpKo4ZdVlNyuFpEuATYE/sX7VlScqG5QBnhEulKQ9gNPZMIfOGzq6W8tNS/Kbnd/wrJvtAPw9b1xrV83y94C3gS1JM8IrHQRbl6qVND2g7lgApXZmtMY8I1yg3Dr4euBp6jrJeUNHdxopTUvMijDYxrUy39/ye+rtwIXAWFJnu3cjwh25zKwQDoQLJOkxN7IYOdy0xKxakg4iVWvZIyIulLQ7MDMiLqp4aGZDImkn4GLgYxExWdKngUMi4vqKh9bzHAgXSNIJwF7AEpwDNGIM0hpzXYk8s5FM0kMRcaiktaQVknUvkco/bVPiva8mra4dGRH7SBoDLCmjJKVZmSQtBm4AfhgR+0vaDFgeEeMrHlrPc45wscaTmiUcSV9qhHOAupSblphBRByan0c1OrcEB0fEBEnL8xjWSNq8gnGYtWqHiLhV0rkAEfG+JJdR6wAOhIs1HdgzIt5teKZ1g5uBxbhpiVlV3pO0KXkmOrex9zKmdaM3JW1P39/y50i9AKxiDoSLtQIYDbxa9UCsdW5aYla5XwG3ATtK+ikwDTiv2iGZDctZwB3AOEkPkzZ/Tqt2SAYOhIs2GnhW0uO4uoCZWUsi4iZJy4CjSDnJx0fEMxUPy2w4xgGTgd2AqaRyao7BOoA3yxWoivJCZmZm1tlqG6wlHQr8BLgM+JGrD1XP30YK5IDXzMzMBlDbGHcsMD8i7pTkMoAdYJOqBzASSHooP6+V9EbdY62kN6oen5mZmVXqJUnXAjOAuyRtgWOwjuDUCDMzM7MSSdoK+BLwdEQ8J2kXYHxELKl4aD3PgbCZmZmZ9SRPy5uZmZlZT3IgbGZmZmY9yYGwmVkbSVolKQZ4rBrk/Pvy6zu0eahmZiOey6eZmbXX6cDWwBTgROAa4H7gzSoHZWbWizwjbGbWRhGxKCJuAZ7Mhx4D7gamSlqdHzdKGtP/Wklfz7PDv1FyrqSVuVTj3ZL2zOfNyefNl/Rc/p3T2/dfaWbWHRwIm5lV7wpgFvBb4AbgpHys3hRgHvAH4FRgJnAxKZC+BNgPWNjvmsOAXwPb5nPMzKyOUyPMzKp3DPBSRMwGkHQCMLnfOfNJKRQzI+JDSVPy8Rn5AbCzpO3qrrk8IuZJ+g6wV3nDNzPrTg6Ezcy6w6vAAcDewIq64yfm1yCt8r1V99pr+fl9vAJoZrYBvzGamVXvTmBXSXMlzQV2Be7qd8404ANgsaSPA3/Ox2cBuwFHAOdHxP/aNGYzs67nGWEzs+qdkZ9Pyc+/rztW8xzwFWAJKUg+DPgB8C3gauCfwILSR2pmNoK4xbKZmZmZ9SSnRpiZmZlZT3IgbGZmZmY9yYGwmZmZmfUkB8JmZmZm1pMcCJuZmZlZT3IgbGZmZmY9yYGwmZmZmfUkB8JmZmZm1pP+D7vuZZZIMja5AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T-Q_DbqzBDL_",
        "colab_type": "text"
      },
      "source": [
        "**Plot for both target classes**\n",
        "\n",
        "Attributions assigned to tokens may take opposite values when computed with regard to class 0 and class 1.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GMpxiV2fBD1W",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 454
        },
        "outputId": "e3f289f2-5b5e-425b-dc5d-47070e33c2f3"
      },
      "source": [
        "# Attributions for word_embeddings: index 0\n",
        "lig_0 = layer_attributions_0[0].squeeze().sum(1)\n",
        "lig_1 = layer_attributions_1[0].squeeze().sum(1)\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(tokenizer(text)[\"input_ids\"])\n",
        "\n",
        "plt.rcParams[\"figure.figsize\"] = [12, 6]\n",
        "plt.bar(list(range(len(lig_0))), lig_0, color='r', alpha=0.5)\n",
        "plt.bar(list(range(len(lig_1))), lig_1, color='g', alpha=0.5)\n",
        "plt.xticks(list(range(len(lig_0))), tokens, rotation='vertical')\n",
        "plt.legend(labels=[\"Target: negative\", \"Target: positive\"])\n",
        "plt.xlabel('Token', fontweight='bold')\n",
        "plt.title(\"Token attributions for positive and negative target class\")\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAAG1CAYAAAA/RKjWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzde7xd853/8ddHiCAaIapuEVV1zY3jmtG6005bRhGlrVRbtD+l7aA6tDLUjA6DMjWGutUtlCnplBFthbaoJBXiOoKUuBRBpVKV6Of3x1rn2Dn2OWevs0/OyYnX8/HYj+x1+a7vd6+99j7v/V3ftRKZiSRJkqTGLdfXDZAkSZL6G0O0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoqU+FBE7R8Tcvm5Hd0XELRFxaPl8QkT8pge3fUhETOmp7TVY5yYRMTMi5kfE0b1Zd6MiYqeIeKyT5cMj4s8RMaA321VVRGREfKiv29GViHgoInbu63YsTfrLeyctaYZoqYeUwaX18beI+EvN9CF93b6O1AvyETExIq7sqmxmfiwzL++BNowo/zAvX7PtqzJzz2a3XdHxwO2ZuWpmntvLdTckM3+dmZu0TkfEnIjYvWb505k5ODPf7psW9l8RcVlEfK92XmZukZlTe7iedx3vvamv65eWFYZoqYeUwWVwZg4GngY+WTPvqr5uX0+KwrL4/bEB8FB3ChpI1Fs81qSlw7L4R1BaqkTEihFxTkQ8Vz7OiYgVO1j36Ih4OCLWK8udGRFPR8QfI+KCiFipXG/niJgbEf8YES9GxPMR8YVO2vCFiHikHKbwZEQcUc5fBbgFWKem1/xg4J+A8eX0/eW6UyPitIj4LbAA+GA570uLVxX/ERF/iohHI2K3mgWL9Zi26+2+s/z3tbLOHdoPD4mIHSNiWrntaRGxY82yqRFxakT8tnyNUyJiWLlsUERcGRHzIuK1suxadfbRr4BdgP8o2/DhiBgSET+OiJci4g8RcVLrj4eyfb+NiLMjYh4wsc42J0bE9RFxbdmu30fE6Jrlm5Vtf60cNvCpmmUfL4+F+RHxbEQcW/vel8+vAIYDPyvbfHxtL2NEjI+I6e3a9I2ImFw+7/AYq/NaNoqIX5X78eWIuCoiVmv3/h4bEQ+U79G1ETGoZvlx5XH6XEQcVq+ORt7Pcvn2EXFXud/uj5rhFhGxYUTcWZb7RUT8sOY4IyJ+EhEvlG28MyK2KOcfDhwCHF/uy5/VvK7dI2KdKM4urV6zrbHlvlihnD4sis/ZqxFxa0Rs0MFLrHe8N7J/vxURDwBvlO/v58vjcl5EfCdqPmMRsVxEnBART5TLr6tp+7vqr/MeDIiIfyrLz4+IGRGxfp31/j4i7ouI1yPimYiYWLOsw89eFJ+fJ8ttPxVL8dk6qUOZ6cOHjx5+AHOA3cvnpwD3AO8H1gTuAk4tl+0MzC2ffxf4PbBmOX02MBlYHVgV+BnwrzXlFpXbXgH4OEWwHdpBe/4e2AgI4KPlulu1b0PN+hOBK9vNm0rRw74FsHxZ71TgS+XyCWWbvlEuGw/8CVi9/T5pXwcwAkhg+ZrlE4DflM9XB14FPlfW/Zlyeo2atj0BfBhYqZw+vVx2RLnvVgYGAFsD7+tgP7W9nnL6x8BN5f4fAfwf8MV2r/drZZtWqrO9icBCYP9ynxwLPFU+XwGYTfGDZSCwKzAf2KQs+zywU/l8aEfvV5392rYvy9c8H9i4Zvk04KCujrE6r+VDwB7AihTH8Z3AOe3acS+wTrm9R4Ajy2V7A38EtgRWAa4u2/ihTt6Hjt7PdYF5FMf8cmWb5vHO5+Zu4Mxyn/4d8Do1xzJwWPlaVwTOAWbWLLsM+F4nn+VfAV+uWXYGcEH5fJ/y/dys3PcnAXd18Pra3qOK+3cmsH65TzYH/ly+xoHla15Y09ZjKL531iu3+V/ANR3VX6eNxwGzgE0ovjdG887nre29ozgeR5bvxajyfd63s89eeQy8zjvH+trAFn39ve3DR9VHnzfAh49l8dHuD+8TwMdrlu0FzCmf7ww8C5wF/AYYUs4P4A1go5pyOwBP1ZT7S7s/wi8C2zfYvhuBY2q21WiIPqXOvNoQ/RwQNcvvBT7Xfp+0r6PeH3UWD9GfA+5tV/fdwISadpxUs+yrwP+Wzw+j+OEyqoH9Uvt6BgBvAZvXLD8CmFrTvqe72N5E4J6a6eUow3H5eAFYrmb5NcDE8vnTZX3va7fNxd6vOvt1sX0JXAl8t3y+MUWoXrmrY6yBfbUvcF+7dny2ZvrfeCdgXkIZgsvpD9N1iO7o/fwWcEW79W8FDqXolV8ErFyz7EraHcs1y1Yr29H6ubuMzkP0l4Bf1XxGnwE+Uk7fQvkDq+a9XgBsUKfexd6jCvv3sJrp71KG4nJ6ZYrjtbWtjwC71SxfmyJkL99g/Y8B+3SwrLP37hzg7M4+exQh+jXg09T58enDR395OJxDWvLWAf5QM/2Hcl6r1YDDKXoA/1TOW5Pij+KM8jToa8D/lvNbzcvMRTXTC4DB9RoQER+LiHsi4pVyWx8HhtVbtwvPdLH82czMmun2r7W72u/D1m2vWzP9Qs3z2n1xBUXImlQOJfi31tPvXRhG0Vvc/r2rrbOr/bHYOpn5N2AuxetZB3imnFdv+5+meJ/+EBF31Dvl3qCrKXruAQ4GbszMBTR2jLWJiLUiYlIUQ0tepwin7Y+hjt6DdVh8X7V/L+vpaFsbAAe0trls999RhMR1gFfK19eqrd5yiMLp5RCF1ymCKXVeR0duAHaIiLWBjwB/A35d064f1LTpFYqgvW7dLbXT4P6t3YeL7dPyNc+rWb4B8NOa9jwCvA28ayhTB9an6ADoqt3bRcTtUQx5+hNwZE276372MvMNijNVRwLPR8TPI2LTBtslLTUM0dKS9xzFH7RWw8t5rV4FPgFcGhHjynkvU/Q0b5GZq5WPIVlctFhJFOOvb6A43btWZq4G3EzxBx6KXqX26s3rbH6rdSMiaqZrX+sbFKGt1QcqbLf9Pmzd9rNdlCMzF2bmP2fm5sCOFPv6812Vo3gPFvLu9662zq7aDUUYAYpxqhSn158rH+vH4hdotm0/M6dl5j4Uw4BuBK7rYPtdteE2YM2IGEMRpq8u51c9xv6lrGtkZr4P+CzvHENdeZ6a/UDxOrvrGYqe6NVqHqtk5ullPatHRO1xVlvvwRTDLnYHhlD0yELnn4U2mfkqMIUiAB4MTKr50fgMcES7dq2UmXfV21SdeY3s39pyz1McS8ULKMayr1Gz/BngY+3aMygzn+3qddaU36iB9a6mGBK0fmYOAS5obXdnn73MvDUz96D48fMocFEDdUlLFUO0tORdA5wUEWtGcXHUdyl6mdpkcQutQ4D/johty97Ji4CzI+L9ABGxbkTs1Y36B1KMiXwJWBQRHwNqbx33R2CNiBjSbt6IqH4HjvcDR0fEChFxAMX40JvLZTOBg8plLRTjhFu9RNGr98EOtnsz8OGIOLi8oGo8xZjQ/+mqQRGxS0SMjOK+ya9TBOO/dVGMLG4Rdx1wWkSsWl4k9k3avXcN2Doi9ovijgpfB/5KMVb1dxQ9rMeX+2Rn4JMUvXYDo7hP9pDMXFi2u6M2/5GO9xtl+Z9QjN9dnSJU041jbFWKMbh/ioh1KcbMNuo6YEJEbF4G3JMrlG3vSuCTEbFX2bM8KIqLLdfLzD8A04GJ5T7cgWKf1r6Gv1L02K5MEVxrdbovS1dTBMH9eecHCRTh8dvxzoWKQ8rPQD31jveq+/d6iv2wY0QMpBg6VBu6L6A4djco27NmROzTSf3t/Qg4NSI2jsKoiFijznqrUvT+vxkR21L8uKCss+5nr+x13yeKC5v/Wr7uLj+T0tLGEC0ted+j+MP+AMWFOr8v5y0mM2+jGEP4s4jYimLs52zgnvL07i8oLvKpJDPnA0dTBJlXKf7ITa5Z/ihF0H+yPPW7DkXoApgXEb+vUN3vKMbdvgycBuyfma2nmL9D0bP1KvDP1ASQ8lT0acBvyzZs3+41zKPoxfpHigB0PPCJzHy5gTZ9gCJwvE5xSvsOitPMjfgaRQ/6kxRj1q+mGN9bxU0UPZetF0buV/bQvUUR8D5Gsb/OBz5fvh+U684p3/sjKX5k1fOvFD/SXovyDh51XE3R+/qTdkOAqhxj/wxsRXGx6M+B/+78Zb8jM2+hGCv7q7K+XzVats62nqHoTf4nijD4DEXgbP17dgjF2O55FJ+zaymCGhQXiv6Borf/YYofM7UuBjYv9+WNHTRhMsUx/kJm3l/Trp8C36f4EfQ68CDFe1vvNdQ73ivt38x8iOL4nETRK/1niusiWl/rD8q2TomI+eVr3a6T+ts7i+I7YwrFZ+diigsa2/sqcEpZx3dZ/IxJR5+95Sh+kD5HMezlo8BXOnu90tIoFh++KEnqKVHc7utDmfnZvm7Le1VEXAs8mpnN9H4v9SJiMMXFehtn5lN93R7pvcCeaEnSMiMitoninsvLRcTeFL3WHfUq92sR8cmIWLkcFnEmxZmuOX3bKum9wxAtSVqWfIDiFnl/Bs4FvpKZ9/Vpi5acfXjnItWNKe7/7ellqZc4nEOSJEmqyJ5oSZIkqSJDtCRJklTR8n3dgO4YNmxYjhgxoq+bIUmSpGXYjBkzXs7Muv+Ta78M0SNGjGD69Ol93QxJkiQtwyLiDx0tcziHJEmSVJEhWpIkSarIEC1JkiRV1C/HREuSJPWlhQsXMnfuXN58882+bop6wKBBg1hvvfVYYYUVGi5jiJYkSapo7ty5rLrqqowYMYKI6OvmqAmZybx585g7dy4bbrhhw+UcziFJklTRm2++yRprrGGAXgZEBGussUblswqGaEmSpG4wQC87uvNeGqIlSZL6mXnz5jFmzBjGjBnDBz7wAdZdd9226bfeeqtH63rttdc4//zze3SbzTjnnHNYsGBB2/THP/5xXnvttV5vR2Rmr1farJaWlvQ/W5EkSX3lkUceYbPNNntnxsSJPVtBhe1NnDiRwYMHc+yxx3a57qJFi1h++WqXxM2ZM4dPfOITPPjgg5XKLSmt/+nesGHDenS773pPgYiYkZkt9da3J1qSJGkZcNFFF7HNNtswevRoPv3pT7f11k6YMIEjjzyS7bbbjuOPP54nnniC7bffnpEjR3LSSScxePDgtm2cccYZbLPNNowaNYqTTz4ZgBNOOIEnnniCMWPGcNxxx3XahhEjRnDyySez1VZbMXLkSB599FEA3njjDQ477DC23XZbxo4dy0033QTAggULOPDAA9l88835h3/4B7bbbru2/5X6K1/5Ci0tLWyxxRZtbTn33HN57rnn2GWXXdhll13a6nz55Zc54YQT+OEPf9jWlokTJ3LmmWd2+LqaZYiWJElaBuy3335MmzaN+++/n80224yLL764bdncuXO56667OOusszjmmGM45phjmDVrFuutt17bOlOmTOHxxx/n3nvvZebMmcyYMYM777yT008/nY022oiZM2dyxhlnADBmzJgO2zFs2DB+//vf85WvfKUtxJ522mnsuuuu3Hvvvdx+++0cd9xxvPHGG5x//vkMHTqUhx9+mFNPPZUZM2a0bee0005j+vTpPPDAA9xxxx088MADHH300ayzzjrcfvvt3H777YvVO378eK677rq26euuu47x48d3+LqaZYiWJElaBjz44IPstNNOjBw5kquuuoqHHnqobdkBBxzAgAEDALj77rs54IADADj44IPb1pkyZQpTpkxh7NixbLXVVjz66KM8/vjjdeuaOXNmh+3Yb7/9ANh6662ZM2dO27ZPP/10xowZw84778ybb77J008/zW9+8xsOOuggALbccktGjRrVtp3rrruOrbbairFjx/LQQw/x8MMPd/r6x44dy4svvshzzz3H/fffz9ChQ1l//fUrva4qvE+0JEnSMmDChAnceOONjB49mssuu4ypU6e2LVtllVW6LJ+ZfPvb3+aII45YbH5rEG7UiiuuCMCAAQNYtGhR27ZvuOEGNtlkk4a28dRTT3HmmWcybdo0hg4dyoQJExq6Bd0BBxzA9ddfzwsvvMD48ePb6q73uppliJb6qYlTJ1Zbf+dq60uS+pf58+ez9tprs3DhQq666irWXXfduuttv/323HDDDYwfP55Jkya1zd9rr734zne+wyGHHMLgwYN59tlnWWGFFVh11VWZP39+U23ba6+9OO+88zjvvPOICO677z7Gjh3LuHHjuO6669hll114+OGHmTVrFgCvv/46q6yyCkOGDOGPf/wjt9xyCzvvvDNAW3vqXVg4fvx4vvzlL/Pyyy9zxx13dPq63v/+9zf1mnpkOEdEXBIRL0ZE3cs2o3BuRMyOiAciYquaZYdGxOPl49CeaI8kSdJ7zamnnsp2223HuHHj2HTTTTtc75xzzuGss85i1KhRzJ49myFDhgCw5557cvDBB7PDDjswcuRI9t9/f+bPn88aa6zBuHHj2HLLLdsuLOxsTHQ93/nOd1i4cCGjRo1iiy224Dvf+Q4AX/3qV3nppZfYfPPNOemkk9hiiy0YMmQIo0ePZuzYsWy66aYcfPDBjBs3rm1bhx9+OHvvvXfbhYW1tthiC+bPn8+6667L2muv3enralaP3OIuIj4C/Bn4cWZuWWf5x4GvAR8HtgN+kJnbRcTqwHSgBUhgBrB1Zr7aWX3e4k6yJ1qS+lK926H1FwsWLGCllVYiIpg0aRLXXHNN290yetvbb7/NwoULGTRoEE888QS77747jz32GAMHDuz1tlS9xV2PDOfIzDsjYkQnq+xDEbATuCciVouItYGdgdsy85WyobcBewPX9ES7JEmStLgZM2Zw1FFHkZmsttpqXHLJJX3WlgULFrDLLruwcOFCMpPzzz+/TwJ0d/TWmOh1gWdqpueW8zqaL0mSpCVgp5124v777+/rZgDF+Ob+Orqg39ziLiIOj4jpETH9pZde6uvmSJIk6T2st0L0s8D6NdPrlfM6mv8umXlhZrZkZsuaa665xBoqSZIkdaW3QvRk4PPlXTq2B/6Umc8DtwJ7RsTQiBgK7FnOkyRJkpZaPTImOiKuobhIcFhEzAVOBlYAyMwLgJsp7swxG1gAfKFc9kpEnApMKzd1SutFhpIkSdLSqqfuzvGZLpYn8P86WHYJ0HeXhUqSJPUz8+bNY7fddgPghRdeYMCAAbQOd7333nt79A4Xr732GldffTVf/epXe2ybnZk+fTo//vGPOffcc5k6dSoDBw5kxx13BOCCCy5g5ZVX5vOf/3yvtKUz/o+FkiRJTap67/4ut9fFvf3XWGMNZs6cWaw7cSKDBw/m2GOP7XK7ixYtYvnlq8W/1157jfPPP7/XQnRLSwstLcWtmadOncrgwYPbQvSRRx7ZK21oRL+5O4ckSZI6dtFFF7HNNtswevRoPv3pT7NgwQIAJkyYwJFHHsl2223H8ccfzxNPPMH222/PyJEjOemkkxg8eHDbNs444wy22WYbRo0axcknnwzACSecwBNPPMGYMWPa/sfCjowYMYLjjz+ekSNHsu222zJ79mwA5syZw6677sqoUaPYbbfdePrppwH4yU9+wpZbbsno0aP5yEc+AhTB+ROf+ARz5szhggsu4Oyzz2bMmDH8+te/ZuLEiZx55pk8+uijbLvttm31zpkzh5EjRwLFfbA/+tGPsvXWW7PXXnvx/PPP99AeXpwhWpIkaRmw3377MW3aNO6//34222wzLr744rZlc+fO5a677uKss87imGOO4ZhjjmHWrFmst956betMmTKFxx9/nHvvvZeZM2cyY8YM7rzzTk4//XQ22mgjZs6cyRlnnAF0/t9+DxkyhFmzZnHUUUfx9a9/HYCvfe1rHHrooTzwwAMccsghHH300QCccsop3Hrrrdx///1Mnjx5se2MGDGCI488km984xvMnDmTnXbaqW3ZpptuyltvvcVTTz0FwLXXXsv48eNZuHAhX/va17j++uuZMWMGhx12GCeeeGKTe7Y+Q7QkSdIy4MEHH2SnnXZi5MiRXHXVVTz00ENtyw444AAGDBgAwN13380BBxwAwMEHH9y2zpQpU5gyZQpjx45lq6224tFHH+Xxxx+vW1frUJJ6PvOZz7T9e/fdd7fV2VrX5z73OX7zm98AMG7cOCZMmMBFF13E22+/Xen1HnjggVx77bXAOyH6scce48EHH2SPPfZgzJgxfO9732Pu3LmVttsox0RLkiQtAyZMmMCNN97I6NGjueyyy5g6dWrbslVWWaXL8pnJt7/9bY444ojF5s+ZM6dSOyKi7vN6LrjgAn73u9/x85//nK233poZM2Y0XM/48eM54IAD2G+//YgINt54Y2bNmsUWW2zRFt6XJHuiJUmSlgHz589n7bXXZuHChVx11VUdrrf99ttzww03ADBp0qS2+XvttReXXHIJf/7znwF49tlnefHFF1l11VWZP39+w+2o7R3eYYcdANhxxx3b6rrqqqvahmY88cQTbLfddpxyyimsueaaPPPMM4ttq7O6N9poIwYMGMCpp57K+PHjAdhkk0146aWX2kL0woULF+uR70mGaEmSpGXAqaeeynbbbce4cePYdNNNO1zvnHPO4ayzzmLUqFHMnj2bIUOGALDnnnty8MEHs8MOOzBy5Ej2339/5s+fzxprrMG4cePYcsst2y4s7GxM9KuvvsqoUaP4wQ9+wNlnnw3Aeeedx6WXXsqoUaO44oor+MEPfgDAcccdx8iRI9lyyy3ZcccdGT169GLb+uQnP8lPf/rTtgsL2xs/fjxXXnklBx54IAADBw7k+uuv51vf+hajR49mzJgx3HXXXRX2YuOiuIVz/9LS0pLTp0/v62ZIfarq7ZS6ul2SJKlxjzzyCJtttllfN6NbFixYwEorrUREMGnSJK655hpuuummHtn2iBEjmD59OsOGDeuR7fWmeu9pRMzIzJZ66zsmWpIk6T1kxowZHHXUUWQmq622Gpdc4v951x2GaEmSpPeQnXbaifvvv3+JbLvqRYj9mWOiJUmSpIoM0ZIkSd3QH68rU33deS8N0ZIkSRUNGjSIefPmGaSXAZnJvHnzGDRoUKVyjomWJEmqaL311mPu3Lm89NJLfd0U9YBBgwYt9l+gN8IQLUmSVNEKK6zAhhtu2NfNUB9yOIckSZJUkSFakiRJqsgQLUmSJFVkiJYkSZIqMkRLkiRJFRmiJUmSpIoM0ZIkSVJFhmhJkiSpIkO0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKeiRER8TeEfFYRMyOiBPqLD87ImaWj/+LiNdqlr1ds2xyT7RHkiRJWpKWb3YDETEA+CGwBzAXmBYRkzPz4dZ1MvMbNet/DRhbs4m/ZOaYZtshSZIk9Zae6IneFpidmU9m5lvAJGCfTtb/DHBND9QrSZIk9YmeCNHrAs/UTM8t571LRGwAbAj8qmb2oIiYHhH3RMS+HVUSEYeX601/6aWXeqDZkiRJUvf09oWFBwHXZ+bbNfM2yMwW4GDgnIjYqF7BzLwwM1sys2XNNdfsjbZKkiRJdfVEiH4WWL9mer1yXj0H0W4oR2Y+W/77JDCVxcdLS5IkSUudngjR04CNI2LDiBhIEZTfdZeNiNgUGArcXTNvaESsWD4fBowDHm5fVpIkSVqaNH13jsxcFBFHAbcCA4BLMvOhiDgFmJ6ZrYH6IGBSZmZN8c2A/4qIv1EE+tNr7+ohSZIkLY2aDtEAmXkzcHO7ed9tNz2xTrm7gJE90QZJkiSpt/g/FkqSJEkVGaIlSZKkigzRkiRJUkWGaEmSJKkiQ7QkSZJUkSFakiRJqsgQLUmSJFVkiJYkSZIqMkRLkiRJFRmiJUmSpIoM0ZIkSVJFhmhJkiSpIkO0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKDNGSJElSRYZoSZIkqSJDtCRJklSRIVqSJEmqyBAtSZIkVWSIliRJkioyREuSJEkVGaIlSZKkigzRkiRJUkWGaEmSJKmiHgnREbF3RDwWEbMj4oQ6yydExEsRMbN8fKlm2aER8Xj5OLQn2iNJkiQtScs3u4GIGAD8ENgDmAtMi4jJmflwu1Wvzcyj2pVdHTgZaAESmFGWfbXZdkmSJElLSk/0RG8LzM7MJzPzLWASsE+DZfcCbsvMV8rgfBuwdw+0SZIkSVpieiJErws8UzM9t5zX3qcj4oGIuD4i1q9YVpIkSVpq9NaFhT8DRmTmKIre5surbiAiDo+I6REx/aWXXurxBkqSJEmN6okQ/Sywfs30euW8Npk5LzP/Wk7+CNi60bI127gwM1sys2XNNdfsgWZLkiRJ3dMTIXoasHFEbBgRA4GDgMm1K0TE2jWTnwIeKZ/fCuwZEUMjYiiwZzlPkiRJWmo1fXeOzFwUEUdRhN8BwCWZ+VBEnAJMz8zJwNER8SlgEfAKMKEs+0pEnEoRxAFOycxXmm2TJEmStCQ1HaIBMvNm4OZ2875b8/zbwLc7KHsJcElPtEOSJEnqDf6PhZIkSVJFhmhJkiSpIkO0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKDNGSJElSRYZoSZIkqSJDtCRJklSRIVqSJEmqyBAtSZIkVWSIliRJkioyREuSJEkVGaIlSZKkigzRkiRJUkWGaEmSJKkiQ7QkSZJUkSFakiRJqsgQLUmSJFW0fF83QF2bOHVitfV3rra+JEmSqrEnWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKvDuH1Ie884okSf2TPdGSJElSRT0SoiNi74h4LCJmR8QJdZZ/MyIejogHIuKXEbFBzbK3I2Jm+ZjcE+2RJEmSlqSmh3NExADgh8AewFxgWkRMzsyHa1a7D2jJzAUR8RXg34Dx5bK/ZOaYZtshSZIk9Zae6IneFpidmU9m5lvAJGCf2hUy8/bMXFBO3gOs1wP1SpIkSX2iJ0L0usAzNdNzy3kd+SJwS830oIiYHhH3RMS+PdAeSZIkaYnq1btzRMRngRbgozWzN8jMZyPig8CvImJWZhpWQkMAACAASURBVD5Rp+zhwOEAw4cP75X2SpIkSfX0RIh+Fli/Znq9ct5iImJ34ETgo5n519b5mfls+e+TETEVGAu8K0Rn5oXAhQAtLS3ZA+2W1I94O0BJ0tKkJ4ZzTAM2jogNI2IgcBCw2F02ImIs8F/ApzLzxZr5QyNixfL5MGAcUHtBoiRJkrTUabonOjMXRcRRwK3AAOCSzHwoIk4BpmfmZOAMYDDwk4gAeDozPwVsBvxXRPyNItCf3u6uHpIkSdJSp0fGRGfmzcDN7eZ9t+b57h2UuwsY2RNtkCRJknqL/2OhJEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKDNGSJElSRYZoSZIkqSJDtCRJklSRIVqSJEmqyBAtSZIkVWSIliRJkioyREuSJEkVLd/XDZAk9ayJUydWW3/nautLkuyJliRJkiqzJ1qS1MZebElqjD3RkiRJUkX2RPeS/ti701dtfq/VK0mS+h9D9DLOYChJAmDixCW7vvQeY4iWVIk/zCRJcky0JEmSVJkhWpIkSarI4RzSe5BDMnqH+1mSll2GaEm9xwubJGnZ8R7/TjdES5LUm97jwUNaVhiitUzx9Ll6mseU5OdAqscQrSXDnhZJUrP8W6KlmCFakpZC/a3nr5n29rfXqooMwlpG9UiIjoi9gR8AA4AfZebp7ZavCPwY2BqYB4zPzDnlsm8DXwTeBo7OzFt7ok2SpGVfUwHccCepCU2H6IgYAPwQ2AOYC0yLiMmZ+XDNal8EXs3MD0XEQcD3gfERsTlwELAFsA7wi4j4cGa+3Wy7JElaUuw9l9QTPdHbArMz80mAiJgE7APUhuh9gInl8+uB/4iIKOdPysy/Ak9FxOxye3f3QLuk3lOlh8reLEndZHjvJf3sO93jom/0RIheF3imZnousF1H62Tmooj4E7BGOf+edmXX7YE2SZIk9R8OL+p3+s2FhRFxOHA4wPDhw/u4NdVNnFqxwM79vOzOHa1U38Ta583U21dld+5opXebWPu8mTr7Y9mdO1qpvom1z5upt6mL3pqp971Ttr+1F/rweLTexuttomwzobS73+m9VWf7evvs+3HnDlerX7ba6ku9ngjRzwLr10yvV86rt87ciFgeGEJxgWEjZQHIzAuBCwFaWlqyB9ot6T3CU5fSe1Bf9NT2w95hvx+7rydC9DRg44jYkCIAHwQc3G6dycChFGOd9wd+lZkZEZOBqyPiLIoLCzcG7u2BNkmStOzphyGtP+qLYNlUnR4XfaLpEF2OcT4KuJXiFneXZOZDEXEKMD0zJwMXA1eUFw6+QhG0Kde7juIixEXA//POHJKkpZ6hRXrP65Ex0Zl5M3Bzu3nfrXn+JnBAB2VPA07riXZIfcXTYVIfMcxK6iP95sJCSXpP6W/hsJn29rfXKkkYoiVJ71GeQeod7mctqwzRWiL67EvTHi31NI8pyc+BVIchWpIkLZXsxdbSzBAtqdf4B1GStKwwREvvRZ6a7R3uZ9Xhj0ktK97rx/Jyfd0ASZIkqb+xJ1pSNfauSpJkT7QkSZJUlT3Ry7r+2GvYV23uj/tKkiT1CUN0b+mPAa0/tlmSJKkXGKIlSe/wx7MkNcQQLUnSe8B7/XZkUk8zREvSssbeZEla4rw7hyRJklSRIVqSJEmqyBAtSZIkVWSIliRJkioyREuSJEkVGaIlSZKkigzRkiRJUkWGaEmSJKkiQ7QkSZJUkSFakiRJqsgQLUmSJFVkiJYkSZIqMkRLkiRJFS3f1w2QpIZMnNjXLZAkqY090ZIkSVJFTYXoiFg9Im6LiMfLf4fWWWdMRNwdEQ9FxAMRMb5m2WUR8VREzCwfY5ppjyRJktQbmu2JPgH4ZWZuDPyynG5vAfD5zNwC2Bs4JyJWq1l+XGaOKR8zm2yPJEmStMQ1G6L3AS4vn18O7Nt+hcz8v8x8vHz+HPAisGaT9UqSJEl9ptkQvVZmPl8+fwFYq7OVI2JbYCDwRM3s08phHmdHxIpNtkeSJEla4rq8O0dE/AL4QJ1FJ9ZOZGZGRHaynbWBK4BDM/Nv5exvU4TvgcCFwLeAUzoofzhwOMDw4cO7arYkSZK0xHQZojNz946WRcQfI2LtzHy+DMkvdrDe+4CfAydm5j01227txf5rRFwKHNtJOy6kCNq0tLR0GNalfsXbtkmS1C81O5xjMnBo+fxQ4Kb2K0TEQOCnwI8z8/p2y9Yu/w2K8dQPNtkeSZIkaYlrNkSfDuwREY8Du5fTRERLRPyoXOdA4CPAhDq3srsqImYBs4BhwPeabI8kSZK0xDX1PxZm5jxgtzrzpwNfKp9fCVzZQfldm6lfkiRJ6gv+j4WSJElSRU31RKuXePGZJEnSUsWeaEmSJKkiQ7QkSZJUkSFakiRJqsgQLUmSJFVkiJYkSZIqMkRLkiRJFRmiJUmSpIoM0ZIkSVJFhmhJkiSpIkO0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKDNGSJElSRYZoSZIkqSJDtCRJklSRIVqSJEmqyBAtSZIkVWSIliRJkioyREuSJEkVGaIlSZKkigzRkiRJUkWGaEmSJKkiQ7QkSZJUkSFakiRJqqipEB0Rq0fEbRHxePnv0A7WezsiZpaPyTXzN4yI30XE7Ii4NiIGNtMeSZIkqTc02xN9AvDLzNwY+GU5Xc9fMnNM+fhUzfzvA2dn5oeAV4EvNtkeSZIkaYlrNkTvA1xePr8c2LfRghERwK7A9d0pL0mSJPWVZkP0Wpn5fPn8BWCtDtYbFBHTI+KeiGgNymsAr2XmonJ6LrBuk+2RJEmSlrjlu1ohIn4BfKDOohNrJzIzIyI72MwGmflsRHwQ+FVEzAL+VKWhEXE4cDjA8OHDqxSVJEmSelSXITozd+9oWUT8MSLWzsznI2Jt4MUOtvFs+e+TETEVGAvcAKwWEcuXvdHrAc920o4LgQsBWlpaOgrrkiRJ0hLX7HCOycCh5fNDgZvarxARQyNixfL5MGAc8HBmJnA7sH9n5SVJkqSlTbMh+nRgj4h4HNi9nCYiWiLiR+U6mwHTI+J+itB8emY+XC77FvDNiJhNMUb64ibbI0mSJC1xXQ7n6ExmzgN2qzN/OvCl8vldwMgOyj8JbNtMGyRJkqTe5v9YKEmSJFVkiJYkSZIqMkRLkiRJFRmiJUmSpIoM0ZIkSVJFhmhJkiSpIkO0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKDNGSJElSRYZoSZIkqSJDtCRJklSRIVqSJEmqyBAtSZIkVWSIliRJkioyREuSJEkVGaIlSZKkigzRkiRJUkWGaEmSJKkiQ7QkSZJUkSFakiRJqsgQLUmSJFVkiJYkSZIqMkRLkiRJFRmiJUmSpIqaCtERsXpE3BYRj5f/Dq2zzi4RMbPm8WZE7FsuuywinqpZNqaZ9kiSJEm9odme6BOAX2bmxsAvy+nFZObtmTkmM8cAuwILgCk1qxzXujwzZzbZHkmSJGmJazZE7wNcXj6/HNi3i/X3B27JzAVN1itJkiT1mWZD9FqZ+Xz5/AVgrS7WPwi4pt280yLigYg4OyJWbLI9kiRJ0hK3fFcrRMQvgA/UWXRi7URmZkRkJ9tZGxgJ3Foz+9sU4XsgcCHwLeCUDsofDhwOMHz48K6aLUmSJC0xXYbozNy9o2UR8ceIWDszny9D8oudbOpA4KeZubBm26292H+NiEuBYztpx4UUQZuWlpYOw7okSZK0pDU7nGMycGj5/FDgpk7W/QzthnKUwZuICIrx1A822R5JkiRpiWs2RJ8O7BERjwO7l9NEREtE/Kh1pYgYAawP3NGu/FURMQuYBQwDvtdkeyRJkqQlrsvhHJ3JzHnAbnXmTwe+VDM9B1i3znq7NlO/JEmS1Bf8HwslSZKkigzRkiRJUkWGaEmSJKkiQ7QkSZJUkSFakiRJqsgQLUmSJFVkiJYkSZIqMkRLkiRJFRmiJUmSpIoM0ZIkSVJFhmhJkiSpIkO0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRUZoiVJkqSKDNGSJElSRYZoSZIkqSJDtCRJklSRIVqSJEmqyBAtSZIkVWSIliRJkipavq8bIKmbJk7s6xZIkvSeZU+0JEmSVJEhWpIkSarIEC1JkiRVZIiWJEmSKjJES5IkSRU1FaIj4oCIeCgi/hYRLZ2st3dEPBYRsyPihJr5G0bE78r510bEwGbaI0mSJPWGZnuiHwT2A+7saIWIGAD8EPgYsDnwmYjYvFz8feDszPwQ8CrwxSbbI0mSJC1xTYXozHwkMx/rYrVtgdmZ+WRmvgVMAvaJiAB2Ba4v17sc2LeZ9kiSJEm9oTfGRK8LPFMzPbectwbwWmYuajdfkiRJWqp1+T8WRsQvgA/UWXRiZt7U803qsB2HA4cDDB8+vLeqlSRJkt6lyxCdmbs3WcezwPo10+uV8+YBq0XE8mVvdOv8jtpxIXAhQEtLSzbZJkmSJKnbemM4xzRg4/JOHAOBg4DJmZnA7cD+5XqHAr3Wsy1JkiR1V7O3uPuHiJgL7AD8PCJuLeevExE3A5S9zEcBtwKPANdl5kPlJr4FfDMiZlOMkb64mfZIkiRJvSGKDuH+paWlJadPn97XzZAkSdIyLCJmZGbd/wulX4boiHgJ+ENft6PGMOBly1q2B8v2t/Za1rJLU52WtezSVra/tbe/ll0SNsjMNesuyUwfTT6A6Za1bE+W7W/ttaxll6Y6LWvZpa1sf2tvfy3b24/euLBQkiRJWqYYoiVJkqSKDNE940LLWraHy/a39lrWsktTnZa17NJWtr+1t7+W7VX98sJCSZIkqS/ZEy1JkiRVZIiWJEmSKlq+rxugvhMRawOvZOZf+7otWnp4XHQsIoYCGwODWudl5p29WP8HMvOF3qpPS7++PibVGD+7yyZ7ontZRPx3RPx9RHRr30fhsxHx3XJ6eERs283mXAE8GhFndrN8QyJijSW5/Q7qHNfIvC62sXLPtaih+v49IrbozTo70PBxERHHNDKvi20MjYhRVcq0K79cRLyvu+Ur1PMl4E7gVuCfy38nNlh2QETc3gPNuLnRFcs6H+2BOrslIkZFxKciYr/WRy/Vu1ZEfKJ8vL+X6mz6+6ab9TZzTFZqc0RcUf5b6fPdbhsrNjJvaRMRK0XEJk1upuHPblnnhnXmbdNkGxqpd8CSrmNZ4oWF3RARkxtY7ZXMnFCn7O7AF4DtgZ8Al2bmYxXq/k/gb8CumblZ2QsxJTO79eGKiAA2z8yHOlnnX4B/y8zXyumhwD9m5kkN1vE4MBO4FLglKxx0EfFvwPeAvwD/C4wCvpGZV3ZR7veZuVVX8zoouyPwI2BwZg6PiNHAEZn51U7KzAc6fF2Z2WXIK/8gfoHiDNGlwDWZ+acuyszqoN4oqs1uhdNGjotyvXr7+b7MHNtFuanApyhe6wzgReC3mfnNBtt3NXAk8DYwDXgf8IPMPKOLcisCnwZGUHMmLjNPaaDOWcA2wD2ZOSYiNgX+JTMbCocR8Utgv67e0y620eW+bbf+TcDXMvPpbtT1YeA/gbUyc8vyh86nMvN7DZS9hOKz+hDF9xUUx+NhDZRdGfhHYHhmfjkiNgY2ycz/aaDsgcAZwFSKz8BOwHGZeX0DZdcC/gVYJzM/FhGbAztk5sUNlG3m+2ZN4Mu8+5hsZF91+5is2uaIeBjYHbgF2Jli/7bJzFd6us5yeaffCZl51pIoW7ONTwJnAgMzc8OIGAOckpmf6qpsu+1U/ez+HvhkZj5bTn8U+I/MHNlJmfPo/O/Q0Q3U+yRwA0U2ebjR9nayvS574CPi3AY29XqjmaM3OZyjezYDvtTJ8gB+WG9BZv4C+EVEDAE+Uz5/BrgIuDIzF3ZR93aZuVVE3Fdu79WIGFj5FbzTnqT4Q9eZj2XmP9WUeTUiPg40ekB/mOLL9zDg3Ii4DrgsM/+vgbJ7ZubxEfEPwBxgP4qel7ohOiJ2AHYE1mz3Bfo+oNFf2GcDewGTATLz/oj4SGcFMnPVsv5TgecpenMDOARYu5FKM/NHwI/KHo8vAA9ExG+BizKzo17MTzSy7aq6Oi4i4jPAwcCG7X5Urgp0+ccUGJKZr5c/HH6cmSdHxAMVmrh5Wf4Qij/qJ1CE8U5DNHAT8Kdy3arDVd7MzDcjgohYMTMfrdg79WdgVkTcBrzROrORP2w1LqqwLsBQ4KGIuLddnY0EgIuA44D/Kss8UP546TJEA9tn5uYV29rqUor3Z4dy+lmKDocuQzRwIrBNZr4IbQH1F0CXIRq4rKz7xHL6/4BrgQ5DdA9939wE/Lps59sNlmlV+Zhsos0XAL8EPkjx/rRtkiK4fbCTOj8ArAusFBFjeSeAvw/o6ozfql0sb6TsJhQ/Nlq/qz4J3NvgNiYC21L8MCMzZ9brJW5A1c/uEcCNZYjfCvhX4ONdlJnejXa1Nxo4iOJv0XLAJcCkzHy9m9u7GPj7LtbZB/huF+ucQOOZo9cYorvnxMy8o7MVIuKfO1m2BvBZ4HPAfcBVwN8Bh1L8wu/MwvJ0S5bbWpN3enqWlAHlF/RfyzpXAho+BVcGstuA2yJiF4oA/NWIuB84ITPv7qR46zH698BPMvNPRSdphwYCg8tytV++rwP7V2jzM+3qafSP26cyc3TN9H+Wr7OrLwig7VTapuXjZeB+4JsRcURmHlSnnX9osF097S6KHwvDgH+vmT8faCQMLx/F2OsDeSe0VLFCRKwA7EvRO7Owi+Oi1XqZuXc36gOYGxGrATdSHMuvAlX2/3+Xj27LzPMrFvlOE9WtnJn3ttuvixose3dEbN7NnqyNMnN8+UONzFwQDb65wHKtAbo0j8aHLQ7LzOsi4ttlvYsioqvPfU9836ycmd9qcN32unNMdqvNmXkuRSfIf1IE6taOhTsz8/4u6twLmACsB9T2/s4H/qlegZp6O/xb2pXWshFxJ7BVZs4vpycCP29wMwvr/N2pfAq/6mc3M6dFxNHAFOBNYPfMfKmLMpdXbVedbcynCPwXlb3fVwNnR8T1wKmZObvi9roK0ABnd9X28gz4UscQ3Q2ZeV37eeUb/FrrUIV665Tr/ZTiV/EVFKdqni8XXRsRjfyKPBf4KfD+iDiN4ktvSf86uwr4ZURcWk5/AWj4w9ruR8Mfga9R9AiMoehh6uxX/f9EMa7zL8BXyh8Nb3a0cvnj5o6IuKyJgPlMFEM6sgxqxwCPNFj2jbJ3dBLFF+1nqOkB7ExEnE3RQ/JLilOyrT0l34+ITof8xOLDSQYCKwBvNDKMpDvKffsH3uktrOoUivGbvy3/WHwQeLxC+QuApygC+50RsQFFD3NX7oqIkZk5q2qDM/MfyqcToxjfPIRiiFGj5S8vf4AOzwpDuJqRmXeU+2bjzPxFOVSi0R7SlyNiI975wb4/xQ+nRvyYIki/QNHjX2V40VvlfmqtdyMaP2twS0TcClxTTo+n8bGob5TfVa31bk8Xx1QPfd/8T0R8PDMrjZkt6698TPZAmx+l6Aj5b4r39YqIuCgzz+ukzsuByyPi05l5QzfqJCIGAV8EtmDxiyi7HPYCrAW8VTP9VjmvEQ9FxMEUnUkbA0dTdCIsERHxMxYP6StTHIcXR0RDZ5HKv5PfAjZn8X21awNlB1B0Wn2BYojRv1NkgJ0oPksfbvS1NCozzynrHpaZL3e2ztLGMdHdEMVFfdeVp85WpPjSGk3RS3NwOWSjo7K7dHJqvtH6NwV2o/gC+2VmNhrwmqnzY2WdALdl5q0Vyv4fxY+GSzNzbrtl38rM73dRfnXgT5n5dhkC3tfAGKvbgANy8XHckzJzrwbaOwz4AcUQlKDoCTgmM+c1UHZEWXYcxRfhb4GvZ+acBsp+geK4elfojogh2eBY2rLXbh+KU+onNFKmqoj4TWb+Xbx7LHhrWFqiF/pFxMk1k0nR2zggMzvteY1iXOfGwJNUD3dNiR4aW1mxzi8DhwOrZ+ZGZQi4IDN366Io5Q+bCylO/b9K8aPlsw0ey7OBbwKzqDlT1khoi4g9KDoGNqf47I0DJmTm1AbKfh/4HcWZPSiGSWzfSE9vRGwFnAdsCTwIrAnsn5ldnlkpA+y7/pg2GFrmA6tQBLu36L3P0IeBY3n3WOxO2xzFsKsdWr+nImIV4O5GP0MR8fe8Owg3ck3CTygC/MEUP8IPAR7JzC4vdIyIEynOev20nLUvcG1m/msDZVemOFu2J8V7cytFj2yHnTnNKHt/O9TVWfByG1MohiIdS3HtyKHASw1+Dp4Ebgcuzsy72i07t+Lws4aU342XUGSot4ED29e9tDJEd0NEPARsmZkZEYdT9DbuTvEL7fLMfNfdMqKLK9Mzs6HTvFEMwJ/UXw4wKEJdNnGglb3CI1j8i/7HXZR510Uc9eZ1UHbNrk6b9aTyj3eHMvP33dxupQtZelM0cdFaWf4fayYHUYwNf6SrXqmyV3YoRa8KFOPrX+uNYTERMQPYFZja+r5ExIOZueUSrHMmxXjO39XUOSs7uTipzjZWoRgmMb9Cmbszs7tnKVrPXm1PEVru6ah3qk65eheuPVAh4C1PcaYwgMey62tUWsttXTM5iOLi1UWZeXwj5ftCFMPMLqAY39w2bCUzZ3RYiHcuZmwNkWUP8bRGjqmIuICiZ3UXiou39wfuzcwvNlD2vswc2/p+lmcJf52Z23dVtiy/FTWf+8y8r5FyfSWKcdfP1+znlSi+L+c0UHZGZm5de+xHxLRs4AYEEfF3mfmbdvPGZeZvu/VCGlD+MDuw7JjcjuJGBp3+mFhaOJyje96qCYV7UYTat4FHyi/hej7ZyfaSxsdKzgBOiuLikZ+WdffExQTv0mxvY+1pqagzpLHB01JXABtR3N2j9Ys+KU4Xd+ZvETE8y7sSlOGp0SD/24iYQ/FL/obW3uxGRPeutP/3TpYlRfDqqt7aH2nLAS10MuxlKdDMRWtk5mL7LIrb8TVydmRfiouC205Fl23p8FR0D6o3tnJJX8/w18x8q7XO8vup089BdHBHg9ZtZAN3NADuK9/Pn1EzFKORzoIoLiL+VWb+vJxeLSL2zcwbOynzFeCrwAdj8QtUV6U4G9SliPh/wFVZ3pEmilsvfiYbGMtaJ3j+NoqLORupt/UC5A0z89SIWB9YO98ZzrWkLMrM/+xGuUuB30UxNBGKz1SXdzAp7VgG4Acy858j4t8pLgxuROsPmtciYkvgBaDTWxiWZzFbzSkfbcuykzuKRMQ5mfn1ePfwCqDhi3Ob8ROKs0Ct3i7nNXInrtZ99XzZ8/8csHon69c6l+JCxlrn1ZnXkxZl5qMAmfm7iGjmYtJeZYjunr+WH+I/UvyiPrZm2Sr1CmTmF3qi4nxnbNnqFL0d3y/D4sY9sf12df1d+W93D+ieuP90C8WdGKr2ZJ8I/CYi7oC2W10d3kjBzPxwFPfePgg4sRwCMCm7uK1eqfKV9pm5SyPrdaH2R9oiij8W+/TAdpeUZi5aq7s9iouWuvJFitP7raeivw/cTe+E6F4dW1m6IyL+ieKuCHtQBM2fdVGmJ+5osBJFeN6zZl6jnQUnZ2ZrQCMzXyuH73QYoikufrqF4g4GtUOY5ncWlNr5cma23VUpi7sQfRnoMkS3C2vLAVtTjE9uxPmUty0FTqW4i8sPaSwsNeNnEfFVis6Y2h86ne6vzDwriltUtg6Z+UKFXt2/lP8uiIh1KC78bOjuRcCFUQzLO4nimBxM1xdsz6A47gIYTjEsKYDVgKfp/HqcK8p/l+j/o9CJ5TOzbRx3+WO40TtxfS+KO4D9I8V32/uAr3dWIHrmTjPd9f52dS423eAP9z5hiO6eYyhumbQmxVWlTwFEcdu3uqfeO+rdadWNg+RDFHdw+P/tnXv8pWO5/98fh5BmHAcdELMrqsHGoHIWbbLtbMMkh+Gldj+FJqWd364fCkWyN3YJOUQUwxbShJ0zZRiUc+WwI0IS8zNyvPYf1/3MWt816/CsZ61nPWt9Xe/Xa72+s5713Ou+5/td63mu+76v6/NZnfxFb4WQdI6Z7dXpWCN5crdycDewCvkLmrK+f56277Ktvpl5t4RT+znAHLlG9vF4IWWeILqXSvuuU1ckHZP6m20tilmHlF6K1rIt5WxitSj+XeyYV4nfQOsnN6+lY4PgQHxy9xJe+HYFHjSVyZfxicNduGTWz8ysrdSW9UHRoMdFg2ZqGm3vVeb1As/hqXVFWbQ+9UxeYJU3aKkP1l7F88c7pigk+ipb2gUz0s9D6o61lapbcJKnmBVJM/upXE3kW6m94WkdHTGXAAVPweo4xtRmDQBJpwEXWyrelNf4fKxD27npZz/uY0V4WtJOZnYpgKR/whWb8vBs3Xdiq9S+k/lPX5StCnJaQ5+Nz4eWyInuM2pRfayxhVALYTllfOTmIzsDD+LpBhd3k25QhMY8w7Ql/BvroAOr1kYgAOTJU5QX7KyHr4DVr5a03UpTC11ny2GHK3e/2xlfiZ6Mr9Rc0ClXMLU9ErjZClTat0pdsTaFHOl3vA4wtzEXdJhR86K1PfLmJqf0nIxXgSfNrONKdprMzmBsgdFZNqSV370i6XNmdkKnYy3aPgCsYzVpyyXw731HbWxJ78BXwLIb9w14ce5jrVstaHsG8FdqWvufxQsj9+nUthckfQtflDglHfo08KiZfaF1q770ewv+Pbg1BdOTcAOtoaxn6Bfp87Sk5S+YLmz6pSZ1AM2OtWg7x1Kdk6RdzWxWnvH2SlpkOBd4Gz45exTY23JIzDXes1sda9F29bzX4SCC6L4j6Q9mtlqJ7/9pPE8396pqD30dimt4LgXMzw7jFeSnmtmhHdqv3u71PF9UtahU7rQ6kPLYMpbEi6vmWr5q+YfxreMLrL2GdbO2WaX9S3heWu5Ke0n30WXqdbm5iQAAHPBJREFUSrrxfwpfQZhf/1LefqtA0gZmNld1RWuSdrQcrnR96Ht96tQbutiKLtpf05zKjDJzK1vcTPMW2PaiaHAVnmKRbYnviU+Sts3Rdmlc3/rD6dBVwJHWRLWmn8iNJT5NnQoR8H3zepdObRcH9qemnXwtcIrlKEyUS2JOx3NOf0CSLS0rWJO0tZldrRbF7pazyL2H/rsuFE/tmhWL5w0Mr8Anctlu4h7A5tZGrUnSzfgOzkeAf8AlOG8d9GKFpLcAmNn/z3FulpIxEzcNy5gI7GxjPQwa21aWAy7pAjPbLf07213NXrvSzLZr3bpaIojuM5IeNbNV27xeSOtS0lrmlatNv8BWUMEhD5K+0SlgHnbkxTr/YWa75Di3VzWR5XEZtfq/bx5ZolnAQVbTDu+mz0vMbJhzoMcgt7Td28zuTs8/jtu5b1ztyPpPq4lgRhnbxao5Sm6KBw8ZE4HXLIfEXXqfQooGku40s/U6HRsvSPo+rs2e6efvhf+e2znb1rcfmGyppCPMHUIz3f/sWpdNvPPoLhftu+vdtrq2v8FVQepNv24zs/flaLs8cBh1BjHAEda+sFDAFDyF6Sr8mj4FVzS5zszyFkR2haQ9zeyHrVJArb3N+Ra4Ydv/SePMmAdcZmYttfjrFjYKLVz1Qv0EqcnO99CqTEHkRJdBp+DrHFzr8iPUaV3meN+D8cK4ZkoOuRQcimJmh6ats8bAsGNqBIDctOAk3C79TXgOa1sjEPVfh/ix1H+7cf6Hmc0ELpVUaCYut7H+HF7kdieek30ztdWtZm2ymf8E4F55VX/u1JXE/U3ed8yMfsiYBlwoL7TbDNibsUVo44Yybz5tKOwoKWmiuaX68nSpaFDHM5L2pGZ6sjteRNYRFdQv7pWUM3o4ntKxGLVrTZ7826kNq3xXyyXk8vSbyZZ+p+PJfcDMstTC/fHi9HdS+z2XvapWtFAcejD9Sp/ZjnrSDZyBB9vPZxOL9DedjV+zSgmiqYkTdJ0TbD0Y6Vi1OeDtPg9DvdIbK9EFaJPrK+DdZtbSElu9a10uaQ0i782O9ZMWgeEv897U5E6MH8fleTbEA6Z3l7m6Lekkan+jRfC86kfMbM82bXqeiafPxlRc23a9tMJ0tJm11Anvx0pli2373Pq4VZCCpZ/gVfI7m9mLHZqMJB2uF1bm3yilRrxoZq+n3/daeBFqyzQDST/FlTheoy6ApougMqVynYQ7Whoe1B9kSXKyQ9tC+sW9IndG/XyTfvOYLN2Omzs9mJ6vCVyYM9VgBp7OUbpsaUO/P8dzz29n7KpwaUoIvey2pfZdmX71kqKQvi+bAcfiixQv4QZA+wM32gC9BLqll4moKsgBT9+93fF79Q/xXTSlxw/NrO0CWJVEEF0A9ZDrm31A5ZXvn8G1LufkXO3oqWCgKEUCw4b2t5nZhhor/F7qFk26MWW8igfQpYnF1/V7q5lNlZtcbGxmL0m6J8+WY8H+Mn3cyUB9wckE3FK75aShCpoElCvhFeQvQb5i01Gjl+tFH/qeiwcCy+F6ybfiOvd75GhbyAhGrmpxdp4+WrSfa2YbdD6zv0i6pWg6kaRtcP3kh/Ab/+q49Ftud1rVZEs/jlvD9122tKG/Uo1+Gvqq323rulC8h36zhZEv4p/9eiZYjhqMuoWvNwN34MoRm5WVPpd2JlqSM/Wl64moKswBl0smtqsb6YcMbClEOkcxFsedg8YEZWk7sK0dNTWty6+SX+sSSasAb8f1Xv8eFshyTcQ1csvkb2b2N0lIWsI8N7tjhX4d8+WSTXfK1UWeoLmMVV9IN/HteriJ9zITf0wu4fQT4CpJzwJ5FScaU1fAA8zb8Cr0h5o064c+7iDZseoBDJr6IFnSytT0f+eY2VMldy8zmy9pP+C7ZnZsmuDlYa6kqWbWGHy0xcxek7S6pDdZnc5tFxTSL+4D18gLdf+rod+O9SZm9gu59nd2XXzAUu5uFwxMtjRxs6QpZnbXAPo6Dr9nHcNYabnsWEt6Se2rCxo/AfzcajUYu+PFd3kKmQ9M7zVf0v1mdhzlakf/My6FuRyuXFSEIkY6H8JzvncAvoSnb64p6ZuUmAMOYGZblvXeZRMr0QVI252HNl58JE3BV2jbuRMW7XMGsA+eDlG/1TcPl+gqraJa7ky1L37R2Rr/Yi9uZjvkbL86bkzzJny7dBn8ht5Rqqcokm4Etu7mJt7vmXhK01gGv3h3HIekr+O52+fhN4hMYu92YP9OF5o0eViZsdt3HbfPq0TSSozNsx/q8faCpN1wfdxrYYEB0CFmdmGJfd6B71T8O7Cfmd2j/NJe9+OB3f8AL9BF+omks/EahEtTWyCfHr5cHaeRXGkkvSCX02zWb960taKqE8figeVDwI+Bn1iJsqV1u0GL4YHSQ/ikYRDpRZWknWXpNXgwndVg7Gg55PUkfcXMjkz/XqLA5Kjbsd6LK9PMxosEx+jY55lMyjXdn6KLiag81/x64IvZzmla0T4IX3nP5SZbBElTcTnJP6Xne+O7Mv8DHD6kC0JABNGFUBsP+k43KElNV53NLI9RBGqhQz0oug0Mq6LITVyqphq7rv9fW4MEkZKiQbPXGs47AC+KepKalXSpN8RekLQTXvD2NvxivzpwX1lpL8NAuiFtm60+y/WA/7vd37UPfW6O50beZGbHpGBiZs4t4aZpKB3S1c4xs70k/ZWxEltZ21x6+KOGelOd+AzuUvhOM/uapNWAVawk2+8q0ovq0s7WxD0OMrpKO1NNntLwvOTc8pTqsgZD0r/iQeXJllRlyk6dTH0chOddrwn8sf4l8tckdD0RVYU54PKagg+b2V/SNevH+A7AesDaZla22UthIp2jGMu2eW2pDm3rtU6XxLe3c2/dmdlFkj7KwhJ5uYLwoqQUlFXxle95wPvJ6VjVY3pEUR5Mj0WoVTl3mjFWVY2dMT+tVmYrk9OArGC009hnAu+xHEVQQ8LX8QLV/075hlvhWsLjmUUa0jeeocS0JligoHN93fOH8JWlPG2LBFMbyO2c/0BBK/WUe3ownhf8L1maRJ781V5IqTZHA28zs+0lvRf4gJmdnqN5L6oTU6jZfn8Nv75eREm232UEyTnoOe0sLUDtSs06/ixJs9qtkGrhGozlcXWoWyR1qsG4P/W3pqQb0vMVJL3HzB7IM+YimNmJwImSTjaz/Qu+Rzs781Ztfgv8VtIBZvahuhzwyfgueJkSqovWfQ6m4z4UFwEXdZF+VgkRRBfjNkmfsgb7XLmKRdsKcjMbI1En6Tjc/jcXkr6H50BvhdulTsOLNEojpRnsg2/7LVjlpIOsXl16xEryYsTfAYfiKh1lc29jsC5p1w5tvoEHy6tIugmfia+MX3hPadewT+wBnAB8F//9/grYU66HekCHto/i+dOjwitm9oykRSQtYmbXSBqXroGwYJfjVrnpQyb7Nh3o2tmyy34n4fmNjZPusuTivgf8AliDsWlnIqelNF6gNxc3jQBfjZtFvvzVXjgr9f1v6flvcVfYPEH03cAqdGFdX0dVtt8Dw/pjy74HsK4lJaqUq3sn0C7NoJcajL/iZmNbpsfauAznl1Mg/cHWTXunaAANPU9EB50DDrCopMXMXWe3weV8M4Y6Th3qwQ0xM4GL5U5TWdC8IZ7zu3OX7/VmXDouLx80l8b7jZkdIenblL9CuhswuUD6RmWFCjQP1tsG8BXPxLNVwlb59Dc2O6iaIP9DwLWSLmdsDlxpclU98le5E9f1wLmSnsK3tMclZmaSNsKLiDO3xFPN7OI2zfrBuXgguCNuwDADKG1bth+raPi1Zrq8+Cu7matToz6wopldIHdqxcxeldTWrVD90Xh/JdUzWHrPSdQWK4Iaj+MTwWx3bgnGpjssRI+r7h/Bv6+TgeNxffUXzGzfHt5zUPQyEd2S2v1mt76PrDk/wvWt/wy8SDKIkvR3DPniUATRBTCzJ4EPpi3oTCLocjO7ulPbhu2lRYFJ+BZeXrI8rvlp2/QZ4K1dtC/C3XgKS7dKAgNPj5DriO4AvF1jpYIm4lJ3eahiJp4VdjTTMm3nIJalqvwhPd6UHsPOr3Gb8s/jK0zL4Eo145m5ePFMUyeykljBzE6X9DmrGTF0pbZRhF5W0YCX0+5LFlROpi4wLZEXJK1Q1+8mdL6B9+O6cCJeALaSpKNItt99eN/xxnPAPXJLeQO2BeZk1/k8+efdYGb/Fxbcs87BbdknyYvWn7USBAT6SNcT0boc8GnUVvd/if+/S8XMjpL0CzyWubIuLWoR0v14WIkgugBZcYG5BmhTHdA2BQj120uvAk+mLYy8/FQuofYtPCfZ8LSOMvkGcIeku+lulaWK9IjH8W3knRibWjMPD9jysCWDn4nD2FWCJfFdjcfbNRjhQq2tzOx1fMXtB+BV+tUOqXQ2BvaQlKldAKVrY2emKk+kWorH8e/fMHMY8HNgVUnn4jta+wyg34PxQuTJ6Xo1CQ8oWpImJqiJO6ikY4CORklmdq5czzuz/f6YlWj7PcJcnB4Z1w6o3yvMzW9uk7S/udzeigPquyhFJqKV5ICn8TWNl9LucNtzqibUOQog6UU8v7flKcAyZrZak7aTgcfMTTi2BNbBjQm6ljSStASwpOWQ6ekFSffgQe9d1G0zWk57UA1YrD71ubi1cWVr0aaSauw241kEr4rumHun5o5cmcb0KVaio2U3aMTMYfqJCqhd9KHPHfGt0VXxQr+JwBFmdmlZffaDtCK8CX4t/ZWZ/XlA/S6Gaz0L13rOdQ1pdp3QkDuGjhqS/hHf8a0s1UXSumaWy869SiRti+9mvBe4kjQRNbNr27TZArgFdxediueAXw5cjedTl5YD3ktMVTURRBeg1c2wgdfM7LEmbe/E86ffiRcVXQK8z3JqLqf3KKRHWhS1kfTL2X5TM7sx/fuSMoPnuj67VgSR9E/AFsAn8XSD+/FCku3Knom3GM978JvG3+U49wR85ay+aO15PLCeaGZ7lTbQLpC0DG4iMCrmMMGAkbQzcHW2OJB23rY0s5+U1N/WZna1pKYOrNZGg199km4LOiPph7iN/EXAGWZ2f8VDGmq6nYhKOhrfLdsQL7L9DW7y9d6Sh9pTTFU1EUQPmGzFQtKXgBfN7CR1YYGtHvRIiyLpeHwr6FK6dPJK7QcmVi9XBLkbD367Mkypciae+s8cuTIlgz/hpj4ddcGbTXRUsyAvzXY8GG4kvQNfgc60dW8APjeMN6MMJW30hmO5r5EF+jvCzA5LNQmNWLuahJgUDhZJE3GFj33xz/OZwI/MbF6lAxtCJK3DwottHU3ZUg74fngu9FHAAwx/DnhlRE704HklJfvvTU2JYfEu2veiR1qU7Oa1Sd2xPBJ3VRQqZIog29O9Ikil1dhmNqHzWS15i6TVLDn+yQ0bskK9oTXFCUrnTFyjN5N33DMd27ayEXWmmXZ2afeqFEAvAsw2swu6b26PSPps4wuSlo9Aur+Y2fOSLsT9GGbidSOHSDrRzArpko9HJJ2Bp4rew1hZ2jzOxqOYA14ZEUQPnn1xqamjzOxhSWvglb956UWPtBBmtlXBplUUKhRWBBmGamy5k9/m6em1lt9g4gvAjZIexFey1wA+I2lpUuFe8IZkkpnVr7CeJWlmZaPJx21p9+s76fln6aC/3ytm9nraHew2iD4PLxafS20XacHbkk8XO8hBSrfbB7eiPxvYyMyeSrU291LQ3GecsknRNAwz+1Ld033SsYHUJIwikc4xYki6BrfCLKJHWrTPQk5eVaRHqA/WpZKOzS4kdUWRK5Z9IUmr5VNxbV/wbctbs+A+R/slgLXS0weGpZgwqI4kG3UmtVz53YF9zWyb6kbVnjTx+yrw4XToKuBIM3uhdau+9PtN4M+4rna9ekrH1eSUr3sdcEPk6paDpPOB75i7cGbHjjGzf5W0jZn9osLhDRWSTge+bWb3Vj2W8U4E0QNC0gVmtpsWtiEVviWYq4o7BaYLkVcpowiSZpOcvMxs3VTBfoeZTenQrspChb4oggyyGlsu8bZeVn0uN2C4o91no5eiqGD8kwp2TsILsgyf0B5oZo9WOrAcSJqAXxsHYsIj6eEmh83MOq4myz0DNkuPybj86A1mdkJ/R/nGJRRQ8pPihEvxupqX6DLOCPITQfSAkPRWM3tC0hdwO+cxhT1WosxVr9QVqC0o7mlW/NOm/cALFapQBOmVFERvma18SVoeT+loF0QXLooKxj+SfgDMNLNn0/PlgeOG+XMhaQq+XZ/pWf8ZmGFmd1c3qs6kSe9UYCs8Ze9FM1urfaugE6GA0j2Sfo/rnjfK0g5tnDGqRE70gDCzLIf5LcCpwF/wbcNZ5g6IbZF0Y0rwzxQcFrzkb28T+z3mOoo4edVTRaHCllRjmNILR+OmNtfgf9fNGVvxvxBmdlj6OQpWtMHgWScLoMFTEySVonLRR04BDjY3s0Kup38qNQvjUki7VgcDq5nZv0h6F5521rEuIaXNLI0XTt8ATDWzbh1eg+ach9eyhAJKfp62IdeCHy/ESnRFJPmZ6cAuuPnKhzs0qQxJ6+Nbwu/HCxsnAdPMrGuHubLTIzRkhil5SeoA00g34HR4jpn9qUO7thbSZnZ8f0YYjCJpF2jLhpXo6zqlYlWJpF+b2bqdjpXQ7/l4geDeZvb+FFTfnGfHTdK/AxvgW+c34degX5rZi2WOOQiaIem7wLLAZYytnYr0vj4TK9HV8RSer/QMsFLeRukm2Mg869KdrxvM7PaUY9W1k1eT9yo7v7gy69JeyNQBksRWNysIvcjiBeOfbwO/lJSZDe2Kp1QNMw9J+io11aI9gYcG0O9kM5ueJEgxs/mS1KlROvfzsCCPex+8hmQVYImSxhoE7VgKD563qzuWV+Iu6IJYiR4wkj6DpxdMAmYBF3RTQSvpEdzC91k8oF0WD8afBD5lZqVIQWnALolFqUIRpF/0og4QBK1IajqZpvvVw16xL2k54AjcIAZ8d+bw+rSUkvq9GdgGz7NdX9Jk3MhjoxxtD8CLCjcAHsHHfIOZXV3ikIMgqJgIogeMpG8A55vZnQXbnwZcaGZXpOfb4SkhZwInmNnGfRtsrc+BuyQWpUpFkF7pUR3g3cDJwMppK3odYCdLTpFBELRH0rbAV3BJzCtx46Z9zOzaHG2/iAfOc83s1TLHGQStSLuZx0o6ibG1UwAM4z171IkgesSQdFdjPmMm89ONYkaXfd7H4F0Se6IKRZAqkXQdcAhwSp2Cyt1m9v5qRxYE3ZEmhF9k4Z2vtg6pfep7BdyZVcCvLEwmghFC0j+a2WWSZjR73czCeKvPRE706PFEKp77cXo+HXgyySu93rpZTwzcJbEPjJx1qaTFcVOYBY6FeFCcJ//8zWY2pyGFM1bEglFkFvA94PvUdr4GxRZ4GokBiwMXD7j/ICiMmV2W/jnfzGbVvyZp1wqGNO6JlegRIwWCh1HLF7wJzx98Dpdm+n0f+7oMv5lMYMAuif1kkIYpvSDp+/iNO1st2At4zcw+maPtbOAAXDJxfUnTgP3MbPvSBhwEJSBprpltUEG/38UtpTN3x+nAg2b22UGPJQh6oYUxzdCrVI0iEUQHLUlFegKOAb5U/xJwTBn5129kepH2krQmNS3dZ4GHgT1CXD8YNSQdjqsXXczYSXupBbaS7gfWztLWkuzkPWa2dpn9BkG/kLQ9sAMuXnB+3UsT8ZTMjkWyQXdEOseIIWkSHtC+D1gyO15GvqAlK3FJi1uDrbikpfrdX8Brkiab2YOwIDBuu53doBP9M+AaYBFc3WMXIHSig1Ejy+c8pO6Y4Y51ZfJ7YDUgm3iumo4FwajwOHAbsBOueZ4xD/h8JSMa50QQPXqci88wd8StZWcAT5fRUb3darKkzpiAp5EE/eUQ4BpJD+Gr/asDnZwIM53o9+CSfpektnvh6TdBMFKY2RoVdT0BuE/SHDxo3wivqbg0jWsk0teCNy4pbfHXks4r0zsiqBHpHCNGli+YKXKkY7ea2dRObQv0tQywHGG3OjAkLYEHxOCmNi+1O7+u3fXAR81sXno+AbjczDZv3zIIho8qdOlT+lpLGnfjgmBYkfQh4HB8IWYxfGEll1xq0B2xEj16ZLPLJyR9FN++aeZi2DNm9hxesLh7Ge8fNGUDasHDepLyBg8rAy/XPX85HQuCkaKVLj1QtrnT041GNJK2zKMTHQRDxul4+sZcBq9w84YigujR48i0QvwF4CS8YCByncYBPQYPZwNzJGWSXB/DzWaCYNTYkGp06S+QdDbwLbze5Ng0lg8MeBxB0CvPmdnsqgfxRiDSOYJgSOjV1EbS+rj1MMD1ZnZH3wYXBANC0izgIDMbqC69pKVxJaIN8Pzoc3EVorL094OgFCR9E1gU+C/GKtzcXtmgximxEj1iSFoDOJCF8wWj6GX06cnUJl0g4yIZjDorAvemAr9B6tK/ArwILIWvRD8cAXQwomTysxvWHTOgdNfPNxqxEj1iJDvr04G7qHMojKKX0WW8mNoEQT9oVeBX9jUuXVsvAb4GTMJdE182s3B6C4KgKRFEjxiSbgmTk/FFmNoEQfVI2ghXxlnDzL4maTVgbzM7suKhBUFXSFoZOBp4m5ltL+m9wAfM7PSKhzbuiCB6xJD0CeBdwJVErtO4ooVV6wIpwyAYz0i60cw2lTQP35lZ8BIuzzWx5P5Pxnf3tjaztSUtB1xZhnxoEJSJpNnAmcC/mdm6khYD7jCzKRUPbdwROdGjxxTcSGNraukckes0woSpTRCAmW2afk7odG5JbGxm60u6I43jWUmLVzSWIOiFFc3sAkmHApjZq5JC6q4EIogePXYF1jSzlzueGYwK5wGzCVObIKiSVyQtSloFlzSJsSviQTAqvCBpBWqf5U1wz4egz0QQPXrcDSwLPFX1QIL+EKY2QTAUnAhcDKwk6ShgGvCVaocUBIU4GLgUmCzpJrxQdlq1QxqfRBA9eiwL3C/pVkLBIQiCoC+Y2bmS5gLb4HnYHzOz+yoeVhAUYTKwPbAqsAsueRfxXglEYeGIUZX8UxAEQRAEw09WkC5pU+DrwHHA/wulp/4TM5MRI4LlIAiCIAjakBURfhQ4zcwulxRSjSWwSNUDCPIh6cb0c56k5+se8yQ9X/X4giAIgiAYCv4o6RRgOvAzSUsQ8V4pRDpHEARBEATBOEHSm4F/AO4ys99JeiswxcyurHho444IooMgCIIgCIKgS2J5PwiCIAiCIAi6JILoIAiCIAiCIOiSCKKDIAhGAEmPSLImj0danH9ten3FAQ81CILgDUFI3AVBEIwGBwJLAzsCewDfA64DXqhyUEEQBG9UYiU6CIJgBDCzy8zsx8Cd6dAtwBXALpKeTo+zJS3X2FbSvmlV+gw5h0p6OElkXiFpzXTe4em80yT9Lr3nroP7XwZBEIwOEUQHQRCMLicAM4CzgDOBvdKxenYETgV+BHwS2Bs4Gg/CvwmsA8xqaLMZ8J/AMumcIAiCoIFI5wiCIBhddgD+aGaHAEj6BLB9wzmn4Wkfe5vZ65J2TMenpwfAKpKWr2tzvJmdKml/4F3lDT8IgmB0iSA6CIJgfPMUsCGwFnB33fE90mvgu5Lz6177S/r5KrFjGQRB0JS4OAZBEIwulwNvl3SMpGOAtwM/azhnGvAaMFvSO4CfpuMzgFWBLYCvmtnfBjTmIAiCcUGsRAdBEIwuM9PP/dLPc+qOZfwO+GfgSjzA3gz4MvBp4GTgMeD80kcaBEEwzgjb7yAIgiAIgiDokkjnCIIgCIIgCIIuiSA6CIIgCIIgCLokguggCIIgCIIg6JIIooMgCIIgCIKgSyKIDoIgCIIgCIIuiSA6CIIgCIIgCLokguggCIIgCIIg6JIIooMgCIIgCIKgS/4XKIdogCUWIEIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T9WIp8XJXrZc",
        "colab_type": "text"
      },
      "source": [
        "### Visualization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tryuEZ0LbNNO",
        "colab_type": "text"
      },
      "source": [
        "#### Helper functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5eNIcNg4-aFZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def summarize_attributions(attributions):\n",
        "    attributions = attributions.sum(dim=-1).squeeze(0)\n",
        "    attributions = attributions / torch.norm(attributions)\n",
        "    return attributions\n",
        "\n",
        "\n",
        "def compute_attributions_ig(ig, input_data_embed):\n",
        "    # Create interpretable layer\n",
        "    if not type(\n",
        "        model.get_input_embeddings()).__name__ == \"InterpretableEmbeddingBase\":    \n",
        "        interpretable_embedding1, interpretable_embedding2,\\\n",
        "        interpretable_embedding3 = configure_interpretable_embeddings()\n",
        "    # Compute attributions for positive and nagative samples (class 1 and 0)\n",
        "    attr_0, delta_0 = ig_attribute(ig, 0, input_data_embed)\n",
        "    attr_1, delta_1 = ig_attribute(ig, 1, input_data_embed)\n",
        "    # Remove interprateble layer used by ig attribution\n",
        "    remove_interpretable_embeddings(interpretable_embedding1, \n",
        "                                    interpretable_embedding2, \n",
        "                                    interpretable_embedding3)\n",
        "    # Return sum over all three layers\n",
        "    attr_0 = torch.stack(attr_0, axis=0).sum(0)\n",
        "    attr_1 = torch.stack(attr_1, axis=0).sum(0)    \n",
        "    return (attr_0, delta_0), (attr_1, delta_1)    \n",
        "\n",
        "\n",
        "def compute_attributions_lig(lig, input_data):  \n",
        "    # Compute attributions for positive and nagative samples (class 1 and 0)\n",
        "    return lig_attribute(lig, 0, input_data), lig_attribute(lig, 1, input_data)\n",
        "\n",
        "\n",
        "def get_visualization_record(text, attributions, scores, true_label,\n",
        "                             all_tokens, approximation_error):\n",
        "    attributions_sum = summarize_attributions(attributions)\n",
        "    return viz.VisualizationDataRecord(\n",
        "        attributions_sum,\n",
        "        torch.max(torch.softmax(scores[0], dim=0)),\n",
        "        torch.argmax(scores),\n",
        "        true_label,\n",
        "        text,\n",
        "        attributions_sum.sum(),\n",
        "        all_tokens,\n",
        "        approximation_error)\n",
        "    \n",
        "\n",
        "def visualize_attributions(text, true_label, ig_object, \n",
        "                           method, layer_name=None):\n",
        "    # Prepare input\n",
        "    input_data, input_data_embed = get_input_data(text)\n",
        "\n",
        "    # Compute attributions\n",
        "    attr_0, attr_1, delta_0, delta_1 = None, None, None, None\n",
        "    if method == \"ig\":\n",
        "        (attr_0, delta_0), (attr_1, delta_1) = \\\n",
        "        compute_attributions_ig(ig_object, input_data_embed)\n",
        "    elif method == \"lig\":    \n",
        "        (attr_0, delta_0), (attr_1, delta_1) = \\\n",
        "        compute_attributions_lig(ig_object, input_data)\n",
        "    else:\n",
        "        return \"method: ig or lig\"    \n",
        "    # Run inference\n",
        "    scores = predict_forward_func(*input_data[0:3], input_data[-1])\n",
        "    # Prepare visualization \n",
        "    indices = input_data[0][0].detach().tolist()\n",
        "    all_tokens = tokenizer.convert_ids_to_tokens(indices)\n",
        "    data_vis_0 = get_visualization_record(text, attr_0, scores, \n",
        "                                          true_label, all_tokens, delta_0)  \n",
        "    data_vis_1 = get_visualization_record(text, attr_1, scores, \n",
        "                                          true_label, all_tokens, delta_1) \n",
        "    # Visualize\n",
        "    print(\"\\nAttribution method: {},\".\n",
        "          format(method), \"class index: 0 (negative)\")\n",
        "    if not layer_name is None:\n",
        "        print(\"Layer: {}\".format(layer_name))\n",
        "    viz.visualize_text([data_vis_0])\n",
        "    print(\"Attribution method: {},\".\n",
        "          format(method), \"Class index: 1 (positive)\")\n",
        "    if not layer_name is None:\n",
        "        print(\"Layer: {}\".format(layer_name))    \n",
        "    viz.visualize_text([data_vis_1])\n",
        "     \n",
        "    return attr_0, attr_1    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D8m9IAb2Z-vn",
        "colab_type": "text"
      },
      "source": [
        "#### Examples"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kGn3780AmD_Z",
        "colab_type": "text"
      },
      "source": [
        "Captum visualization library shows in green tokens that push the prediction towards the target class. Those driving the score towards the reference value are marked in red. As a result, words perceived as positive will appear in green if attribution is performed against class 1 (positive) but will be highlighted in red with an attribution targeting class 0 (negative).\n",
        "\n",
        "Because importance scores ar assigned to tokens, not words, some examples may show, that attribution is highly dependent on tokenization. Classification results may vary between runs.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UHyNJIc39rUr",
        "colab_type": "text"
      },
      "source": [
        "Browse examples:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ka-dgLIEgYPA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68,
          "referenced_widgets": [
            "5cadbc0eee10468cadb03d67882649a6",
            "aabd5e96eee646fbbcb1750f63393605",
            "aa1d749cbef34e3e84da730b68a59968",
            "2bc3aa5e05b84aa0b82abe812b98323d",
            "2a660a7941ad4ba0abebf0c83571472b",
            "ce3c5628d3ae4d6cb7a62891209144df",
            "925e3cd2eb424bbd8dafb4e5e869acc5",
            "38fad618758b44e987bff1488ebe9f10"
          ]
        },
        "outputId": "2c434650-b827-4c50-ff90-a157d872b713"
      },
      "source": [
        "# Run predictions\n",
        "eval_pred_result = trainer.predict(eval_dataset)\n",
        "predictions = np.argmax(eval_pred_result.predictions, axis=1)\n",
        "\n",
        "# Find correctly classified and misclassifed samples\n",
        "eval_samples = [tokenizer.decode(x.input_ids, skip_special_tokens=True) \\\n",
        "                for x in eval_dataset]\n",
        "eval_preds = list(zip(eval_pred_result.label_ids, predictions))\n",
        "positive_pred_as_positive = [sample for sample, (real_label, pred_label) \\\n",
        "                             in zip(eval_samples, eval_preds) \\\n",
        "                             if real_label == pred_label and real_label == 1]  \n",
        "negative_pred_as_negative = [sample for sample, (real_label, pred_label) \\\n",
        "                             in zip(eval_samples, eval_preds) \\\n",
        "                             if real_label == pred_label and real_label == 0] \n",
        "positive_pred_as_negative = [sample for sample, (real_label, pred_label) \\\n",
        "                             in zip(eval_samples, eval_preds) \\\n",
        "                             if real_label != pred_label and real_label == 1]                                                           \n",
        "negative_pred_as_positive = [sample for sample, (real_label, pred_label) \\\n",
        "                             in zip(eval_samples, eval_preds) \\\n",
        "                             if real_label != pred_label and real_label == 0]\n",
        "\n",
        "# Browse\n",
        "# print('\\n'.join(positive_pred_as_positive))   \n",
        "# print('\\n'.join(negative_pred_as_negative))    \n",
        "# print('\\n'.join(positive_pred_as_negative))  \n",
        "# print('\\n'.join(negative_pred_as_positive))                                             "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5cadbc0eee10468cadb03d67882649a6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Prediction', max=109.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "45IN_FSKe2Bk",
        "colab_type": "text"
      },
      "source": [
        "Use our examples or pick your own by setting the `text_vis` and `true_label_vis` variables below."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j6FcN8liZyda",
        "colab_type": "text"
      },
      "source": [
        "##### Positive"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IHHxBoJ7oMOM",
        "colab_type": "text"
      },
      "source": [
        "A correctly classified positive sample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGrav0a8oecG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 899
        },
        "outputId": "6ded1cf0-c633-4910-e3f6-846f798d3c21"
      },
      "source": [
        "# We'll use our example\n",
        "text_vis = text\n",
        "true_label_vis = true_label\n",
        "\n",
        "ig_0, ig_1 = visualize_attributions(text_vis, true_label_vis, ig, \"ig\")\n",
        "lig_0, lig_1 = \\\n",
        "visualize_attributions(text_vis, true_label_vis, lig_we, \"lig\",\n",
        "                       layer_name=\"electra.embeddings.word_embeddings\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/attr/_models/base.py:189: UserWarning: In order to make embedding layers more interpretable they will be replaced with an interpretable embedding layer which wraps the original embedding layer and takes word embedding vectors as inputs of the forward function. This allows us to generate baselines for word embeddings and compute attributions for each embedding dimension. The original embedding layer must be set back by calling `remove_interpretable_embedding_layer` function after model interpretation is finished. \n",
            "  \"In order to make embedding layers more interpretable they will \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Attribution method: ig, class index: 0 (negative)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>1</b></text></td><td><text style=\"padding-right:2em\"><b>1 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>visually imaginative , thematically instructive and thoroughly delightful , it takes us on a roller-coaster ride from innocence to experience without even a hint of that typical kiddie-flick sentimentality . </b></text></td><td><text style=\"padding-right:2em\"><b>-2.20</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> visually                    </font></mark><mark style=\"background-color: hsl(0, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> imaginative                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thematic                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ally                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ins                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##truct                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ive                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> and                    </font></mark><mark style=\"background-color: hsl(0, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thoroughly                    </font></mark><mark style=\"background-color: hsl(0, 75%, 66%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> delightful                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> takes                    </font></mark><mark style=\"background-color: hsl(0, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> on                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> roller                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> coaster                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ride                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> from                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> innocence                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> experience                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> without                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> even                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hint                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> typical                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> kidd                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ie                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> flick                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sentimental                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ity                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Attribution method: ig, Class index: 1 (positive)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>1</b></text></td><td><text style=\"padding-right:2em\"><b>1 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>visually imaginative , thematically instructive and thoroughly delightful , it takes us on a roller-coaster ride from innocence to experience without even a hint of that typical kiddie-flick sentimentality . </b></text></td><td><text style=\"padding-right:2em\"><b>2.24</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> visually                    </font></mark><mark style=\"background-color: hsl(120, 75%, 87%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> imaginative                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thematic                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ally                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ins                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##truct                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ive                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> and                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thoroughly                    </font></mark><mark style=\"background-color: hsl(120, 75%, 58%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> delightful                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> takes                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> on                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> roller                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> coaster                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ride                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> from                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> innocence                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> experience                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> without                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> even                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hint                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> typical                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> kidd                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ie                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> flick                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sentimental                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ity                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Attribution method: lig, class index: 0 (negative)\n",
            "Layer: electra.embeddings.word_embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>1</b></text></td><td><text style=\"padding-right:2em\"><b>1 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>visually imaginative , thematically instructive and thoroughly delightful , it takes us on a roller-coaster ride from innocence to experience without even a hint of that typical kiddie-flick sentimentality . </b></text></td><td><text style=\"padding-right:2em\"><b>-2.20</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> visually                    </font></mark><mark style=\"background-color: hsl(0, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> imaginative                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thematic                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ally                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ins                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##truct                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ive                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> and                    </font></mark><mark style=\"background-color: hsl(0, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thoroughly                    </font></mark><mark style=\"background-color: hsl(0, 75%, 66%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> delightful                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> takes                    </font></mark><mark style=\"background-color: hsl(0, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> on                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> roller                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> coaster                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ride                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> from                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> innocence                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> experience                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> without                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> even                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hint                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> typical                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> kidd                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ie                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> flick                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sentimental                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ity                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Attribution method: lig, Class index: 1 (positive)\n",
            "Layer: electra.embeddings.word_embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>1</b></text></td><td><text style=\"padding-right:2em\"><b>1 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>visually imaginative , thematically instructive and thoroughly delightful , it takes us on a roller-coaster ride from innocence to experience without even a hint of that typical kiddie-flick sentimentality . </b></text></td><td><text style=\"padding-right:2em\"><b>2.24</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> visually                    </font></mark><mark style=\"background-color: hsl(120, 75%, 87%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> imaginative                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thematic                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ally                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ins                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##truct                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ive                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> and                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> thoroughly                    </font></mark><mark style=\"background-color: hsl(120, 75%, 58%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> delightful                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ,                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> takes                    </font></mark><mark style=\"background-color: hsl(120, 75%, 91%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> on                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> roller                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> coaster                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ride                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> from                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> innocence                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> experience                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> without                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> even                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> hint                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> typical                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> kidd                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ie                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> -                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> flick                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> sentimental                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> ##ity                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YJUKqnKJaBDo",
        "colab_type": "text"
      },
      "source": [
        "##### Negative"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VVf89iWsECJp",
        "colab_type": "text"
      },
      "source": [
        "A correctly classified negative sample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ku4Jq-9zf-K8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 801
        },
        "outputId": "178cbe52-6109-4cfe-ca81-0182299dc184"
      },
      "source": [
        "text_vis = 'the film makes a fatal mistake : it asks us to care about a young \\\n",
        "man whose only apparent virtue is that he is not quite as unpleasant as some \\\n",
        "of the people in his life.'\n",
        "true_label_vis = 0\n",
        "\n",
        "ig_0, ig_1 = visualize_attributions(text_vis, true_label_vis, ig, \"ig\")\n",
        "lig_0, lig_1 = \\\n",
        "visualize_attributions(text_vis, true_label_vis, lig_we, \"lig\",\n",
        "                       layer_name=\"electra.embeddings.word_embeddings\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/attr/_models/base.py:189: UserWarning: In order to make embedding layers more interpretable they will be replaced with an interpretable embedding layer which wraps the original embedding layer and takes word embedding vectors as inputs of the forward function. This allows us to generate baselines for word embeddings and compute attributions for each embedding dimension. The original embedding layer must be set back by calling `remove_interpretable_embedding_layer` function after model interpretation is finished. \n",
            "  \"In order to make embedding layers more interpretable they will \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Attribution method: ig, class index: 0 (negative)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>0 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>the film makes a fatal mistake : it asks us to care about a young man whose only apparent virtue is that he is not quite as unpleasant as some of the people in his life.</b></text></td><td><text style=\"padding-right:2em\"><b>3.25</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> makes                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> fatal                    </font></mark><mark style=\"background-color: hsl(120, 75%, 62%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mistake                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> asks                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> care                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> about                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> young                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> man                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> whose                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> only                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> apparent                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> virtue                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> he                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> not                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quite                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(120, 75%, 78%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> unpleasant                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> some                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> people                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> his                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> life                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Attribution method: ig, Class index: 1 (positive)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>0 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>the film makes a fatal mistake : it asks us to care about a young man whose only apparent virtue is that he is not quite as unpleasant as some of the people in his life.</b></text></td><td><text style=\"padding-right:2em\"><b>-3.22</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> makes                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> fatal                    </font></mark><mark style=\"background-color: hsl(0, 75%, 69%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mistake                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> asks                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> care                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> about                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> young                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> man                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> whose                    </font></mark><mark style=\"background-color: hsl(0, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> only                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> apparent                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> virtue                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> he                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> not                    </font></mark><mark style=\"background-color: hsl(0, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quite                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(0, 75%, 83%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> unpleasant                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> some                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> people                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> his                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> life                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Attribution method: lig, class index: 0 (negative)\n",
            "Layer: electra.embeddings.word_embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>0 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>the film makes a fatal mistake : it asks us to care about a young man whose only apparent virtue is that he is not quite as unpleasant as some of the people in his life.</b></text></td><td><text style=\"padding-right:2em\"><b>3.25</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> makes                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> fatal                    </font></mark><mark style=\"background-color: hsl(120, 75%, 62%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mistake                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> asks                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> care                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> about                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> young                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> man                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> whose                    </font></mark><mark style=\"background-color: hsl(120, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> only                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> apparent                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> virtue                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> he                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> not                    </font></mark><mark style=\"background-color: hsl(120, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quite                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(120, 75%, 78%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> unpleasant                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> some                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> people                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> his                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> life                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Attribution method: lig, Class index: 1 (positive)\n",
            "Layer: electra.embeddings.word_embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>0 (1.00)</b></text></td><td><text style=\"padding-right:2em\"><b>the film makes a fatal mistake : it asks us to care about a young man whose only apparent virtue is that he is not quite as unpleasant as some of the people in his life.</b></text></td><td><text style=\"padding-right:2em\"><b>-3.22</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> makes                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> fatal                    </font></mark><mark style=\"background-color: hsl(0, 75%, 69%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mistake                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> :                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> it                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> asks                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> us                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> care                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> about                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> young                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> man                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> whose                    </font></mark><mark style=\"background-color: hsl(0, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> only                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> apparent                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> virtue                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> he                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> is                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> not                    </font></mark><mark style=\"background-color: hsl(0, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> quite                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(0, 75%, 83%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> unpleasant                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> as                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> some                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> of                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> the                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> people                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> in                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> his                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> life                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gzZhcUMKn3My",
        "colab_type": "text"
      },
      "source": [
        "##### Misclassified"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VCHm7T-ih-zZ",
        "colab_type": "text"
      },
      "source": [
        "Pick an example. Results may vary between runs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_8sGOEfHNtQl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Negative examples misclassified as positive\n",
        "#print('\\n\\n'.join(negative_pred_as_positive))\n",
        "\n",
        "# Positive examples misclassified as negative\n",
        "#print('\\n\\n'.join(positive_pred_as_negative)) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NiLOTdTGu2Kf",
        "colab_type": "text"
      },
      "source": [
        "Visualize"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPlFy4MoHhmn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 703
        },
        "outputId": "9c1f638f-1b1b-42fe-c38e-4ac5535f8465"
      },
      "source": [
        "text_vis = \"a tv style murder mystery with a few big screen moments ( including one that seems to be made for a different film altogether ).\"\n",
        "true_label_vis = 0\n",
        "\n",
        "ig_0, ig_1 = visualize_attributions(text_vis, true_label_vis, ig, \"ig\")\n",
        "lig_0, lig_1 = \\\n",
        "visualize_attributions(text_vis, true_label_vis, lig_we, \"lig\",\n",
        "                       layer_name=\"electra.embeddings.word_embeddings\")\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/captum/attr/_models/base.py:189: UserWarning: In order to make embedding layers more interpretable they will be replaced with an interpretable embedding layer which wraps the original embedding layer and takes word embedding vectors as inputs of the forward function. This allows us to generate baselines for word embeddings and compute attributions for each embedding dimension. The original embedding layer must be set back by calling `remove_interpretable_embedding_layer` function after model interpretation is finished. \n",
            "  \"In order to make embedding layers more interpretable they will \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Attribution method: ig, class index: 0 (negative)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>1 (0.98)</b></text></td><td><text style=\"padding-right:2em\"><b>a tv style murder mystery with a few big screen moments ( including one that seems to be made for a different film altogether ).</b></text></td><td><text style=\"padding-right:2em\"><b>-1.19</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> tv                    </font></mark><mark style=\"background-color: hsl(0, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> style                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> murder                    </font></mark><mark style=\"background-color: hsl(0, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mystery                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> with                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> few                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> big                    </font></mark><mark style=\"background-color: hsl(0, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> screen                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> moments                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> including                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> one                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> seems                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> be                    </font></mark><mark style=\"background-color: hsl(0, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> made                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> for                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> different                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(0, 75%, 73%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> altogether                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Attribution method: ig, Class index: 1 (positive)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>1 (0.98)</b></text></td><td><text style=\"padding-right:2em\"><b>a tv style murder mystery with a few big screen moments ( including one that seems to be made for a different film altogether ).</b></text></td><td><text style=\"padding-right:2em\"><b>1.26</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> tv                    </font></mark><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> style                    </font></mark><mark style=\"background-color: hsl(0, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> murder                    </font></mark><mark style=\"background-color: hsl(120, 75%, 84%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mystery                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> with                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> few                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> big                    </font></mark><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> screen                    </font></mark><mark style=\"background-color: hsl(0, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> moments                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> including                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> one                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> seems                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> be                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> made                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> for                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> different                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(120, 75%, 65%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> altogether                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Attribution method: lig, class index: 0 (negative)\n",
            "Layer: electra.embeddings.word_embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>1 (0.98)</b></text></td><td><text style=\"padding-right:2em\"><b>a tv style murder mystery with a few big screen moments ( including one that seems to be made for a different film altogether ).</b></text></td><td><text style=\"padding-right:2em\"><b>-1.19</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> tv                    </font></mark><mark style=\"background-color: hsl(0, 75%, 88%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> style                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> murder                    </font></mark><mark style=\"background-color: hsl(0, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mystery                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> with                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 90%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> few                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> big                    </font></mark><mark style=\"background-color: hsl(0, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> screen                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> moments                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> including                    </font></mark><mark style=\"background-color: hsl(120, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> one                    </font></mark><mark style=\"background-color: hsl(0, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> seems                    </font></mark><mark style=\"background-color: hsl(120, 75%, 98%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> be                    </font></mark><mark style=\"background-color: hsl(0, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> made                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> for                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> different                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(0, 75%, 73%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> altogether                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Attribution method: lig, Class index: 1 (positive)\n",
            "Layer: electra.embeddings.word_embeddings\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table width: 100%><tr><th>True Label</th><th>Predicted Label</th><th>Attribution Label</th><th>Attribution Score</th><th>Word Importance</th><tr><td><text style=\"padding-right:2em\"><b>0</b></text></td><td><text style=\"padding-right:2em\"><b>1 (0.98)</b></text></td><td><text style=\"padding-right:2em\"><b>a tv style murder mystery with a few big screen moments ( including one that seems to be made for a different film altogether ).</b></text></td><td><text style=\"padding-right:2em\"><b>1.26</b></text></td><td><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [CLS]                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> tv                    </font></mark><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> style                    </font></mark><mark style=\"background-color: hsl(0, 75%, 93%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> murder                    </font></mark><mark style=\"background-color: hsl(120, 75%, 84%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> mystery                    </font></mark><mark style=\"background-color: hsl(120, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> with                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(0, 75%, 92%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> few                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> big                    </font></mark><mark style=\"background-color: hsl(120, 75%, 86%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> screen                    </font></mark><mark style=\"background-color: hsl(0, 75%, 95%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> moments                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> (                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> including                    </font></mark><mark style=\"background-color: hsl(0, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> one                    </font></mark><mark style=\"background-color: hsl(120, 75%, 97%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> that                    </font></mark><mark style=\"background-color: hsl(120, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> seems                    </font></mark><mark style=\"background-color: hsl(0, 75%, 99%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> to                    </font></mark><mark style=\"background-color: hsl(0, 75%, 96%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> be                    </font></mark><mark style=\"background-color: hsl(120, 75%, 94%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> made                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> for                    </font></mark><mark style=\"background-color: hsl(120, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> a                    </font></mark><mark style=\"background-color: hsl(120, 75%, 89%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> different                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> film                    </font></mark><mark style=\"background-color: hsl(120, 75%, 65%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> altogether                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> )                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> .                    </font></mark><mark style=\"background-color: hsl(0, 75%, 100%); opacity:1.0;                     line-height:1.75\"><font color=\"black\"> [SEP]                    </font></mark></td><tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}