ECE_Fourth_Year_Opinions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%run ./setup_notebook.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd \n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "# from plotly.subplots import make_subplot\n",
    "from geopy.geocoders import Nominatim\n",
    "from skimage import io\n",
    "# import ipywidgets as widgets\n",
    "import re\n",
    "from ipywidgets import interact, interact_manual\n",
    "import folium\n",
    "import time\n",
    "from collections import defaultdict\n",
    "import json\n",
    "# helpful modules\n",
    "from scipy.spatial.distance import euclidean, pdist, squareform\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PIE_WIDTH=500\n",
    "PIE_HEIGHT=500\n",
    "TABLE_CELL_HEIGHT=50\n",
    "TABLE_HEADER_HEIGHT=60\n",
    "PLOT_BGCOLOR=\"#f6f3ea\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "intro"
    ]
   },
   "source": [
    "<div style=\"font-family:'Trebuchet MS', 'Impact', sans-serif; \n",
    "            margin-bottom: 20px; letter-spacing: 6px; text-align:center;\n",
    "            font-size: 300%;\"><b>University of Toronto ECE2T2 Class Profile</b></div>\n",
    "\n",
    "<div style=\"font-size: 16px;text-align:center;\"><i>Number of Respondents: 54</i></div>\n",
    "<!-- <div style=\"font-size: 16px;text-align:center;\"><i>All Plots show data</i></div> -->\n",
    "\n",
    "<p style=\"text-align:start;\">Welcome to the University of Toronto ECE2T2 Class Profile -- Inspired by Waterloo Software Engineering 2021 Class Profile. <br>\n",
    "\n",
    "We come here to give you all the deets, all the information on our graduating class: their passions, compassions, and the future they have built in their times at UofT.\n",
    "   \n",
    "Let's find out about who these 52 people of University of Toronto ECE2T2 are, and where they be at!</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"Electrical_and_Computer_Engineering_2023_-_Class_Profile_Survey_Submissions_2023-06-07.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Strip white space in columns\n",
    "df.columns = df.columns.str.strip() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "outputs": [],
   "source": [
    "print(len(df))\n",
    "df.head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "## Data Cleaning\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Modify these column names, if they still exist\n",
    "if {\"Untitled long answer field (1)\", \"Untitled long answer field (2)\", \"Untitled long answer field (3)\"}.issubset(df.columns):\n",
    "  df.rename(columns={\n",
    "                      \"Untitled long answer field (1)\" : \"1. Share a story, happy or sad, from your time in ECE\",\n",
    "                      \"Untitled long answer field (2)\" : \"2. What is something you regret over your time at UofT?\",\n",
    "                      \"Untitled long answer field (3)\" : \"3. Give a piece of advice to your fellow ECE 2T2 - 2023 grads\"\n",
    "                    }, inplace=True)\n",
    "\n",
    "  # Check column data has indeed been changed\n",
    "  index = df.columns.str.contains(\"1. Share a story, happy or sad, from your time in ECE\") | \\\n",
    "  df.columns.str.contains(\"2. What is something you regret over your time at UofT?\") | \\\n",
    "  df.columns.str.contains(\"3. Give a piece of advice to your fellow ECE 2T2 - 2023 grads\")\n",
    "\n",
    "  indices = df.columns[index]\n",
    "  assert len(indices) == 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop unnecessary article. Non-useful data.\n",
    "# Proof that all data under this column are null\n",
    "if \"https://money.usnews.com/money/personal-finance/family-finance/articles/where-do-i-fall-in-the-ameri\" + \\\n",
    "          \"can-economic-class-system\" in df.columns:\n",
    "        assert df[\"https://money.usnews.com/money/personal-finance/family-finance/articles/where-do-i-fall-in-the-ameri\"\n",
    "                \"can-economic-class-system\"].isna().sum() == len(df)\n",
    "\n",
    "        df.drop(\"https://money.usnews.com/money/personal-finance/family-finance/articles/where-do-i-fall-in-the-american-economic-class-system\", \n",
    "                axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Write out columns to a text file for analysis\n",
    "with open(\"all_columns.txt\", mode=\"w\", encoding=\"utf-8\") as f:\n",
    "    f.write(\"\\n\".join(df.columns))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "## Data Exploration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "big_heading"
    ]
   },
   "source": [
    "# Class Profile"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What Program Were You in First Year?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What program were you in first year?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TO-DO! Make trace subplots with this\n",
    "fig = px.pie(df, names=\"What program were you in first year?\", color=\"What program were you in first year?\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\", textposition=\"inside\")\n",
    "fig.update_layout(margin=dict(l=2, r=2, t=10, b=2))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What Program Are You in Now?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What program are you in now?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"What program are you in now?\", color=\"What program are you in now?\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\")\n",
    "fig.update_layout(margin=dict(l=2, r=2, t=10, b=2))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What 2 Areas Did You Specialize In?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What 2 areas did you specialize in?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sort specialized areas to remove duplicates. (Ex. Area 6 + Area 5 is equivalent to Area 5 + Area 6)\n",
    "df_specialization = remove_redundancy(df[\"What 2 areas did you specialize in?\"]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_specialization.sort_values(ascending=False, inplace=True)\n",
    "\n",
    "# hover texT\n",
    "areas_long_form = df_specialization.str.split(\", \").map(lambda x: \"<br>\".join(x)).tolist()\n",
    "\n",
    "# Simplified areas\n",
    "areas_simplified = df_specialization.map(\n",
    "    lambda word: \" + \".join(re.findall('(Area.*?):', word))\n",
    ")\n",
    "\n",
    "df_specialization = areas_simplified\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.groupby(\"What 2 areas did you specialize in?\", as_index=False).count()[[\"What 2 areas did you specialize in?\", \"Submission ID\"]]\n",
    "fig = px.pie(df_specialization, names=\"What 2 areas did you specialize in?\", color=\"What 2 areas did you specialize in?\",\n",
    "             hole=0.4)\n",
    "fig.update_traces(hovertemplate=\"%{text}<br><i>Count:</i> %{value}<br> %{percent}\",\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "                  hovertext=areas_long_form,\n",
    "                  text=areas_simplified,\n",
    "                  textposition=\"inside\",\n",
    "                )\n",
    "fig.update_layout(margin=dict(l=2, r=2, t=10, b=2))\n",
    "# fig.update_layout(legend=dict(traceorder=\"reversed\",\n",
    "#                               yanchor=\"top\", xanchor=\"center\",\n",
    "#                               orientation=\"h\",\n",
    "#                               x=0.5, y=-0.1),\n",
    "#                               showlegend=True)\n",
    "fig.show(height=PIE_HEIGHT, width=PIE_WIDTH)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exchange Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Have you participated in an exchange?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Have you participated in an exchange?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"If yes, where did you do your exchange?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(\n",
    "    df.loc[(df[\"Have you participated in an exchange?\"] == \"no\") & \\\n",
    "        df[\"If yes, where did you do your exchange?\"].notna(), [\"Have you participated in an exchange?\", \n",
    "                                                                \"If yes, where did you do your exchange?\"]].shape[0] == 0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(\n",
    "    df.loc[(df[\"Have you participated in an exchange?\"] == \"yes\") & \\\n",
    "        df[\"If yes, where did you do your exchange?\"].isna(), [\"Have you participated in an exchange?\", \n",
    "                                                                \"If yes, where did you do your exchange?\"]].shape[0] == 0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"If you did not do an exchange, what were your reasons for not?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"If you did not do an exchange, what were your reasons for not?\"].fillna(\"No answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(\n",
    "    df.loc[(df[\"Have you participated in an exchange?\"] == \"yes\") & \\\n",
    "        (df[\"If you did not do an exchange, what were your reasons for not?\"] != \"No answer\"), :].shape[0] == 0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# assert(\n",
    "#     df.loc[(df[\"Have you participated in an exchange?\"] == \"no\") & \\\n",
    "#         (df[\"If you did not do an exchange, what were your reasons for not?\"] == \"No answer\"), :].shape[0] == 0\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_no_exchange_reasons = \\\n",
    "    process_multiple_choice_survey(df[\"If you did not do an exchange, what were your reasons for not?\"], column=\"No Exchange Reasons\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_exchange_answers = pd.concat([df[[\"Have you participated in an exchange?\",\n",
    "                                     \"If yes, where did you do your exchange?\"]], df_no_exchange_reasons[[\"No Exchange Reasons\"]]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_figure_with_dropdown(df_exchange_answers, \n",
    "                            options=[\"Have you participated in an exchange?\",\n",
    "                                     \"If yes, where did you do your exchange?\",\n",
    "                                     \"No Exchange Reasons\"],\n",
    "                            labels=break_text([\"Have you participated in an exchange?\",\n",
    "                                     f\"If yes, where did you do your exchange? <i>(Sample Size: {df['If yes, where did you do your exchange?'].notna().sum()})</i>\",\n",
    "                                     \"If you did not do an exchange, what were your reasons for not?\"], \n",
    "                                     char_limit=27),\n",
    "                            showticklabels=[False, False, False],\n",
    "                            sort_traces=True,\n",
    "                            textposition=\"auto\",\n",
    "                            horizontal=True\n",
    "    )\n",
    "# change_to_dark_mode_plotly(fig)\n",
    "fig.update_xaxes(showticklabels=False)\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Profile Information (Gender, Sexual Orientation, Domestic/International, Ethnicities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Are you a domestic or international student?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What are your Ethnicities?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What are your Ethnicities?\"].fillna(\"Did not wish to disclose\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What are your Ethnicities?\"] = df[\"What are your Ethnicities?\"].str.replace(\"\\s*origins\\s*\", \"\", regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Are you a domestic or international student?\"].fillna(\"Did not wish to disclose\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"Are you a domestic or international student?\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = create_figure_with_dropdown(df,\n",
    "                                  options=[\"What is your gender?\", \"Are you a domestic or international student?\",\n",
    "                                           \"What is your sexual orientation?\", \"What are your Ethnicities?\"],\n",
    "                                  labels=break_text([\"What is your gender?\", \"Are you a domestic or international student?\",\n",
    "                                           \"What is your sexual orientation?\", \"What are your Ethnicities?\"]),\n",
    "                                  showticklabels=[False, False, False, False],\n",
    "                                  sort_traces=True,\n",
    "                                  textposition=\"auto\")\n",
    "fig.update_layout(margin=dict(l=10, r=10),\n",
    "                  hovermode=\"x unified\",\n",
    "                  legend=dict(y=1, yanchor=\"top\", orientation=\"v\")\n",
    ")\n",
    "fig.update_yaxes(showticklabels=False)\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Do you identify as an Indigenous person, that is, First Nations, Metis, or Inuk (Inuit)?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Do you identify as an Indigenous person, that is, First Nations, Metis, or Inuk (Inuit)?\"].fillna(\"Did not wish to disclose\", \n",
    "                                                                                                       inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig = create_table(df,\n",
    "                  \"Do you identify as an Indigenous person, that is, First Nations, Metis, or Inuk (Inuit)?\",\n",
    "                  label=break_text(\"Do you identify as Indigenous, that is, First Nations, Metis, or Inuk (Inuit)?\", char_limit=27),\n",
    "                  cell_height=TABLE_CELL_HEIGHT,\n",
    "                  header_height=TABLE_HEADER_HEIGHT, base_height=350)\n",
    "fig.update_layout(title_font=dict(size=19), title_y=0.94)\n",
    "fig.show(renderer=\"svg\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Location Profile!"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Childhood, Sweet Childhood~\n",
    "\n",
    "Where did 4th Year ECEs attend high school?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Where did you attend high school?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_high_school = df.loc[df[\"Where did you attend high school?\"].notna(), \"Where did you attend high school?\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geolocator = Nominatim(user_agent=\"new_agent\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Hard Code Unique Locations (I think their API blocked me so I commented this out and just manually encoded the coordinates in a later cell)\n",
    "# high_school_locs = \\\n",
    "#  [dict(country=\"China\"), dict(country=\"Canada\", city=\"Toronto\"),\n",
    "#  dict(country=\"Canada\", state=\"British Columbia\"),\n",
    "#  dict(country=\"Nicaragua\"), # Central America\n",
    "#  \"London, Ontario\", # Southern Ontario\n",
    "#  \"USA\"]\n",
    "\n",
    "# identified_high_school_locs = []\n",
    "# for loc in high_school_locs:\n",
    "#     identified_high_school_locs.append(geolocator.geocode(query=loc))\n",
    "#     time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# identified_high_school_coordinates = [(location.latitude, location.longitude) for location in identified_high_school_locs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_high_school.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "identified_high_school_coordinates = [(35.00007, 104.9999), \n",
    "                                      (43.65348, -79.38393),\n",
    "                                      (55.00125, -125.0024),\n",
    "                                      (12.60902, -85.29369),\n",
    "                                      (42.98324, -81.24337),\n",
    "                                      (39.78373, -100.4459)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_high_school_merged = map_location_setup(df_high_school, identified_high_school_coordinates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_high_school_merged.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_high_school_merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fake size to make the map markers look better\n",
    "df_high_school_merged[\"fake_size\"] = df_high_school_merged[\"Where did you attend high school?\"] * 5 + 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter_geo(df_high_school_merged,\n",
    "                     lat=df_high_school_merged.Coordinates.str[0],\n",
    "                     lon=df_high_school_merged.Coordinates.str[1],\n",
    "                     size=\"fake_size\",\n",
    "                     hover_name=\"index\")\n",
    "fig.update_geos(projection_type=\"natural earth\",\n",
    "                showocean=True, oceancolor=\"LightBlue\",\n",
    "                showcountries=True, countrycolor=\"#4e4e4e\",\n",
    "                bgcolor=PLOT_BGCOLOR)\n",
    "fig.update_traces(hovertemplate=\"<b>%{hovertext}</b><br><br>%{text} attended high school here! <br>\" + \\\n",
    "                  \"<b>Lat</b>: %{lat}<br><b>Lon</b>: %{lon}<extra></extra>\", text=df_high_school_merged[\"Where did you attend high school?\"],\n",
    "                  marker=dict(color=\"crimson\"),\n",
    "                  mode=\"text+markers\",\n",
    "                  textposition=\"top right\",\n",
    "                  texttemplate=\"<span style='font-weight:500'>%{text}</span>\")\n",
    "fig.update_layout(margin={\"r\":0,\"t\":0,\"l\":0,\"b\":0})\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", yanchor=\"bottom\", x=0.98, y=0, text=f\"Sample Size: {len(df_high_school)}\", font=dict(size=16), ax=0, ay=0)\n",
    ")\n",
    "fig.show(renderer=\"notebook\", width=1110, height=600)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<span style=\"font-size:1.5em\">Where are their homes?</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df['Where is home?'], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_home = df['Where is home?'].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_home, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geolocator = Nominatim(user_agent=\"app\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Hard Code Unique Locations\n",
    "# identified_home_locs = \\\n",
    "#  [geolocator.geocode(query=dict(country=\"South Korea\")), geolocator.geocode(query=dict(country=\"Canada\", state=\"Saskatchewan\")),\n",
    "#  geolocator.geocode(query=dict(country=\"China\", city=\"Hong Kong\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Egypt\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Canada\", state=\"British Columbia\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Canada\", city=\"Toronto\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Canada\", state=\"Saskatchewan\")),\n",
    "#  geolocator.geocode(query=dict(country=\"India\")),\n",
    "#  geolocator.geocode(query=dict(country=\"United Arab Emirates\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Brazil\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Bangladesh\")),\n",
    "#  geolocator.geocode(query=dict(country=\"USA\")),\n",
    "#  geolocator.geocode(query=dict(country=\"Pakistan\"))\n",
    "#  ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_home.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# identified_home_coordinates = [(location.latitude, location.longitude) for location in identified_home_locs]\n",
    "identified_home_coordinates = [\n",
    "    (36.63839, 127.6961),\n",
    "    (55.53213, -106.1412),\n",
    "    (22.27933, 114.1628),\n",
    "    (26.25405, 29.26755),\n",
    "    (55.00125, -125.0024),\n",
    "    (43.65348, -79.38393),\n",
    "    (55.53213, -106.1412),\n",
    "    (22.35111, 78.66774),\n",
    "    (24.00025, 53.99948),\n",
    "    (-10.3333, -53.2),\n",
    "    (24.47693, 90.29344),\n",
    "    (39.78373, -100.4459),\n",
    "    (30.33084, 71.2475)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_home_merged = map_location_setup(df_home, identified_home_coordinates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_home_merged.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_home_merged[\"fake_size\"] = df_home_merged[\"Where is home?\"] * 5 + 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter_geo(df_home_merged,\n",
    "                     lat=df_home_merged.Coordinates.str[0],\n",
    "                     lon=df_home_merged.Coordinates.str[1],\n",
    "                     size=\"fake_size\",\n",
    "                     hover_name=\"index\")\n",
    "fig.update_geos(projection_type=\"natural earth\",\n",
    "                showocean=True, oceancolor=\"LightBlue\",\n",
    "                showcountries=True, countrycolor=\"#4e4e4e\",\n",
    "                bgcolor=PLOT_BGCOLOR)\n",
    "fig.update_traces(hovertemplate=\"<b>%{hovertext}</b><br><br>%{text} students live here! <br>\" + \\\n",
    "                  \"<b>Lat</b>: %{lat}<br><b>Lon</b>: %{lon}<extra></extra>\", text=df_home_merged[\"Where is home?\"],\n",
    "                  mode=\"text+markers\",\n",
    "                  marker_color=\"#5a7db6\",\n",
    "                  textposition=\"top right\",\n",
    "                  texttemplate=\"<span style='font-weight:500'>%{text}</span>\")\n",
    "fig.update_layout(margin={\"r\":0,\"t\":0,\"l\":0,\"b\":0})\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", yanchor=\"bottom\", x=0.98, y=0, text=f\"Sample Size: {len(df_home)}\", font=dict(size=16), ax=0, ay=0)\n",
    ")\n",
    "fig.show(renderer=\"notebook\", width=1110, height=600)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In first year, where did you live?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"In first year, where did you live?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"In first year, where did you live?\"].fillna(\"No answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"In first year, where did you live?\", color=\"In first year, where did you live?\")\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\")\n",
    "fig.update_layout(margin=dict(l=0, r=0, t=2, b=2), font=dict(size=18))\n",
    "fig.show(height=900, width=900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_first_year_locations = df.loc[df[\"In first year, where did you live?\"].isin(\n",
    "    [\"Chestnut\", \"Victoria\", \"University College\", \"Innis\", \"St. Micheals College\"]\n",
    "), \"In first year, where did you live?\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "first_year_coordinates = []\n",
    "for query in [\"Chestnut Residence\", \"University College Ontario\", \"Innis college toronto\", \"Victoria College\", \"Elmsley Hall\"]:\n",
    "    response = json.loads(requests.get(\n",
    "        f\"https://nominatim.openstreetmap.org/search?q={query}&format=json&limit=1\"\n",
    "    ).text)\n",
    "    first_year_coordinates.append((response[0][\"lat\"], response[0][\"lon\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_first_year_merged = map_location_setup(df_first_year_locations, first_year_coordinates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_first_year_merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_first_year_merged.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.scatter_mapbox(df_first_year_merged,\n",
    "                     lat=df_first_year_merged.Coordinates.str[0],\n",
    "                     lon=df_first_year_merged.Coordinates.str[1],\n",
    "                     size=\"In first year, where did you live?\",\n",
    "                     hover_name=\"index\",\n",
    "                     mapbox_style=\"open-street-map\",\n",
    "                     center={\"lat\":  43.65992116412125, \"lon\": -79.39048393551872}, zoom=12)\n",
    "# fig.update_geos(projection_type=\"natural earth\",\n",
    "#                 showocean=True, oceancolor=\"LightBlue\",\n",
    "#                 showcountries=True, countrycolor=\"#4e4e4e\",\n",
    "#                 bgcolor=\"#D0E3F1\")\n",
    "fig.update_traces(hovertemplate=\"<b>%{hovertext}</b><br><br>%{text} students lived here! <br>\" + \\\n",
    "                  \"<b>Lat</b>: %{lat}<br><b>Lon</b>: %{lon}<extra></extra>\",\n",
    "                  text=df_first_year_merged[\"In first year, where did you live?\"].astype(str),\n",
    "                  mode=\"markers+text\",\n",
    "                  marker=dict(\n",
    "                      color=\"darkgreen\",\n",
    "                      opacity=1,\n",
    "                      sizemin=7,\n",
    "                  ),\n",
    "                  )\n",
    "fig.update_layout(margin={\"r\":0,\"t\":0,\"l\":0,\"b\":0})\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In your last year, where did you live?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"In your last year, where did you live?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"In your last year, where did you live?\"].fillna(\"No answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"In your last year, where did you live?\", color=\"In your last year, where did you live?\")\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\", textposition=\"inside\")\n",
    "fig.update_layout(margin=dict(l=0, r=0, t=0, b=0))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Family Info!\n",
    "\n",
    "Who are the proud parents?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Did you or your parents immigrate to Canada?\"], check_unique=True)\n",
    "df[\"Did you or your parents immigrate to Canada?\"].fillna(\"Did not say\", inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Was one of your parents born as a Canada Citizen?\"], check_unique=True)\n",
    "df[\"Was one of your parents born as a Canada Citizen?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table_with_dropdown(df, [\"Did you or your parents immigrate to Canada?\",\n",
    "                                \"Was one of your parents born as a Canada Citizen?\"], \n",
    "                                labels=break_text(\n",
    "                                    [\"Did you or your parents immigrate to Canada?\",\n",
    "                                \"Was one of your parents born as a Canada Citizen?\"],\n",
    "                                char_limit=25), reindex=[False, True],\n",
    "                                reindex_orders=[None, [\"Yes\", \"No\", \"Did not say\"]])\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Education Level of Parents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Education Level of Parents\"], check_unique=True)\n",
    "df[\"Education Level of Parents\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df, y=\"Education Level of Parents\", color=\"Education Level of Parents\")\n",
    "fig.update_layout(showlegend=True,\n",
    "                  hovermode=\"y unified\", legend=dict(\n",
    "                      xanchor=\"right\", x=0.95, yanchor=\"bottom\", y=-0.05\n",
    "                  ))\n",
    "fig.update_yaxes(title_text=\"\", showticklabels=False, categoryorder=\"total ascending\", showgrid=False)\n",
    "fig.update_xaxes(visible=False)\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram_h\"][\"hovertemplate\"])\n",
    "add_percent_labels_for_hist(df[\"Education Level of Parents\"], fig, horizontal=True)\n",
    "fig.show(width=800)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Family Income Before UofT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Family Income Before UofT\"], check_unique=True)\n",
    "df[\"Family Income Before UofT\"].fillna(\"Did not wish to disclose\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fig = px.histogram(df, x=\"Family Income Before UofT\", color=\"Family Income Before UofT\")\n",
    "fig.update_xaxes(categoryorder='array', categoryarray=['32,048 or less',  '$32,048 - $53,413', \n",
    "                                                        '$53,413 - $106,827', '$106,827 - $373,894', \n",
    "                                                        '$373,894 and up', 'Did not wish to disclose', \n",
    "                                                         ])\n",
    "fig.update_layout(showlegend=False,\n",
    "                  hovermode=\"x unified\", font=dict(color=\"#2e2e2e\"),\n",
    "                  margin=dict(t=2, b=2,))\n",
    "fig.update_xaxes(title_text=\"\", tickmode=\"array\", tickvals=['32,048 or less',  '$32,048 - $53,413', \n",
    "                                                        '$53,413 - $106,827', '$106,827 - $373,894', \n",
    "                                                        '$373,894 and up', 'Did not wish to disclose', \n",
    "                                                         ], ticktext=['$32,048 or less', '$32,048 - 53,413', \n",
    "                                                         '$53,413 - 106,827', '$106,827 to 373,894', \n",
    "                                                        '$373,894 and up', 'Did not wish to disclose',\n",
    "                                                         ])\n",
    "fig.update_yaxes(visible=False)\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram_h\"][\"hovertemplate\"],\n",
    "                  marker_color=px.colors.qualitative.Pastel2[0])\n",
    "add_percent_labels_for_hist(df[\"Family Income Before UofT\"], fig)\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Language Spoken at Home?\n",
    "\n",
    "We are a diverse group, UofT ECEs!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename({\"What languages do you speak at home? (If more than 1, please section by a comma. ie English, Italian, French)\":\n",
    "            \"What languages do you speak at home?\"}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What languages do you speak at home?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "language_at_home = df[\"What languages do you speak at home?\"].dropna()\n",
    "language_at_home=language_at_home.str.lower()\n",
    "language_at_home=language_at_home.str.replace(r'[-./?!,\":;()\\']', '', regex=True).str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(language_at_home, file_path=\"embed_images/Language_WordCloud.png\", width=1000, height=600, \n",
    "                  relative_scaling=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map('./embed_images/Language_WordCloud.png')\n",
    "fig.update_layout(margin=dict(b=30))\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", yanchor=\"bottom\",\n",
    "    xanchor=\"right\", x=0.98, y=-0.05, text=f\"Sample Size: {len(language_at_home)}\", font=dict(size=16), ax=0, ay=0)\n",
    ")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_table(df, column=\"What languages do you speak at home?\", label=\"Top 5 Language Combos\", cell_height=TABLE_CELL_HEIGHT,\n",
    "            header_height=TABLE_HEADER_HEIGHT, limit=5, base_height=330)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How Many Percentage of ECE2T2 Students Share the Same Religion as Their Parents? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your religious affiliation?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What is your religious affiliation?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is the religious affiliation of your parent(s)?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What is the religious affiliation of your parent(s)?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_religion(row):\n",
    "    if row[\"What is your religious affiliation?\"] == \"Did not say\" or \\\n",
    "        row[\"What is the religious affiliation of your parent(s)?\"] == \"Did not say\":\n",
    "        return \"Did not say\"\n",
    "\n",
    "    return \"Same Religion as Parent\" if row[\"What is your religious affiliation?\"] == row[\"What is the religious affiliation of your parent(s)?\"] \\\n",
    "        else \"Different Religion from Parent\"\n",
    "\n",
    "series_is_same_religion = df.apply(apply_religion, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(names=series_is_same_religion, color=series_is_same_religion)\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "                  textposition=\"inside\",\n",
    "                  hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=2, r=2))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What are the Religious Pairings if Child and Parent Share Different Religions? Same Religions?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_combined_religion = df.loc[:, [\"What is your religious affiliation?\", \"What is the religious affiliation of your parent(s)?\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_combined_religion[\"Student+Parent_Combined_Religion\"] = df[\"What is your religious affiliation?\"] + \", \" + df[\"What is the religious affiliation of your parent(s)?\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_combined_religion.loc[df_combined_religion[\"What is your religious affiliation?\"] == \\\n",
    "                                df[\"What is the religious affiliation of your parent(s)?\"], \n",
    "                                \"Student+Parent_Combined_Religion\"] = \\\n",
    "                                df_combined_religion.loc[df_combined_religion[\"What is your religious affiliation?\"] == \\\n",
    "                                df[\"What is the religious affiliation of your parent(s)?\"], \n",
    "                                \"Student+Parent_Combined_Religion\"].str.split(\", \").str[0].str.strip()\n",
    "                                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_figure_with_dropdown(df_combined_religion,\n",
    "                            options=[\"Student+Parent_Combined_Religion\", \"Student+Parent_Combined_Religion\"],\n",
    "                            filters=[\n",
    "                                df_combined_religion[\"What is your religious affiliation?\"] == \\\n",
    "                                df[\"What is the religious affiliation of your parent(s)?\"],\n",
    "                                df[\"What is your religious affiliation?\"] != \\\n",
    "                                df[\"What is the religious affiliation of your parent(s)?\"]],\n",
    "                            labels=[\"Same Religion\", \"Different Religion<br><i>Format: child_religion, parent_religion</i>\"],\n",
    "                            sort_traces=True,\n",
    "                            showticklabels=[False, False],\n",
    "                            textposition=\"inside\",\n",
    "                            add_legend=True\n",
    ")   \n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_layout(legend=dict(xanchor=\"center\", yanchor=\"top\", orientation=\"h\", x=0.5, y=-0.1))\n",
    "fig.layout.updatemenus[0].y = -0.6\n",
    "fig.show(renderer=\"notebook\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How Many Siblings Do You Have?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many siblings do you have?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"How many siblings do you have?\"].fillna(\"Did not wish to disclose\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plotly's Legend ordering is based on input data.\n",
    "df_siblings = df[\"How many siblings do you have?\"].sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"How many siblings do you have?\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_siblings, y=\"How many siblings do you have?\", color=\"How many siblings do you have?\")\n",
    "fig.update_yaxes(categoryorder=\"array\",\n",
    "                 categoryarray=['0', '1', '2', '3', '4', '5 or more', 'Did not wish to disclose'],\n",
    "                 title=\"\",\n",
    "                 showticklabels=False)\n",
    "fig.update_layout(showlegend=True, margin=dict(t=0, b=0, l=2, r=2),\n",
    "                  legend=dict(xanchor=\"right\", yanchor=\"top\", x=0.98, font=dict(size=20)))\n",
    "fig.update_traces(hovertemplate=\"How many siblings do you have?<br>\" + less_cringe_hovertext_template[\"histogram_h\"][\"hovertemplate\"])\n",
    "add_percent_labels_for_hist(df[\"How many siblings do you have?\"], fig, horizontal=True)\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Professional Profile\n",
    "\n",
    "The careers we ECE2T2s lived as we chased after our boundless dreams"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Did You Participate in An Enriched Program in High School?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Did you participate in an enriched program?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Did you participate in an enriched program?\", color=\"Did you participate in an enriched program?\",\n",
    "             hole=0.5)\n",
    "fig.update_xaxes(categoryorder=\"array\",\n",
    "                 categoryarray=['IB', 'AP', 'TOPS', 'No'])\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "                  hovertemplate=\"Did You Participate in An Enriched Program in High School?<br>\" + less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=2, r=2))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In What Year did ECE2T2s Start Their University Studies?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"In what year did you start your university studies?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_year_of_start_univ = df[\"In what year did you start your university studies?\"].dropna().astype(int)\n",
    "df_year_of_start_univ_cleaned = pd.DataFrame(df_year_of_start_univ.value_counts()).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_year_of_start_univ_cleaned.rename({\"index\" : \"year\",\n",
    "                                     \"In what year did you start your university studies?\":\n",
    "                                     \"Number of Students\"}, axis=1, inplace=True)\n",
    "df_year_of_start_univ_cleaned.sort_values(\"year\", ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(df_year_of_start_univ_cleaned)\n",
    "fig = px.scatter(df_year_of_start_univ_cleaned, x=\"year\", y=[0,0,0], text=df_year_of_start_univ_cleaned[\"Number of Students\"])\n",
    "\n",
    "fig.update_traces(marker_size=[30, 100, 30],\n",
    "                  hovertemplate=\"<b>Year:</b> %{x}<br><b>Number of Students:</b> %{text}\",\n",
    "                  textposition=\"top center\")\n",
    "\n",
    "fig.update_yaxes(\n",
    "    visible=False,\n",
    "    range=[0,0]\n",
    ")\n",
    "\n",
    "fig.update_xaxes(\n",
    "    dtick=\"Y1\",\n",
    "    linecolor=\"white\",\n",
    "    title=\"\",\n",
    "    showgrid=False,\n",
    "    tickfont=dict(size=16, family=\"Open Sans, Verdana, sans serif\")\n",
    ")\n",
    "\n",
    "# Don't know why this one line is taking so long\n",
    "# fig.add_annotation(xanchor=\"right\",  text=\"1 person did not answer\")\n",
    "\n",
    "fig.update_layout(\n",
    "    font=dict(size=13),\n",
    "    height=300\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What was your dream career as a kid?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What was your dream career as kid?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dream_career = df[\"What was your dream career as kid?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_dream_career, delimiter=\"!@!\", file_path=\"./embed_images/DreamCareer_WordCloud.png\", is_frequency=True, relative_scaling=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/DreamCareer_WordCloud.png\")\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=0, r=0))\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df, \"What was your dream career as kid?\", label=\"Top 5 Dream Careers\", \n",
    "                cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT, base_height=330, limit=5)\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The Progression of Your Grades Over 8 Semesters (in %)\n",
    "\n",
    "I took the average percentage scores (%) of ECE2T2s every semester, and plotted their distributions in boxplots."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[[\"What was your high school admissions average? (% percentage score)\",\n",
    "# \"What was your first-year 1st semester average (percentage)? \",\n",
    "# \"What was your first-year 2nd semester average (percentage)? \",\n",
    "# \"What was your second-year first semester average (%)?\",\n",
    "# \"What was your second-year second semester average (%)?\",\n",
    "# \"What was your third-year first semester average (%)?\",\n",
    "# \"What was your third-year second semester average (%)?\",\n",
    "# \"What was your fourth-year first semester average (%)?\"\n",
    "# ]].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_before = [\"What was your high school admissions average? (% percentage score)\",\n",
    "\"What was your first-year 1st semester average (percentage)?\",\n",
    "\"What was your first-year 2nd semester average (percentage)?\",\n",
    "\"What was your second-year first semester average (%)?\",\n",
    "\"What was your second-year second semester average (%)?\",\n",
    "\"What was your third-year first semester average (%)?\",\n",
    "\"What was your third-year second semester average (%)?\",\n",
    "\"What was your fourth-year first semester average (%)?\"\n",
    "]\n",
    "columns_now = [\"High school admissions average?\", \"First-year 1st semester average\", \"First-year 2nd semester average\",\n",
    "    \"Second-year first semester average\", \"Second-year second semester average\", \"Third-year first semester average\", \n",
    "    \"Third-year second semester average\", \"Fourth-year first semester average\"]\n",
    "\n",
    "repl_dict = {columns_before[i]:columns_now[i] for i in range(8)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.rename(repl_dict, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_grades = df.rename(repl_dict, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_semester_scores = df_grades[[\"Respondent ID\", \"High school admissions average?\", \"First-year 1st semester average\", \"First-year 2nd semester average\",\n",
    "    \"Second-year first semester average\", \"Second-year second semester average\", \"Third-year first semester average\", \n",
    "    \"Third-year second semester average\", \"Fourth-year first semester average\"]]\n",
    "# Remove troll answers\n",
    "df_semester_scores = df_semester_scores[df_semester_scores != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_semester_scores.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_melted = pd.melt(df_semester_scores, id_vars=[\"Respondent ID\"], \n",
    "                           value_vars=[\"High school admissions average?\", \"First-year 1st semester average\", \"First-year 2nd semester average\",\n",
    "    \"Second-year first semester average\", \"Second-year second semester average\", \"Third-year first semester average\", \n",
    "    \"Third-year second semester average\", \"Fourth-year first semester average\"], var_name=\"percentage_type\", value_name=\"percentage\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_scores_melted.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.box(df_scores_melted, y=\"percentage\", x=\"percentage_type\", color=\"percentage_type\", points=\"all\", range_y=[45, 105],\n",
    "color_discrete_sequence=px.colors.qualitative.G10)\n",
    "fig.update_traces(jitter=0.03, hovertemplate='<b>%{x}</b><br>%{y}%'\n",
    "                  )\n",
    "fig.update_xaxes(showticklabels=False, title=\"\", showgrid=False)\n",
    "fig.update_layout(margin=dict(r=20, l=40, t=5, b=2), font=dict(size=13), showlegend=True, boxgroupgap=0.4,\n",
    "                legend=dict(\n",
    "                      xanchor=\"center\", yanchor=\"top\", x=0.5, y=-0.03, orientation=\"h\",\n",
    "                    title=\"\", font=dict(size=15)\n",
    "                ))\n",
    "\n",
    "# for i, slider in enumerate(fig.layout.sliders):\n",
    "#     slider[\"tickcolor\"] = \"white\"\n",
    "#     slider[\"currentvalue\"].font.color = \"black\"\n",
    "#     slider[\"currentvalue\"].prefix = \"<b>Percentage Type:</b> \"\n",
    "#     slider.font.color = \"#D0E3F1\"\n",
    "#     pass\n",
    "  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What was your CGPA, Cumulative Grade Point Average (% score)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_checker(df[\"What was your CGPA, Cumulative Grade Point Average (% score)?\"], check_unique=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Thoughts and Details on Coursework \n",
    "\n",
    "What were ECE2T2s' favourite courses? What courses did they take? Did they overload courses?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Which term would you consider was the most difficult?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which term would you consider was the most difficult?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Which term would you consider was the most difficult?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Which term would you consider was the most difficult?\", \n",
    "                   color=\"Which term would you consider was the most difficult?\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "                  textposition=\"inside\"                    \n",
    ")\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=2, r=2))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_melted = df_scores_melted[~df_scores_melted[\"percentage\"].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_scores_melted.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_scores_melted[\"percentage_type\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "percentage_semester_index = {\n",
    "    \"High school admissions average?\" : 0,\n",
    "    'First-year 1st semester average' : 1,\n",
    "    'First-year 2nd semester average' : 2,\n",
    "    'Second-year first semester average' : 3,\n",
    "    'Second-year second semester average' : 4,\n",
    "    'Third-year first semester average' : 5,\n",
    "    'Third-year second semester average' : 6,\n",
    "    'Fourth-year first semester average' : 7,\n",
    "}\n",
    "\n",
    "difficulty_semester_index = {\n",
    "    'first-year first semester' : 1,\n",
    "    'first-year second-semester' : 2,\n",
    "    'second-year first semester' : 3,\n",
    "    'second-year second semester' : 4,\n",
    "    'third-year first semester' : 5,\n",
    "    'third-year second semester' : 6,\n",
    "    'fourth-year first semester' : 7,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"difficulty_semester_index\"] = df[\"Which term would you consider was the most difficult?\"].replace(difficulty_semester_index)\n",
    "df_scores_melted[\"percentage_semester_index\"] = df_scores_melted[\"percentage_type\"].replace(percentage_semester_index)\n",
    "df_scores_melted[\"percentage_average\"] = df_scores_melted.groupby(\"percentage_type\")[\"percentage\"].transform(np.mean) # average for everyone in a specific semester\n",
    "df_scores_melted[\"own_average\"] = df_scores_melted[df_scores_melted[\"percentage_type\"] != \"High school admissions average?\"] \\\n",
    "                                .groupby(\"Respondent ID\")[\"percentage\"].transform(np.mean) # a respondent's average throughout all semesters\n",
    "df_scores_melted[\"grade_rank\"] = df_scores_melted[df_scores_melted[\"percentage_type\"] != \"High school admissions average?\"] \\\n",
    "                                .groupby(\"Respondent ID\")[\"percentage\"].rank(method=\"dense\", ascending=True) # Ranking of you grades throughout all semester. Ex \n",
    "                                                                                                             # You scored highest in first semester first year, so that rank is 1. \n",
    "                                                                                                             # Followed by second highest in first semester 2nd year, so that rank is 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_difficulty = df[[\"Respondent ID\", \"Which term would you consider was the most difficult?\", \"difficulty_semester_index\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_difficulty_percentage = pd.merge(df_difficulty, df_scores_melted, left_on=[\"Respondent ID\", \"difficulty_semester_index\"], \n",
    "                                                   right_on=[\"Respondent ID\", \"percentage_semester_index\"],\n",
    "                                                   how='inner')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For each semester that ECE2T2s perceived as the hardest semester, how many of them actually scored an average grade lower than their average grade across all University semesters?  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_difficulty_percentage[\"semester_grade_type\"] = np.where(\n",
    "    df_difficulty_percentage[\"percentage\"] < df_difficulty_percentage[\"own_average\"],\n",
    "    \"Hardest semester grade lower than<br>personal average\",\n",
    "    \"Hardest semester grade higher than<br>personal average\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_difficulty_percentage.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_difficulty_percentage[(df_difficulty_percentage[\"Which term would you consider was the most difficult?\"] == \"second-year second semester\")\n",
    "#                          & (df_difficulty_percentage[\"isLower\"] == True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts = df_difficulty_percentage.groupby([\"semester_grade_type\", \"Which term would you consider was the most difficult?\"], as_index=False).count() \\\n",
    "                                [[\"semester_grade_type\", \"Which term would you consider was the most difficult?\", \"Respondent ID\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts[\"difficulty_per_semester_sum\"] = df_scored_lower_or_higher_counts.groupby(\"Which term would you consider was the most difficult?\", as_index=False)[\"Respondent ID\"].transform('sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts[\"Plot Text\"] = \\\n",
    "    df_scored_lower_or_higher_counts[\"Respondent ID\"].astype(str) + \" (\" + \\\n",
    "    (df_scored_lower_or_higher_counts[\"Respondent ID\"] / df_scored_lower_or_higher_counts[\"difficulty_per_semester_sum\"] * 100).round(2).astype(str) + \"%)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_index = {**difficulty_semester_index, **{\"Semester grade lower than<br>personal average\": 8, \"Semester grade higher than<br>personal average\" : 9}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts_sorted = df_scored_lower_or_higher_counts.sort_values([\"semester_grade_type\", \"Which term would you consider was the most difficult?\"], key=lambda x: x.map(\n",
    "    combined_index\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_difficulty_percentage.sort_values(\"semester_grade_type\", key=lambda x: x.map(combined_index), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig = px.histogram(df_difficulty_percentage, x=\"semester_grade_type\", color=\"semester_grade_type\", \n",
    "             facet_col=\"Which term would you consider was the most difficult?\", facet_col_wrap=2,\n",
    "             category_orders={\"Which term would you consider was the most difficult?\": \n",
    "                              ['first-year first semester', 'first-year second-semester', \n",
    "                                'second-year first semester', 'second-year second semester',\n",
    "                                'third-year first semester', 'third-year second semester',\n",
    "                                'fourth-year first semester']})\n",
    "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[-1]))\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"bar\"][\"hovertemplate\"])\n",
    "fig.update_xaxes(showticklabels=False, title=\"\", showgrid=False)\n",
    "fig.update_yaxes(showgrid=False)\n",
    "add_percent_labels_for_hist(df_difficulty_percentage[\"semester_grade_type\"], fig, horizontal=False, custom=True, \n",
    "                            custom_text_arr=df_scored_lower_or_higher_counts_sorted[\"Plot Text\"].reset_index(drop=True))\n",
    "fig.update_layout(height=700, width=900, minreducedheight=350, minreducedwidth=350, font=dict(size=15),\n",
    "                 showlegend=True, legend=dict(xanchor=\"right\", yanchor=\"bottom\", x=0.9, y=0))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Which term would you consider was the most easy?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which term would you consider was the most easy?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Which term would you consider was the most easy?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Which term would you consider was the most easy?\", \n",
    "                   color=\"Which term would you consider was the most easy?\")\n",
    "fig.update_layout(showlegend=True)\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{value} (%{percent})\")\n",
    "fig.update_layout(legend=dict(yanchor=\"top\", xanchor=\"center\",\n",
    "                                x=0.5, y=-0.1, orientation=\"v\"),\n",
    "                                margin=dict(l=0, r=0, t=2)\n",
    "                                )\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "We chillin' in 4th Year ;)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "percentage_semester_index = {\n",
    "    \"High school admissions average?\" : 0,\n",
    "    'First-year 1st semester average' : 1,\n",
    "    'First-year 2nd semester average' : 2,\n",
    "    'Second-year first semester average' : 3,\n",
    "    'Second-year second semester average' : 4,\n",
    "    'Third-year first semester average' : 5,\n",
    "    'Third-year second semester average' : 6,\n",
    "    'Fourth-year first semester average' : 7,\n",
    "}\n",
    "\n",
    "easiness_semester_index = {\n",
    "    'first-year first semester' : 1,\n",
    "    'first-year second-semester' : 2,\n",
    "    'second-year first semester' : 3,\n",
    "    'second-year second semester' : 4,\n",
    "    'third-year first semester' : 5,\n",
    "    'third-year second semester' : 6,\n",
    "    'fourth-year first semester' : 7,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"easiness_semester_index\"] = df[\"Which term would you consider was the most easy?\"].replace(easiness_semester_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_easiness = df[[\"Respondent ID\", \"Which term would you consider was the most easy?\", \"easiness_semester_index\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_easiness_percentage = pd.merge(df_easiness, df_scores_melted, left_on=[\"Respondent ID\", \"easiness_semester_index\"], \n",
    "                                                   right_on=[\"Respondent ID\", \"percentage_semester_index\"],\n",
    "                                                   how='inner')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For each semester that ECE2T2s perceived as the easiest semester, how many of them actually scored an average grade higher than their average grade across all University semesters? \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_easiness_percentage[\"semester_grade_type\"] = np.where(\n",
    "    df_easiness_percentage[\"percentage\"] < df_easiness_percentage[\"own_average\"],\n",
    "    \"Easiest semester grade lower than<br>personal average\",\n",
    "    \"Easiest semester grade higher than<br>personal average\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts = df_easiness_percentage.groupby([\"semester_grade_type\", \"Which term would you consider was the most easy?\"], as_index=False).count() \\\n",
    "                                [[\"semester_grade_type\", \"Which term would you consider was the most easy?\", \"Respondent ID\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts[\"difficulty_per_semester_sum\"] = df_scored_lower_or_higher_counts.groupby(\"Which term would you consider was the most easy?\", as_index=False)[\"Respondent ID\"].transform('sum')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts[\"Plot Text\"] = \\\n",
    "    df_scored_lower_or_higher_counts[\"Respondent ID\"].astype(str) + \" (\" + \\\n",
    "    (df_scored_lower_or_higher_counts[\"Respondent ID\"] / df_scored_lower_or_higher_counts[\"difficulty_per_semester_sum\"] * 100).round(2).astype(str) + \"%)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_index = {**difficulty_semester_index, **{\"Semester grade lower than<br>personal average\": 8, \"Semester grade higher than<br>personal average\" : 9}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scored_lower_or_higher_counts_sorted = df_scored_lower_or_higher_counts.sort_values([\"semester_grade_type\", \"Which term would you consider was the most easy?\"], key=lambda x: x.map(\n",
    "    combined_index\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_easiness_percentage.sort_values(\"semester_grade_type\", key=lambda x: x.map(combined_index), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig = px.histogram(df_easiness_percentage, x=\"semester_grade_type\", color=\"semester_grade_type\", \n",
    "             facet_col=\"Which term would you consider was the most easy?\", facet_col_wrap=2,\n",
    "             category_orders={\"Which term would you consider was the most easy?\": \n",
    "                              ['first-year first semester', 'first-year second-semester', \n",
    "                                'second-year first semester', 'second-year second semester',\n",
    "                                'third-year first semester', 'third-year second semester',\n",
    "                                'fourth-year first semester']})\n",
    "fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[-1]))\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"bar\"][\"hovertemplate\"])\n",
    "fig.update_xaxes(showticklabels=False, title=\"\", showgrid=False)\n",
    "fig.update_yaxes(showgrid=False)\n",
    "add_percent_labels_for_hist(df_easiness_percentage[\"semester_grade_type\"], fig, horizontal=False, custom=True, \n",
    "                            custom_text_arr=df_scored_lower_or_higher_counts_sorted[\"Plot Text\"].reset_index(drop=True))\n",
    "fig.update_layout(height=1000, minreducedheight=150, minreducedwidth=150, font=dict(size=15), showlegend=True,\n",
    "                  legend=dict(xanchor=\"right\", yanchor=\"bottom\", x=1, y=0))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Favourite Profs\n",
    "\n",
    "ECE2T2s were allowed to answer multiple favourite profs in their answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Favorite Professors\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.rename({\"Favourite Professors (if multiple, separate with comma)?\": \"Favorite Professors\"}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Favorite Professors\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"Favorite Professors\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_profs = process_multiple_choice_survey(df[\"Favorite Professors\"].dropna(), column=\"FavProfs\", delimeter=\",\\s*and\\s*|,\\s*\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "## Compute Similarity Matrix using Longest Common Subsequence as Distance\n",
    "\n",
    "Students enter their professor names in many different ways, so we need to cluster similar inputs under one ground truth value and replace all inputs under that cluster with the ground truth value. (ex. group entries of Belinda Wang and Belinda Yang under one cluster and replace all of those entries with Belinda Wang)\n",
    " \n",
    "To solve this problem,\n",
    "1. We first compute the similarity matrix between all unique inputs in the survey.\n",
    "2. Let the clustering algorithm to create several clusters.\n",
    "3. Finally, decide manually (using your own brain) the ideal ground truth value for those clusters.\n",
    "\n",
    "**IMPORTANT!! Only rerun this piece of code when survey inputs change and new clusters are required**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from similarity.longest_common_subsequence import LongestCommonSubsequence\n",
    "# LCS = LongestCommonSubsequence()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # from scipy.spatial.distance import pdist, squareform\n",
    "# def compare_lcs(u, v):\n",
    "#     min_length = min(len(u[0]), len(v[0]))\n",
    "#     return min(-0.001, -(1 - (abs((LCS.distance(u[0], v[0]) - len(u[0]) - len(v[0])) / 2) / min_length)))\n",
    "#     # return -((1 - cosine.similarity_profiles(cosine.get_profile(u[0]), cosine.get_profile(v[0]))) + levenshtein.distance(u[0], v[0]))\n",
    "\n",
    "# precomputed_similarity_matrix = squareform(pdist(np.expand_dims(df_profs[\"FavProfs\"].unique(), axis=1), compare_lcs)) \n",
    "# df_precompute_sim_matrix = pd.DataFrame(\n",
    "#     precomputed_similarity_matrix,\n",
    "#     index=df_profs[\"FavProfs\"].unique(),\n",
    "#     columns=df_profs[\"FavProfs\"].unique(),\n",
    "# )\n",
    "\n",
    "# # Generate similarity matrix between every possible professor input\n",
    "# df_precompute_sim_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Distribution of Similarity Matrix\n",
    "# px.histogram(df_precompute_sim_matrix[df_precompute_sim_matrix != 0].max(axis=1), title=\"Distribution of Largest Common Subsequence Metric\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "### Removing Unique Professor Names\n",
    "We apply this filter to try out best to not cluster professor names that were only entered in one way. \n",
    "For example, if we all entered Hamid's name as Hamid, then there is no need to rewrite Hamid is any other way.\n",
    "\n",
    "I only included inputs that have an LCS distance > -0.3, because I thought most inputs that have an LCS distance < -0.3 appeared to be unique professor names. I deducted this through trial and error. For each input, I found their most closely associated inputs with the LCS distance closest to 0 and put this information in a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Enter that similarity scores you'd like to see here. Ex find all similarity scores less than -0.5\n",
    "# MAT = df_precompute_sim_matrix[(df_precompute_sim_matrix[df_precompute_sim_matrix != 0].max() > -0.3)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def gather_closest_distances_for_each_word(row):\n",
    "#     closest_distances = row[row == row[row != 0].max()]\n",
    "#     best_matches_for_each_word = closest_distances.index.tolist()\n",
    "#     return [best_matches_for_each_word, row[row != 0].max()]\n",
    "\n",
    "\n",
    "# df_MAT = MAT.apply(gather_closest_distances_for_each_word, axis=1, result_type=\"expand\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_MAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Per analysis above, We only include words iff they have at least one LCS distance < -0.3 with another word.\n",
    "# df_precompute_sim_matrix = df_precompute_sim_matrix.loc[df_precompute_sim_matrix[(df_precompute_sim_matrix > -0.3) & (df_precompute_sim_matrix != 0)].any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_precompute_sim_matrix = df_precompute_sim_matrix[df_precompute_sim_matrix.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit data to AP Cluster model\n",
    "# from sklearn.cluster import AffinityPropagation\n",
    "# ap_cluster = AffinityPropagation(random_state=5, affinity=\"precomputed\").fit(df_precompute_sim_matrix.to_numpy())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "### Automated Text Cleaning Using AP Clustering \n",
    "\n",
    "ONLY RUN ONCE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "outputs": [],
   "source": [
    "# # ONLY RUN ONCE. This codeblock is used to memoize user prompt input.\n",
    "\n",
    "# replacement_tracker = [] # Keep track of user prompt input\n",
    "# for label in sorted(np.unique(ap_cluster.labels_)):\n",
    "#     prompt_tuple = []\n",
    "#     indices = df_precompute_sim_matrix.index[np.where(ap_cluster.labels_ == label)]\n",
    "#     text = input(f\"Cluster: {indices}. Would you like to replace this cluster? (Y/N)\")\n",
    "#     replacement_text = None\n",
    "#     if text == \"Y\" or text == \"y\":\n",
    "#         replacement_text = input(f\"Cluster: {indices}. Enter the text your would like to replace this cluster with:\")\n",
    "#     else:\n",
    "#         pass\n",
    "\n",
    "#     prompt_tuple.append(text)\n",
    "#     prompt_tuple.append(indices.tolist())\n",
    "#     prompt_tuple.append(replacement_text)\n",
    "#     replacement_tracker.append(prompt_tuple)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "outputs": [],
   "source": [
    "'''\n",
    "    Write out replacement file. In the form of [Should_I_Replace_Cluster_With_Text, Replacement_Text]\n",
    "    [[\"Y\", \"Ashvin\"], [\"Y\", \"Bruno\"], [\"N\", None]]\n",
    "'''\n",
    "\n",
    "# json_obj = json.dumps(replacement_tracker)\n",
    "# with open(\"./survey_text_replacements/35_Favorite_Professors.json\", \"w\") as f:\n",
    "#     f.write(json_obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(f\"./survey_text_replacements/35_Favorite_Professors.json\") \n",
    "list_of_profs = json.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for text, indices, replacement_text in list_of_profs:\n",
    "    if text == \"Y\" or text == \"y\":\n",
    "        df_profs[\"FavProfs\"].replace({index: replacement_text for index in indices}, inplace=True)\n",
    "    else:\n",
    "        continue\n",
    "    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "### Manual Text Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_multiple_choice_answers[\"FavProfs\"].sort_values().unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "replacement_dict = {\"Adve\" : \"Ravi Adve\", \"Ashvin Gole\" : \"Ashvin Goel\", \"Kschischang\" : 'Frank Kschischang',\n",
    "                    \"Papernot\" : \"Michael Papernot\", \"Rose\" : \"Jonathan Rose\", \"Tarek\" : \"Tarek Abdelrahman\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_profs[\"FavProfs\"].replace(replacement_dict, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_profs[\"FavProfs\"].sort_values().unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_top_10_profs = df_profs.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_top_10_profs = df_top_10_profs.head(10).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_top_10_profs.columns = [\"FavProfs\", \"num_students\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_top_10_profs[\"percent_students\"] = (df_top_10_profs[\"num_students\"] / len(df) * 100).round(2)\n",
    "df_top_10_profs[\"Plot Text\"] = df_top_10_profs[\"num_students\"].astype(str) + \" (\" + df_top_10_profs[\"percent_students\"].astype(str) + \"%)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(df_top_10_profs, x=\"FavProfs\", y=\"num_students\", color=\"FavProfs\", text=df_top_10_profs[\"Plot Text\"])\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"bar\"][\"hovertemplate\"],\n",
    "                  )\n",
    "fig.update_layout(showlegend=True, legend=dict(xanchor=\"right\", yanchor=\"top\", x=1, y=1),\n",
    "                 margin=dict(t=2, b=2))\n",
    "fig.update_xaxes(showticklabels=False, title=\"\")\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Favourite/Least Favourite Course in ECE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your favourite required course in ECE?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What is your favourite required course in ECE?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your LEAST favourite required course in ECE?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What is your LEAST favourite required course in ECE?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table_with_dropdown(df, columns=[\"What is your favourite required course in ECE?\", \n",
    "                                 \"What is your LEAST favourite required course in ECE?\"],\n",
    "                                 labels=break_text([\"What is your favourite required course in ECE?\", \n",
    "                                 \"What is your LEAST favourite required course in ECE?\"])\n",
    "                                )\n",
    "\n",
    "fig.update_layout(showlegend=False)\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Which technical electives did you take?  (course codes only)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which technical electives did you take?  (course codes only)\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tech_electives = df[\"Which technical electives did you take?  (course codes only)\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tech_electives = df_tech_electives.str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "generate_word_map(df_tech_electives, \n",
    "                  file_path=\"./embed_images/TechElectives_WordCloud.png\")\n",
    "fig = plot_word_map(\"./embed_images/TechElectives_WordCloud.png\")\n",
    "add_sample_size_annotation_for_figure(fig, df_tech_electives, ref=\"paper\")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df_tech_electives, label=\"TOP 10 technical electives\", limit=10,\n",
    "             cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What Complimentary Studies (CSS)/Humanities & Social Science (HSS) electives & Science/Math courses did you take?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.rename({\"What Complimentary Studies (CSS)/Humanities &amp; Social Science (HSS) electives did you take? (course codes only)\" : \n",
    "           \"electives\"}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"electives\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_electives = df[\"electives\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_electives = df_electives.str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What Science/Math did you take? (course code only)\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_math_science = df[\"What Science/Math did you take? (course code only)\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_math_science = df_math_science.str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_math_science.replace('CHE\\xa0353.00', 'CHE353', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_extra_courses = pd.concat([df_electives, df_math_science], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table_with_dropdown(df_extra_courses, columns=[\"electives\", \n",
    "                                 \"What Science/Math did you take? (course code only)\"],\n",
    "                                 labels=break_text([\n",
    "                                    f\"Top 10 Complimentary Studies (CSS)/Humanities & Social Science (HSS) electives. <i>(Sample Size: {len(df_electives)})</i>\",\n",
    "                                    f\"Top 10 Science/Math Courses (course code only) <i>(Sample Size: {len(df_math_science)})</i>\"\n",
    "                                 ], char_limit=40),\n",
    "                                 limit=10,\n",
    "                                 base_height=230\n",
    "                                )\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Have you ever take summer courses?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Have you ever take summer courses?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Have you ever take summer courses?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df, x=\"Have you ever take summer courses?\", color=\"Have you ever take summer courses?\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "add_percent_labels_for_hist(df[\"Have you ever take summer courses?\"], fig, horizontal=False)\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig = px.histogram(df, x=\"Have you ever overloaded (taken 6 or more courses in a semester)?\", \n",
    "#                    color=\"Have you ever overloaded (taken 6 or more courses in a semester)?\")\n",
    "# fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DQ check for no overloaded semesters but overloaded terms exist.\n",
    "\n",
    "# term_filter = (\n",
    "#  (df[\"What terms did you overload?\"].notna()) & \n",
    "#  ((df[\"Have you ever overloaded (taken 6 or more courses in a semester)?\"] == \"no\") |\n",
    "#  (df[\"Have you ever overloaded (taken 6 or more courses in a semester)?\"] == \"Did not answer\"))\n",
    "# )\n",
    "\n",
    "# df.loc[term_filter, [\"Have you ever overloaded (taken 6 or more courses in a semester)?\", \"What terms did you overload?\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What terms did you overload?\"].dropna(inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Course Overload Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Have you ever overloaded (taken 6 or more courses in a semester)?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Have you ever overloaded (taken 6 or more courses in a semester)?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What terms did you overload?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_figure_with_dropdown(df, options=[\"Have you ever overloaded (taken 6 or more courses in a semester)?\", \n",
    "                                               \"What terms did you overload?\"],\n",
    "                                        labels=break_text([\"Have you ever overloaded (taken 6 or more courses in a semester)?\", \n",
    "                                                \"Which terms did you overload? (For those who did overload)\"\n",
    "                                                f\"<i> (Sample Size: {df['What terms did you overload?'].notna().sum()})</i>\"], char_limit=30),\n",
    "                                        showticklabels=[False,\n",
    "                                                         False],\n",
    "                                        sort_traces=True)\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_layout(legend=dict(yanchor=\"top\", y=1))\n",
    "# fig.layout.updatemenus[0].y = -0.4\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Failures\n",
    "\n",
    "<i>why you still no doctor la</i>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Have you ever failed a course?\"].fillna(\"Too embarassed to say\", inplace=True)\n",
    "df[\"Have you ever failed a midterm/exam?\"].fillna(\"Too embarassed to say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Have you ever failed a course?\"], check_unique=True)\n",
    "# data_checker(df[\"Have you ever failed a midterm/exam?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_figure_with_dropdown(df, [\"Have you ever failed a course?\", \"Have you ever failed a midterm/exam?\"],\n",
    "                            is_order_manually=True, manual_category_orders=[[\"yes\", \"no\", \"Too embarassed to say\"], None])\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_layout(legend=dict(yanchor=\"top\", y=1))\n",
    "fig.layout.updatemenus[0].y = -0.2\n",
    "fig.show(renderer=\"notebook\")\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Contentness with UofT Life"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How satisfied are you with your academic experience at UofT? (Out of 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How satisfied are you with your academic experience at UofT?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_uoft_satisfaction = df[\"How satisfied are you with your academic experience at UofT?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = create_rating_distributions(\"out of 10\", \"Contentedness with UofT Academic Experience<br><i>(Rating out of 10)</i>\",\n",
    "                            df_uoft_satisfaction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How satisfied are you with your student life at UofT?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How satisfied are you with your student life at UofT?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_student_life = df[\"How satisfied are you with your student life at UofT?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = create_rating_distributions(\"out of 10\", \"Contentedness with UofT Student Life Experience<br><i>(Rating out of 10)</i>\",\n",
    "                            df_student_life)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show(height=600)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Would you choose your same program again?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Would you choose your same program again?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Would you choose your same program again?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df, column=\"Would you choose your same program again?\", label=break_text(\"Would you choose your same program again?\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## If you responded no, which program would you have chosen instead?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"If you responded no, which program would you have chosen instead?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_which_progam_instead = df[\"If you responded no, which program would you have chosen instead?\"].dropna().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df_which_progam_instead, names=\"If you responded no, which program would you have chosen instead?\")\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "                  textposition=\"inside\",\n",
    "                  hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "fig.update_layout(margin=dict(l=0, t=2, b=2, r=0))\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", x=0.98, y=0.02, text=f\"Sample Size: {len(df_which_progam_instead)}\", font=dict(size=16), ax=0, ay=0)\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How likely is it you would recommend this department to other prospective students?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How likely is it you would recommend this department to other prospective students?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_likely_to_recommend_program = \\\n",
    "    df[\"How likely is it you would recommend this department to other prospective students?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = \\\n",
    "    create_rating_distributions(\"out of 10\", \n",
    "                                \"Likely to Recommend ECE Program<br><i>(Rating out of 10)</i>\",\n",
    "                                df_likely_to_recommend_program)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I developed close friendships within my ECE class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"I developed close friendships within my ECE class\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_closeness_of_friendships = df[\"I developed close friendships within my ECE class\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = \\\n",
    "    create_rating_distributions(\"out of 5\", \n",
    "                                \"Closeness of Friendships Developed<br><i>(Rating out of 5)</i>\",\n",
    "                                df_closeness_of_friendships)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall, my class is tight-knit and friendly with each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# data_checker(df[\"Overall, my class is tight-knit and friendly with each other\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tightness = df[\"Overall, my class is tight-knit and friendly with each other\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = \\\n",
    "    create_rating_distributions(\"out of 5\", \n",
    "                                \"Tightness and Friendliness of Class<br><i>(Rating out of 5)</i>\",\n",
    "                                df_tightness)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many close friends would you say you have in the ECE 2T2 cohort? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many close friends would you say you have in the ECE 2T2 cohort?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_close_friends_str = df[\"How many close friends would you say you have in the ECE 2T2 cohort?\"] \\\n",
    "                       .dropna().astype(str) \\\n",
    "                       .str.split(\".\").str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_close_friends_sorted = df[\"How many close friends would you say you have in the ECE 2T2 cohort?\"].dropna() \\\n",
    "                         .sort_values().astype(int).astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_close_friends_str, color=\"value\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_xaxes(categoryorder=\"array\", categoryarray=df_close_friends_sorted.unique(), title_text=\"Number of Close Friends\",\n",
    "showgrid=False)\n",
    "fig.update_layout(margin=dict(t=5, b=2))\n",
    "add_percent_labels_for_hist(df_close_friends_str, fig, horizontal=False)\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", yanchor=\"middle\", x=1, y=11, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_close_friends_str)}\", font=dict(size=17))\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many friends would you say you have in the ECE 2T2 cohort? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many friends would you say you have in the ECE 2T2 cohort?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_friends_str = df[\"How many friends would you say you have in the ECE 2T2 cohort?\"] \\\n",
    "                       .dropna().astype(str) \\\n",
    "                       .str.split(\".\").str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_friends_sorted = df[\"How many friends would you say you have in the ECE 2T2 cohort?\"].dropna() \\\n",
    "                         .sort_values().astype(int).astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_friends_str, color=\"value\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_xaxes(categoryorder=\"array\", categoryarray=df_friends_sorted.unique(), title_text=\"Number of Friends\", showgrid=False)\n",
    "add_percent_labels_for_hist(df_friends_str, fig, horizontal=False)\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", yanchor=\"middle\", x=0.75, y=14, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_friends_str)}\")\n",
    ")\n",
    "fig.update_layout(margin=dict(t=5, b=2), font=dict(size=17))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Overall, my feelings about the ECE department are\"], check_unique=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall, my feelings about the ECE departments are"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Overall, my feelings about the ECE department are\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Overall, my feelings about the ECE department are\", color=\"Overall, my feelings about the ECE department are\")\n",
    "fig.update_traces(texttemplate=\"%{label} <br> %{value} (%{percent}%)\",\n",
    "                  textposition=\"inside\",\n",
    "                  hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=0, r=0))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ECE Mental Health"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## During your time at UofT, have you experienced any of the following?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"During your time at UofT, have you experienced any of the following?\"],\n",
    "#             check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"During your time at UofT, have you experienced any of the following?\"].fillna(\"No answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_experience = process_multiple_choice_survey(df[\"During your time at UofT, have you experienced any of the following?\"],\n",
    "                                                      column=\"mental\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_experience_meta = get_percentages_for_multiple_choice_survey(df_mental_experience[\"mental\"], \n",
    "                                                                       df[\"During your time at UofT, have you experienced any of the following?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(df_mental_experience_meta, x=\"index\", y=\"count\", text=\"percentage\", color=\"index\", labels={\n",
    "      \"index\" :  \"Mental Disorder\"\n",
    "})\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"bar\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{value} (%{text})\")\n",
    "fig.update_xaxes(title=\"\", showticklabels=False, showgrid=False)\n",
    "fig.update_layout(showlegend=True, legend=dict(yanchor=\"top\", xanchor=\"right\", x=1, y=1))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Which aspects, if any, contributed to your mental health negatively?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which aspects, if any, contributed to your mental health negatively?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Which aspects, if any, contributed to your mental health negatively?\"].fillna(\"No answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_negative = process_multiple_choice_survey(df[\"Which aspects, if any, contributed to your mental health negatively?\"],\n",
    "                                                      column=\"mental_negative_impact\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_negative_meta = get_percentages_for_multiple_choice_survey(df_mental_negative[\"mental_negative_impact\"], df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(df_mental_negative_meta, x=\"index\", y=\"count\", text=\"percentage\", color=\"index\", labels={\n",
    "      \"index\" :  \"Contributions to Negative Mental Health\"\n",
    "})\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"bar\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{value} (%{text})\")\n",
    "fig.update_xaxes(title=\"\", showticklabels=False, showgrid=False)\n",
    "fig.update_layout(showlegend=True, legend=dict(yanchor=\"top\", xanchor=\"right\", x=1, y=1),\n",
    "                  margin=dict(t=5, b=2))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Have you sought out mental health support at the university?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Have you sought out mental health support at the university?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Have you sought out mental health support at the university?\"].fillna(\"No answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Have you sought out mental health support at the university?\",\n",
    "       color=\"Have you sought out mental health support at the university?\")\n",
    "\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\")\n",
    "fig.update_layout(margin=dict(b=0, l=0, r=0, t=0))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Which mental health supports did you seek out?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which mental health supports did you seek out?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_supports_multiple_choice = df[\"Which mental health supports did you seek out?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_supports = process_multiple_choice_survey(df_mental_supports_multiple_choice,\n",
    "                                                      column=\"supports\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_supports = get_percentages_for_multiple_choice_survey(df_mental_supports[\"supports\"], df_mental_supports_multiple_choice)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(df_mental_supports, x=\"index\", y=\"count\", text=\"percentage\", color=\"index\", labels={\n",
    "      \"index\" :  \"Mental Health Resources\"\n",
    "})\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"bar\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{value} (%{text})\")\n",
    "fig.update_xaxes(title=\"\", showticklabels=False, showgrid=False)\n",
    "add_sample_size_annotation_for_figure(fig, df_mental_supports_multiple_choice, x=1, y=-0.05, ref=\"paper\")\n",
    "fig.update_layout(showlegend=True, legend=dict(yanchor=\"top\", xanchor=\"right\", x=1, y=1),\n",
    "                  margin=dict(b=2, t=5))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Would you attribute changes in your mental health, if any, to your university experiences?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Would you attribute changes in your mental health, if any, to your university experiences?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mental_changes = df[\"Would you attribute changes in your mental health, if any, to your university experiences?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = \\\n",
    "    create_rating_distributions(\"out of 5\", \n",
    "                                \"Mental Changes due to University Experiences<br><i>(Rating out of 5)</i>\",\n",
    "                                df_mental_changes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do you think UofT has adequate mental health resources fro students?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Do you think UofT has adequate mental health resources for students?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df, column=\"Do you think UofT has adequate mental health resources for students?\",\n",
    "            label=break_text(\"Do you think UofT has adequate mental health resources for students?\", char_limit=25))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PEY Profile\n",
    "\n",
    "ECE2T2's PEY Experience"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Did you take a course over PEY?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Did you take a course over PEY?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Did you take a course over PEY?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Did you take a course over PEY?\", color=\"Did you take a course over PEY?\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\", textposition=\"inside\")\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=0, r=0))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What area(s) did you do your PEY Co-op?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What area(s) did you do your PEY Co-op?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pey_area = df[\"What area(s) did you do your PEY Co-op?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_pey_area, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_area = process_multiple_choice_survey(df_pey_area, column=\"Area\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_area[\"Area\"], file_path=\"./embed_images/PEY_Area_WordCloud.png\", is_frequency=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/PEY_Area_WordCloud.png\")\n",
    "add_sample_size_annotation_for_figure(fig, df_pey_area, ref=\"paper\", x=1)\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_table(df_area, column=\"Area\", label=\"Top 10 Co-op Areas\", limit=10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Where did you do your PEY-Co-op?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Where did you do your PEY Co-op?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Where did you do your PEY Co-op?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Where did you do your PEY Co-op?\", color=\"Where did you do your PEY Co-op?\", hole=0.7)\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\", hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "fig.update_layout(margin=dict(b=0, l=0, t=0, r=0))\n",
    "fig.show(width=900)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What was your hourly pay for PEY?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What was your hourly pay for PEY?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.box(df, y=\"What was your hourly pay for PEY?\", notched=True, points=\"all\")\n",
    "fig.update_xaxes(showgrid=False)\n",
    "fig.update_layout(yaxis_range=[20, df[\"What was your hourly pay for PEY?\"].max()],\n",
    "                  margin=dict(t=0, b=0))\n",
    "add_sample_size_annotation_for_figure(fig, df[\"What was your hourly pay for PEY?\"].dropna(), ref=\"paper\")\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Who was your employer?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Who was your employer?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "employer_series = df[\"Who was your employer?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(employer_series, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_employer = process_multiple_choice_survey(employer_series, column=\"employer\", delimeter=\"[.,]\\s*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_employer.replace([\"GeoTab\"], \"Geotab\", inplace=True)\n",
    "df_employer.replace([\"Intel of Canada\"], \"Intel\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_employer[\"employer\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_employer[\"employer\"], file_path=\"./embed_images/PEY_Employer_WordCloud.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(file_path=\"./embed_images/PEY_Employer_WordCloud.png\")\n",
    "add_sample_size_annotation_for_figure(fig, employer_series, ref=\"paper\")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_table(df_employer, column=\"employer\", label=\"TOP 10 Employers\", limit=10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ECE2T2's Ratings of Their Work Experience"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Please rate your work term experience.\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_work_term_exp = df[\"Please rate your work term experience.\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = create_rating_distributions(\"out of 10\", \"Work Experience Rating\", df_work_term_exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Out of 10, Did ECE2T2's Think That the Pandemic Affeted Their Job Search for PEY Co-op Negatively?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Do you think that the pandemic affected your search for a PEY Co-op negatively?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_negative_affect = df[\"Do you think that the pandemic affected your search for a PEY Co-op negatively?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = create_rating_distributions(\"out of 10\", \"How much negative impact did the pandemic have<br>on your search for a PEY Co-op?\", df_negative_affect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show(height=600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Will you be returning to your past PEY employer?\"], check_unique=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Will ECE2T2's Be Returning to Their Past PEY Employer?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Will you be returning to your past PEY employer?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df, column=\"Will you be returning to your past PEY employer?\", label=\"\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# After-Grad Life"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"After graduation, my next step is..\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"After graduation, my next step is..\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"After graduation, my next step is..\", color=\"After graduation, my next step is..\",\n",
    "             hole=0.4)\n",
    "fig.update_layout(margin=dict(b=0, t=0))\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "                  textposition=\"outside\",\n",
    "                  hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Will you obtain your Professional Engineering (P.Eng) designation?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Will you obtain your Professional Engineering (P.Eng) designation?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Will you obtain your Professional Engineering (P.Eng) designation?\"].fillna(\"Did not answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df, column=\"Will you obtain your Professional Engineering (P.Eng) designation?\", label=\"\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## If you are doing grad school, where will you study?\n",
    "\n",
    "At the time of the survey response, not everyone was set on going to grad school or they were waiting for confirmation of enrollment, so not everyone currently going to grad school answered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"If you are doing grad school, where will you study?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_where_to_study = df[\"If you are doing grad school, where will you study?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_where_to_study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df_where_to_study, names=\"If you are doing grad school, where will you study?\")\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\")\n",
    "fig.update_layout(height=600, width=600, margin=dict(l=2, r=2, t=0, b=0))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# After-Grad Life: Employment"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regardless of your current circumstance, where do you prefer for your Full-Time job to be?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Regardless of your current circumstance, where do you prefer for your Full-Time job to be?\"],\n",
    "#              check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Regardless of your current circumstance, where do you prefer for your Full-Time job to be?\"].fillna(\"No Answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df, x=\"Regardless of your current circumstance, where do you prefer for your Full-Time job to be?\",\n",
    "                   color=\"Regardless of your current circumstance, where do you prefer for your Full-Time job to be?\")\n",
    "add_percent_labels_for_hist(df[\"Regardless of your current circumstance, where do you prefer for your Full-Time job to be?\"], \n",
    "                            fig)\n",
    "fig.update_xaxes(showticklabels=False, title=\"\", categoryorder=\"total descending\")\n",
    "fig.update_layout(showlegend=True, legend=dict(xanchor=\"right\", x=1, y=1, yanchor=\"top\", title=\"\", font=dict(size=20)),\n",
    "                  margin=dict(t=2, b=0))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## If you are working, what is your Full-Time Base Salary for your role?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"If you are working, what is your Full-Time Base Salary for your role?\"],\n",
    "#              check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Currency: USD, CAD, EUR (1)\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full_time_base_salary = df[\"If you are working, what is your Full-Time Base Salary for your role?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Currency: USD, CAD, EUR (1)\"] = df[\"Currency: USD, CAD, EUR (1)\"].str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_currency = df[\"Currency: USD, CAD, EUR (1)\"].iloc[df_full_time_base_salary.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_currency, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.loc[df[\"Currency: USD, CAD, EUR (1)\"].isna() & \n",
    "#        df[\"If you are working, what is your Full-Time Base Salary for your role?\"].notna(), [\"If you are working, what is your Full-Time Base Salary for your role?\", \"Currency: USD, CAD, EUR (1)\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_currency = df_currency.dropna()\n",
    "df_full_time_base_salary = df_full_time_base_salary.loc[df_full_time_base_salary != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(df_currency)), len(df_full_time_base_salary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Full-Time Base Salary (CAD)\"] = df_full_time_base_salary * np.where(\n",
    "    df_currency == \"USD\", 1.33, 1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig = px.box(df, x=\"Full-Time Base Salary (CAD)\", points=\"all\")\n",
    "fig.update_layout(margin=dict(l=0, t=10, b=60))\n",
    "fig.update_xaxes(range=[50000, df[\"Full-Time Base Salary (CAD)\"].max() * 11/10])\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, text=f\"Sample Size: {len(df_full_time_base_salary)}\", font=dict(size=16), ax=0, ay=0)\n",
    ")\n",
    "fig"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## If you are working, what is your Full-Time Total Compensation for your role?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<i>$0 guy</i>, I appreciate you for standing up to evil corporate power cuz I, too, am a victim of it. I was once too afraid to stand up, but now you have given me voice and reason. I commend you comrade. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"If you are working, what is your Full-Time Total Compensation for your role?\"],\n",
    "#              check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Currency: USD, CAD, EUR (2)\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full_time_compensation = df[\"If you are working, what is your Full-Time Total Compensation for your role?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Currency: USD, CAD, EUR (2)\"] = df[\"Currency: USD, CAD, EUR (2)\"].str.upper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Currency: USD, CAD, EUR (2)\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(df.loc[\n",
    "#     df[\"Currency: USD, CAD, EUR (2)\"].isna(), [\"If you are working, what is your Full-Time Total Compensation for your role?\",\n",
    "#                                                \"Currency: USD, CAD, EUR (2)\"]\n",
    "# ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_currency_2 = df[\"Currency: USD, CAD, EUR (2)\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Full-Time Compensation (CAD)\"] = df_full_time_compensation * np.where(\n",
    "    df_currency_2 == \"USD\", 1.33, 1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.box(df, x=\"Full-Time Compensation (CAD)\", points=\"all\")\n",
    "fig.update_xaxes(range=[0, df[\"Full-Time Compensation (CAD)\"].max() * 11/10])\n",
    "fig.update_layout(margin=dict(l=0, t=10, b=60))\n",
    "add_sample_size_annotation_for_figure(fig, df_full_time_compensation, ref=\"paper\")\n",
    "fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you are working, what is your Full-Time Total Compensation for your role?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What company are you going to for Full-Time Work?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which company are you going to for Full-Time work?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_full_time_employer = df[\"Which company are you going to for Full-Time work?\"].dropna().str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_full_time_employer, \n",
    "                  file_path=\"./embed_images/Full_Time_Employer_WordCloud.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Full_Time_Employer_WordCloud.png\")\n",
    "add_sample_size_annotation_for_figure(fig, df_full_time_employer, ref=\"paper\")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(pd.DataFrame({\"Full Time Employer\": df_full_time_employer.tolist()}), column=\"Full Time Employer\", label=\"TOP 10 Full-Time Employers\",\n",
    "limit=10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## When did you start and finish your Full-Time Job Search?\n",
    "\n",
    "For everyone who indicated a start time, but didn't indicate an end time, I imputed an end time with the date of the survey's publification (June 20th).\n",
    "\n",
    "The earliest result for this survey was collected on *Insert Data Here*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"When did you start your Full-Time Job Search?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"When did you stop your your Full-Time Job Search?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_start_end_job_search = df.loc[:, [\"Respondent ID\", \"When did you start your Full-Time Job Search?\",\n",
    "                              \"When did you stop your your Full-Time Job Search?\"]]\n",
    "\n",
    "df_start_end_job_search.columns = [\"Task\", \"Start\", \"Finish\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_start_end_job_search[\"Finish\"].fillna(str(datetime.now()).split()[0], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = px.timeline(df_start_end_job_search, x_start=\"Start\", x_end=\"Finish\", y=\"Task\", color=\"Task\",\n",
    "                 color_discrete_sequence=px.colors.qualitative.Pastel)\n",
    "fig.update_yaxes(showticklabels=False, showgrid=False, title=\"ECE2T2 Student\")\n",
    "fig.update_traces(hovertemplate=\"<b>Start</b>: %{base}<br><b>Finish:</b> %{x}<extra></extra>\")\n",
    "fig.update_layout(title_text=\"\", width=900, height=600)\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Props to all of y'all who wrapped up your job search.\n",
    "Also giving flowers to those who are working hard to find a job in the present.\n",
    "\n",
    "To the person who claims to have ended their job search in Sep 2023, either you are one cocky engineer, or you were offered a job but still want to look for other options until the said date."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Are you returning to companies you have previously worked at?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Are you returning to the company where you did your PEY?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Are you returning to the company where you did your PEY?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Are you returning to the company where you did a summer internship?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Are you returning to the company where you did a summer internship?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table_with_dropdown(df, columns=[\"Are you returning to the company where you did your PEY?\", \n",
    "                          \"Are you returning to the company where you did a summer internship?\"])\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What are the reasons you chose your Full-Time Job?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What are the reasons you chose your Full-Time Job?\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What are the reasons you chose your Full-Time Job?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reason_full_time = process_multiple_choice_survey(df[\"What are the reasons you chose your Full-Time Job?\"], column=\"Reason_Full_Time\")\n",
    "df_reason_full_time_with_percentages = get_percentages_for_multiple_choice_survey(df_reason_full_time[\"Reason_Full_Time\"], df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_reason_full_time_with_percentages.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.bar(df_reason_full_time_with_percentages, y=\"index\", x=\"count\",\n",
    "       color=\"index\")\n",
    "fig.update_yaxes(categoryorder=\"total ascending\", title=\"\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram_h\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{value} (%{text})\", text=df_reason_full_time_with_percentages[\"percentage\"])\n",
    "fig.update_layout(font=dict(size=13), margin=dict(t=5, b=2))\n",
    "fig.show(width=800)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# After-Grad Life: Financial Situation\n",
    "\n",
    "How much debt are you graduating with?\n",
    "How much of your tuition did you pay on your own (%) ?\n",
    "Did you receive OSAP at least once?\n",
    "Did you receive UTAPS at least once?\n",
    "Would you consider donating to UofT in the future?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How much debt are you graduating with?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How much debt are you graduating with?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_debt = df[\"How much debt are you graduating with?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_debt.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_debt_binned = pd.cut(df_debt, right=True, bins=[-1, 10000, 20000, 30000, 40000, 50000], labels = [\"${} - {}\".format(debt,  debt + 10000) for debt in range(0, 40001, 10000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_debt_binned, orientation=\"h\", color=\"value\")\n",
    "# fig.update_traces(),\n",
    "# texttemplate=\"%{value}\"\n",
    "# )\n",
    "\n",
    "add_percent_labels_for_hist(df_debt_binned, fig, horizontal=True)\n",
    "fig.update_yaxes(title_text=\"debt\", categoryorder=\"array\", categoryarray=df_debt_binned.sort_values(ascending=False).unique(),\n",
    "                showgrid=False)\n",
    "# fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram_h\"][\"hovertemplate\"],\n",
    "#                   text=df_debt_percentages[\"percentage\"],\n",
    "#                   texttemplate=\"%{value} (%{text})\")\n",
    "fig.update_layout(bargap=0.2, margin=dict(t=5, b=2))\n",
    "add_sample_size_annotation_for_figure(fig, df_debt_binned, ref=\"paper\")\n",
    "# fig.show()\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How much of your tuition did you pay on your own (%) ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How much of your tuition did you pay on your own (%) ?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tuition_owned = df[\"How much of your tuition did you pay on your own (%) ?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tuition_owned_binned = pd.cut(df_tuition_owned, bins=[-1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 101], labels=[\"{} to {}%\".format(i, i + 10) for i in range(0, 100, 10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_tuition_owned_binned, color=\"value\", orientation=\"h\")\n",
    "fig.update_yaxes(categoryorder=\"array\", categoryarray=df_tuition_owned_binned.sort_values(ascending=False).unique())\n",
    "fig.update_layout(margin=dict(t=5, b=2))\n",
    "add_percent_labels_for_hist(df_tuition_owned_binned, fig, horizontal=True)\n",
    "add_sample_size_annotation_for_figure(fig, df_tuition_owned, ref=\"paper\")\n",
    "fig"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other Financial Profiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Did you receive OSAP at least once?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Did you receive OSAP at least once?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Did you receive UTAPS at least once?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Did you receive UTAPS at least once?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Would you consider donating to UofT in the future?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Would you consider donating to UofT in the future?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table_with_dropdown(df, columns=[\"Did you receive OSAP at least once?\", \"Did you receive UTAPS at least once?\",\n",
    "                          \"Would you consider donating to UofT in the future?\"], reindex=[False, True, True],\n",
    "                          reindex_orders=[None, [\"yes\", \"no\", \"Did not specify\"], [\"yes\", \"no\", \"Did not specify\"]], base_height=250)\n",
    "fig.show(renderer=\"notebook\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# After-Grad Life: Miscellnaneous"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What will you do after an ECE-related career, apart from retirement?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What will you do after an ECE-related career, apart from retirement?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"What will you do after an ECE-related career, apart from retirement?\"] = \\\n",
    "    df[\"What will you do after an ECE-related career, apart from retirement?\"].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ece_after_retirement = df[\"What will you do after an ECE-related career, apart from retirement?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_ece_after_retirement, file_path=\"./embed_images/AfterECE.png\", is_frequency=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/AfterECE.png\")\n",
    "add_sample_size_annotation_for_figure(fig, df_ece_after_retirement, ref=\"paper\")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Are you planning to do Grad Trip?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Are you planning to do Grad Trip?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Are you planning to do Grad Trip?\"].fillna(\"Did not specify\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df, column=\"Are you planning to do Grad Trip?\", label=\"\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Post-Grad Contentness, Out of 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Please score your post-grad contentedness\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_post_grad_contentedness = df[\"Please score your post-grad contentedness\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_post_grad_contentedness_str = df_post_grad_contentedness.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig_avg, fig = create_rating_distributions(\"out of 5\", \"Post-Grad Contentedness<br><i>(Out of 5)</i>\", df_post_grad_contentedness)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig_avg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Habits and Preferences"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many hours a week do you spend with your parents (online/digitally)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many hours a week do you spend with your parents (online/digitally)?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hours_with_parents = df[\"How many hours a week do you spend with your parents (online/digitally)?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hours_with_parents_str = df_hours_with_parents.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_hours_with_parents_str, color=\"value\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_xaxes(title=\"hours\", categoryorder=\"array\", categoryarray=df_hours_with_parents.sort_values().astype(str),\n",
    "showgrid=False)\n",
    "fig.update_yaxes(showgrid=False)\n",
    "fig.add_vline(x=2.5, line_width=3, line_dash=\"dash\", line_color=px.colors.sequential.Viridis[5])\n",
    "fig.add_annotation(x=2.5, y=12, text=f\"Average: {format(df_hours_with_parents.mean(), '.2f')} hours<br>a week\", ax=10, xanchor=\"left\")\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", x=1, y=-0.15, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_hours_with_parents)}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many hours a week do you spend with your parents (in-person)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many hours a week do you spend with your parents (in-person)?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hours_with_parents_in_person = df[\"How many hours a week do you spend with your parents (in-person)?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hours_with_parents_in_person_str = df_hours_with_parents_in_person.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_hours_with_parents_in_person_str, color=\"value\")\n",
    "fig.update_traces(hovertemplate=less_cringe_hovertext_template[\"histogram\"][\"hovertemplate\"])\n",
    "fig.update_xaxes(title=\"hours\", categoryorder=\"array\", categoryarray=df_hours_with_parents_in_person.sort_values().astype(str),\n",
    "                showgrid=False)\n",
    "fig.add_vline(x=5.55, line_width=3, line_dash=\"dash\", line_color=px.colors.sequential.Viridis[5])\n",
    "fig.add_annotation(x=5.55, y=12, text=f\"Average: {format(df_hours_with_parents_in_person.mean(), '.2f')} hours<br>a week\", ax=10, xanchor=\"left\")\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", x=1, y=-0.2, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_hours_with_parents_in_person)}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How often do you exercise?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How often do you exercise?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"How often do you exercise?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"How often do you exercise?\", color=\"How often do you exercise?\")\n",
    "fig.update_traces(textinfo=\"label+percent\", hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"],\n",
    "                  texttemplate=\"%{label}<br>%{value} (%{percent})\")\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=0, r=0))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What types of physical activity do you do?\n",
    "\n",
    "Apparently someone's favourite physical activity is walking with <i>Julien Zhu</i>.\n",
    "That's great! If you can get your 150 minutes with him every week"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What types of physical activity do you do?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_activity = df[\"What types of physical activity do you do?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_activity_processed = process_multiple_choice_survey(df_activity, column=\"activity\", delimeter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_activity_cleaned = df_activity_processed[\"activity\"].str.strip().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## ONLY RUN ONCE. Clustering Algorithm to Replace Repeating Words\n",
    "\n",
    "# df_activity_sim_matrix = generate_sim_matrix(df_activity_cleaned)\n",
    "# df_activity_sim_matrix_cleaned = remove_unique(df_activity_sim_matrix, threshold=0.25)\n",
    "# ap_cluster = fit_data(df_activity_sim_matrix_cleaned)\n",
    "# generate_replacement_file(ap_cluster, df_activity_sim_matrix_cleaned, output_file='./survey_text_replacements/Physical_Activity.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "execute_replacements(df_activity_cleaned, infile='./survey_text_replacements/Physical_Activity.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_activity_cleaned, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_activity_manual_cluster = df_activity_cleaned.replace([\"biking\"], \"Biking\") \\\n",
    "                   .replace([\"Dance\", \"dancing\"], \"Dancing\") \\\n",
    "                   .replace([\"go to the gym\", \"gym\", \"Gymming\"], \"Gym\") \\\n",
    "                   .replace([\"Hockey\", \"hockey\"], \"Hockey\") \\\n",
    "                   .replace([\"weight lifting\", \"Weight-Lifting\", \"Weight-lifting\", \"Weightlifting\",\n",
    "                             \"weightlifting\", \"weight lifting\", \"Weight lifting\"], \"Weight Lifting\") \\\n",
    "                   .replace([\"working out\", \"workout\"], \"workout\")  \\\n",
    "                   .replace([\"Walk\", \"Walking\", \"walking\"], \"Walking\") \\\n",
    "                   .replace([\"run\"], \"running\") \\\n",
    "                   .replace(['Rock climbing', 'Rock Climbing', 'rock climbing', \"Climbing\", \"climbing\"], \"Rock Climbing\") \\\n",
    "                   .replace([\"Tennis\", \"tennis\"], \"Tennis\") \\\n",
    "                   .replace([\"Swimming\", \"swimming\"], \"Swimming\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_activity_manual_cluster, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_activity_manual_cluster, file_path=\"./embed_images/Physical_Activities_WordCloud.png\",\n",
    "                  max_words=50, is_frequency=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Physical_Activities_WordCloud.png\")\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_activity)}\", font=dict(size=15))\n",
    ")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df_activity_manual_cluster, \n",
    "                   label=f\"Top 10<br>Physical Activities<br><i>(Sample Size:{len(df_activity)})</i>\", \n",
    "                   cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT, base_height=320, limit=10)\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How often do you cook for yourself?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How often do you cook for yourself?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"How often do you cook for yourself?\"].fillna(\"Not sure\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"How often do you cook for yourself?\")\n",
    "fig.update_layout(margin=dict(l=0, r=0, b=0, t=0))\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\", \n",
    "                  textposition=\"inside\", hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "\n",
    "fig.show(height=600, width=600)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Where do you get groceries from?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Where do you get groceries from?\"],\n",
    "#              check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_groceries_multiple_choice = \\\n",
    "df[\"Where do you get groceries from?\"].str.strip().dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_groceries = process_multiple_choice_survey(df_groceries_multiple_choice, column=\"groceries\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_groceries[\"groceries\"].sort_values(key=lambda x: x.str.lower()).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_groceries[\"groceries\"].replace([\"City Market\", \"Citymarket\"], \"City Market\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Farm Boy\", \"Farmboy\"], \"Farm Boy\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Independent\", \"Independent City Market\", \"Independent supermarket\"], \"Indepedent City Market\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Loblaws\", \"loblaws\"], \"Loblaws\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"No Frills\", \"No frills\", \"No frills because it's close\", \"NoFrills\", \"Nofrills\"], \"No Frill\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Shopper's Drug Mart\", \"shoppers\", \"Shoppers\"], \"Shoppers\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Sobey\", \"sobey's\"], \"Sobey's\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Wallmart\", \"Walmart\"], \"Walmart\", inplace=True)\n",
    "df_groceries[\"groceries\"].replace([\"Freshco\", \"freshco\"], \"Freshco\", inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_groceries_processing = df_groceries.loc[df_groceries[\"groceries\"] != \"Loblaws Frescho\", \"groceries\"]\n",
    "df_groceries_final = pd.concat([df_groceries_processing, pd.Series([\"Loblaws\", \"Freshco\"])]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_groceries_final, file_path=\"./embed_images/Groceries_WordCloud.png\", is_frequency=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_groceries_final.name = \"groceries\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Groceries_WordCloud.png\")\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_groceries_multiple_choice)}\", font=dict(size=15))\n",
    ")\n",
    "fig.show(renderer=\"png\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df_groceries_final, \n",
    "                   label=f\"Top 10 Choice of<br><b>Groceries</b><br><i>(Sample Size:{len(df_groceries_multiple_choice)})</i>\",\n",
    "                   limit=10)\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Where are your favourite places to eat on campus?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Eating on campus! Where are your favourite places to eat on campus?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_campus_food_multiple_choice = \\\n",
    "df[\"Eating on campus! Where are your favourite places to eat on campus?\"].str.strip().dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_campus_food = process_multiple_choice_survey(df_campus_food_multiple_choice,\n",
    "                                                column=\"fav_food_campus\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_campus_food[\"fav_food_campus\"].sort_values(key=lambda x: x.str.lower()).unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_campus_food[\"fav_food_campus\"].replace([\"Fat Bastard Burrito\"], \"Fat Bastards\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Food Trucks\", \"Food trucks\"], \"Food Trucks\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Hard Hat Cafe\", \"Hard Hat Cafe!\", \"hardhat\"], \"Hard Hat Cafe!\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Med sci building, Med Sci building cafeteria\"], \"Med Sci Building\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Pink food truck\", \"Pink Truck\"], \"Pink Truck\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Pit\", \"pit?\"], \"Pit\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Second Cup\", \"Second cup\"], \"Second Cup\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Subway\", \"subway\", \"Subway - 12 in tuna\"], \"Subway\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Veda\", \"Veda Cafe\", \"Veda is the most decent\"], \"Veda Cafe\", inplace=True)\n",
    "df_campus_food[\"fav_food_campus\"].replace([\"Med sci building\"], \"Med Sci building cafeteria\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_campus_food[\"fav_food_campus\"], is_frequency=True, file_path=\"./embed_images/Favourite_Food_On_Campus_WordCloud.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Favourite_Food_On_Campus_WordCloud.png\")\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_campus_food_multiple_choice)}\", font=dict(size=15))\n",
    ")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df_campus_food, column=\"fav_food_campus\", \n",
    "                   label=f\"Top 5 Choice of<br><b>Campus Food</b><br><i>(Sample Size:{len(df_campus_food_multiple_choice)})</i>\", \n",
    "                   cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT, base_height=320, limit=5)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do you budget your money?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Do you budget your money\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Do you budget your money\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df, names=\"Do you budget your money\")\n",
    "fig.update_traces(texttemplate=\"%{label} <br> %{value} (%{percent})\", textposition=\"inside\")\n",
    "fig.update_layout(margin=dict(l=0, r=0, t=0, b=0))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What makes you happy?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What makes you happy?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_happy_multiple_choice = df[\"What makes you happy?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_happy = process_multiple_choice_survey(df[\"What makes you happy?\"].dropna(), column=\"Happiness\",\n",
    "                                                          delimeter=\"[,.]\\s+\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ONLY RUN ONCE\n",
    "\n",
    "# df_happy_sim_matrix = generate_sim_matrix(df_happy_multiple_choice[\"Happiness\"])\n",
    "# df_happy_sim_matrix_cleaned = remove_unique(df_happy_sim_matrix, threshold=0.25)\n",
    "# ap_cluster = fit_data(df_happy_sim_matrix_cleaned)\n",
    "# generate_replacement_file(ap_cluster, df_happy_sim_matrix_cleaned, output_file='./survey_text_replacements/Happy.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "execute_replacements(df_happy, \"Happiness\", infile='./survey_text_replacements/Happy.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_happy_multiple_choice[\"Happiness\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_happy.replace([\"Hanging out with friends and small acts of kindness:)\", \n",
    "                                  \"Hanging out with friends\", \n",
    "                                  \"Hanging with friends\",\n",
    "                                  \"Eating out with friends\",\n",
    "                                  \"hanging out in general\",\n",
    "                                  \"hanging out with friends\",\n",
    "                                  \"Friends\",\n",
    "                                  \"Friends \\ud83d\\ude01\",\n",
    "                                  \"seeing my friends\"\n",
    "                                  \"Spending time with my friends\",\n",
    "                                  \"chilling with homies\"], \"Friends 😁\", inplace=True)\n",
    "\n",
    "df_happy.replace([\"reading\", \"Reading\"], \"Reading\", inplace=True)\n",
    "df_happy.replace([\"Gaming\"], \"Video Games\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_happy_multiple_choice[\"Happiness\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_happy[\"Happiness\"], is_frequency=True, file_path=\"./embed_images/Happiness_WordCloud.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Happiness_WordCloud.png\")\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_happy_multiple_choice)}\", font=dict(size=15))\n",
    ")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df_happy, column=\"Happiness\", \n",
    "                   label=f\"Top 5 <br><b>Happiness Inducing</b><br>Moments <i>(Sample Size:{len(df_happy_multiple_choice)})</i>\", \n",
    "                   cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT, base_height=320, limit=5)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What are some hobbies that you picked up during the last 5 years?\n",
    "\n",
    "Not a surprise that physical activity appears to be a subset of hobbies <3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What are some hobbies that you picked up during the last 5 years?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hobbies_multiple_choice = df[\"What are some hobbies that you picked up during the last 5 years?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hobbies = \\\n",
    "    process_multiple_choice_survey(df_hobbies_multiple_choice, column=\"hobbies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## ONLY RUN ONCE. Clustering Algorithm to Replace Repeating Words\n",
    "\n",
    "# df_hobbies_sim_matrix = generate_sim_matrix(df_hobbies[\"hobbies\"])\n",
    "# df_hobbies_sim_matrix_cleaned = remove_unique(df_hobbies_sim_matrix, threshold=0.25)\n",
    "# ap_cluster = fit_data(df_hobbies_sim_matrix_cleaned)\n",
    "# generate_replacement_file(ap_cluster, df_hobbies_sim_matrix_cleaned, output_file='./survey_text_replacements/Hobbies.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "execute_replacements(df_hobbies, column=\"hobbies\", infile=\"./survey_text_replacements/Hobbies.json\")\n",
    "\n",
    "df_hobbies[\"hobbies\"].replace([\"listening to music\", \"music\"], \"Music\", inplace=True)\n",
    "df_hobbies[\"hobbies\"].replace([\"Cooking\", \"cooking\"], \"Cooking\", inplace=True)\n",
    "df_hobbies[\"hobbies\"].replace([\"hiking\"], \"Hiking\", inplace=True)\n",
    "df_hobbies[\"hobbies\"].replace([\"weight lifting\"], \"Weight lifting\", inplace=True)\n",
    "df_hobbies[\"hobbies\"].replace([\"Gym\"], \"Gymming\", inplace=True)\n",
    "df_hobbies[\"hobbies\"].replace([\"running\"], \"Running\", inplace=True)\n",
    "df_hobbies[\"hobbies\"].replace([\"dancing\", \"going out dancing\"], \"Dancing\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_hobbies[\"hobbies\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_hobbies[\"hobbies\"], is_frequency=True, max_words=20,\n",
    "                  file_path=\"./embed_images/Hobbies_WordCloud.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Hobbies_WordCloud.png\")\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_hobbies_multiple_choice)}\", font=dict(size=15))\n",
    ")\n",
    "fig.show(renderer=\"png\", scale=1.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df_hobbies, column=\"hobbies\", \n",
    "                   label=f\"Top 10 <br><b>Hobbies <i>(Sample Size:{len(df_hobbies_multiple_choice)})</i>\", \n",
    "                   cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT, base_height=320, limit=10)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How often do you go out?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How often do you go out?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"How often do you go out?\"].fillna(\"No Answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_histogram(df[\"How often do you go out?\"])\n",
    "fig.update_traces(marker_color=px.colors.qualitative.Pastel1[2])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Are you going to grad ball?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Are you going to grad ball?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Are you going to grad ball?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_pie(df[\"Are you going to grad ball?\"])\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many hackathons have you attended?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many hackathons have you attended?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hacks = df[\"How many hackathons have you attended?\"].dropna()\n",
    "df_hacks_str = df_hacks.astype(str).map(lambda x: x.split(\".\")[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_histogram(df_hacks_str)\n",
    "add_sample_size_annotation_for_figure(fig, df_hacks_str, ref=\"paper\", y=1, y_anchor=\"top\")\n",
    "fig.update_xaxes(categoryorder=\"array\", categoryarray=df_hacks_str.sort_values(key=lambda x: x.astype(int)).unique())\n",
    "fig"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What clubs, design teams, or committees have you been a member of at UofT?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What clubs, design teams, or committees have you been a member of at UofT?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_extra_multiple_choice = df[\"What clubs, design teams, or committees have you been a member of at UofT?\"].dropna().str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_extra_multiple_choice, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_extracurriculars = process_multiple_choice_survey(df_extra_multiple_choice, delimeter=\",\\s*|\\sand\\s*\",\n",
    "                               column=\"extracurriculars\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # ## ONLY RUN ONCE. Clustering Algorithm to Replace Repeating Words\n",
    "# df_extra_sim_matrix = generate_sim_matrix(df_extracurriculars[\"extracurriculars\"])\n",
    "# df_extra_sim_matrix_cleaned = remove_unique(df_extra_sim_matrix, threshold=0.25)\n",
    "# ap_cluster = fit_data(df_extra_sim_matrix_cleaned)\n",
    "# generate_replacement_file(ap_cluster, df_extra_sim_matrix_cleaned, output_file='./survey_text_replacements/Extracurriculars.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "execute_replacements(df_extracurriculars, column=\"extracurriculars\", infile=\"./survey_text_replacements/Extracurriculars.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_extracurriculars[\"extracurriculars\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_extracurriculars[\"extracurriculars\"].replace([\"ieeee\"], \"IEEE\")\n",
    "df_extracurriculars = df_extracurriculars.loc[df_extracurriculars[\"extracurriculars\"] != \"Project Include\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_extracurriculars[\"extracurriculars\"], file_path=\"./embed_images/Extracurriculars_WordCloud.png\", \n",
    "                  is_frequency=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "no_margin"
    ]
   },
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Extracurriculars_WordCloud.png\")\n",
    "fig.add_annotation(\n",
    "    dict(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0,\n",
    "    text=f\"Sample Size: {len(df_extra_multiple_choice)}\", font=dict(size=15))\n",
    ")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = create_table(df_extracurriculars, column=\"extracurriculars\", \n",
    "                   label=f\"Top 10 <br><b>Extracurriculars<br><i>(Sample Size:{len(df_extra_multiple_choice)})</i>\", \n",
    "                   cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT, base_height=320, limit=10)\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What Intramural Sports have you played at UofT?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What intramural sports have you played at UofT?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_intramural_multiple_choice = df[\"What intramural sports have you played at UofT?\"].dropna().str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_intramural = process_multiple_choice_survey(df_intramural_multiple_choice, column=\"intramural\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_intramural[\"intramural\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df_intramural, names=\"intramural\")\n",
    "fig.update_layout(margin=dict(l=0, b=0, r=0, t=0))\n",
    "fig.update_traces(texttemplate=\"%{label} <br> %{value} (%{percent})\",\n",
    "                  textposition=\"inside\")\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", x=0.98, y=-0.1, text=f\"{len(df_intramural_multiple_choice)} respondents<br>total\", \n",
    "         font=dict(size=16), ax=0, ay=0)\n",
    ")\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How often did you participate in an intramural sport?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How often did you participate in an intramural sport?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"How often did you participate in an intramural sport?\"].fillna(\"Did not say\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df, x=\"How often did you participate in an intramural sport?\",\n",
    "                   color=\"How often did you participate in an intramural sport?\")\n",
    "fig.update_xaxes(title=\"\", categoryorder=\"total descending\")\n",
    "add_percent_labels_for_hist(df[\"How often did you participate in an intramural sport?\"], fig)\n",
    "fig.update_layout(margin=dict(b=2, t=10))\n",
    "fig.update_traces(marker_color=px.colors.qualitative.Pastel2[4])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What activities did you participate in off-campus?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What activities did you participate in off-campus?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_off_campus_activites_multiple_choice = df[\"What activities did you participate in off-campus?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_off_campus_activites_multiple_choice, check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_off_campus_activities = process_multiple_choice_survey(df_off_campus_activites_multiple_choice, column=\"off_act\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_off_campus_activities[\"off_act\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # ## ONLY RUN ONCE. Clustering Algorithm to Replace Repeating Words\n",
    "# df_off_sim_matrix = generate_sim_matrix(df_off_campus_activities[\"off_act\"])\n",
    "# df_off_sim_matrix_cleaned = remove_unique(df_off_sim_matrix, threshold=0.15)\n",
    "# ap_cluster = fit_data(df_off_sim_matrix_cleaned)\n",
    "# generate_replacement_file(ap_cluster, df_off_sim_matrix_cleaned, output_file='./survey_text_replacements/Off_Campus_Activities.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "execute_replacements(df_off_campus_activities, column=\"off_act\", infile=\"./survey_text_replacements/Off_Campus_Activities.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df_off_campus_activities[\"off_act\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_off_campus_activities[\"off_act\"].replace([\"dance\", \"Dance\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "create_table(df_off_campus_activities, column=\"off_act\",\n",
    "             label=f\"Preferred Off-Campus <br>Activities<br><i>Sample Size: {len(df_off_campus_activites_multiple_choice)}\",\n",
    "             cell_height=TABLE_CELL_HEIGHT, header_height=TABLE_HEADER_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many times did you fall ill or were injured over the past 5 years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many times did you fall ill or were injured over the past 5 years?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"How many times did you fall ill or were injured over the past 5 years?\"].fillna(\"No Answer\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"How many times did you fall ill or were injured over the past 5 years?\"].sort_values().unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df, x=\"How many times did you fall ill or were injured over the past 5 years?\", \n",
    "             color=\"How many times did you fall ill or were injured over the past 5 years?\")\n",
    "\n",
    "fig.update_xaxes(title=\"\", categoryorder=\"array\", \n",
    "                 categoryarray=df[\"How many times did you fall ill or were injured over the past 5 years?\"].sort_values().unique())\n",
    "\n",
    "add_percent_labels_for_hist(df[\"How many times did you fall ill or were injured over the past 5 years?\"], fig)\n",
    "fig.update_layout(\n",
    "        margin=dict(t=2, b=2, l=0)\n",
    ")\n",
    "fig.update_traces(marker_color=px.colors.qualitative.Pastel1[2])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What is your preferred operating system?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your preferred operating system?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fav_op = df[\"What is your preferred operating system?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_fav_op, color=\"value\")\n",
    "\n",
    "fig.update_xaxes(showticklabels=False, title=\"\", categoryorder=\"total descending\")\n",
    "\n",
    "add_percent_labels_for_hist(df[\"What is your preferred operating system?\"], fig)\n",
    "add_sample_size_annotation_for_figure(fig, df_fav_op, ref=\"paper\", y=-0.06)\n",
    "fig.update_layout(showlegend=True,\n",
    "                  legend=dict(\n",
    "                      xanchor=\"right\", x=1, yanchor=\"top\", y=1,\n",
    "                      title=\"Operating System\",\n",
    "                      font=dict(size=19)\n",
    "                  ),\n",
    "                  margin=dict(t=2, b=2)\n",
    "    )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What is your preferred mobile operating system?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your preferred mobile operating system?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mobile_op = df[\"What is your preferred mobile operating system?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_mobile_op, color=\"value\")\n",
    "\n",
    "fig.update_xaxes(showticklabels=False, title=\"\", categoryorder=\"total descending\")\n",
    "\n",
    "add_percent_labels_for_hist(df[\"What is your preferred mobile operating system?\"], fig)\n",
    "add_sample_size_annotation_for_figure(fig, df_mobile_op, ref=\"paper\", y=-0.06)\n",
    "fig.update_layout(showlegend=True,\n",
    "                  legend=dict(\n",
    "                      xanchor=\"right\", x=1, yanchor=\"top\", y=1,\n",
    "                      font=dict(size=19), title=\"Preferred Mobile OS\"\n",
    "                  ),\n",
    "                  margin=dict(t=2, b=2)\n",
    "    )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What is your favourite programming language?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your favourite programming language?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programming_language_multiple_choice = df[\"What is your favourite programming language?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## REGEX DELIMITER OR-CHAINING ORDER MATTERS\n",
    "df_programming_language = process_multiple_choice_survey(df_programming_language_multiple_choice, column=\"prog_lang\",\n",
    "                                                         delimeter=\"[,\\s]*and|,\\s*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programming_language_with_percent = get_percentages_for_multiple_choice_survey(df_programming_language[\"prog_lang\"], \n",
    "                                           df_programming_language_multiple_choice) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_programming_language_with_percent.columns = [\"prog_lang\", \"count\", \"percentage\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(df_programming_language_with_percent, names=\"prog_lang\", values=\"count\",)\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{text})\",  text=df_programming_language_with_percent[\"percentage\"],\n",
    "                  textposition=\"inside\")\n",
    "fig.update_layout(margin=dict(l=0, r=0, t=2, b=2))\n",
    "fig.add_annotation(\n",
    "    dict(xanchor=\"right\", x=0.98, y=-0.1, text=f\"{len(df_programming_language_multiple_choice)} respondents<br>total\", \n",
    "         font=dict(size=16), ax=0, ay=0)\n",
    ")\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What is your favourite code editor?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"What is your favourite code editor?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fav_code_editor = df[\"What is your favourite code editor?\"].dropna().str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_fav_code_sim_matrix = generate_sim_matrix(df_fav_code_editor)\n",
    "# df_fav_code_sim_matrix_cleaned = remove_unique(df_fav_code_sim_matrix, threshold=0.3)\n",
    "# ap_cluster = fit_data(df_fav_code_sim_matrix_cleaned)\n",
    "# generate_replacement_file(ap_cluster, df_fav_code_sim_matrix_cleaned, output_file='./survey_text_replacements/FavCode.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "execute_replacements(df_fav_code_editor, infile=\"./survey_text_replacements/FavCode.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_fav_code_editor, orientation=\"h\", color=\"value\")\n",
    "add_percent_labels_for_hist(df_fav_code_editor, fig, horizontal=True)\n",
    "add_sample_size_annotation_for_figure(fig, df_fav_code_editor, ref=\"paper\")\n",
    "fig.update_yaxes(categoryorder=\"total ascending\", title=\"\")\n",
    "fig.update_layout(margin=dict(t=5, b=5))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How many committed relationships have you been in during university?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many committed relationships have you been in during university?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_committed = df[\"How many committed relationships have you been in during university?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_committed.astype(str), color=\"value\")\n",
    "fig.update_xaxes(title=\"Number of Committed Relationships\", categoryorder=\"array\", categoryarray=df_committed.sort_values().astype(str))\n",
    "add_percent_labels_for_hist(df_committed, fig, horizontal=False)\n",
    "fig.add_vline(x=0.5, line_width=3, line_dash=\"dash\", line_color=px.colors.sequential.Viridis[5])\n",
    "fig.add_annotation(x=0.5, y=10, text=f\"Average:<br>{format(df_committed.mean(), '.2f')} relationships\", align=\"left\", ax=10, xanchor=\"left\")\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", x=1, y=-0.15, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_committed)}\")\n",
    "fig.update_traces(marker_color=px.colors.qualitative.Pastel1[2])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Have you dated another ECE 2T2?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Have you dated another ECE 2T2?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dated = df[\"Have you dated another ECE 2T2?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_dated, color=\"value\")\n",
    "add_percent_labels_for_hist(df[\"Have you dated another ECE 2T2?\"], fig, horizontal=False)\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", x=1, y=-0.15, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_dated)}\")\n",
    "fig.update_xaxes(title=\"Have you dated another ECE 2T2?\")\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How has your understanding of relationships changed in the past 5 years?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How has your understanding of relationships changed in the past 5 years?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_long_term_understanding = df[\"How has your understanding of relationships changed in the past 5 years?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_word_map(df_long_term_understanding, file_path=\"./embed_images/Understanding_of_relationships.png\", is_frequency=True, relative_scaling=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plot_word_map(\"./embed_images/Understanding_of_relationships.png\")\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", yanchor=\"bottom\", x=1, y=0, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_long_term_understanding)}\")\n",
    "fig.show(renderer=\"png\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Which of the following have you done before university?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"Which of the following have you done before university?\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_before_university_multiple_choice = df[\"Which of the following have you done before university?\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_before_university = process_multiple_choice_survey(df_before_university_multiple_choice, column=\"before_uni\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_before_university, y=\"before_uni\", color=\"before_uni\")\n",
    "fig.update_yaxes(title=\"\", categoryorder=\"total ascending\", showticklabels=False)\n",
    "add_percent_labels_for_hist(df_before_university_multiple_choice, fig, horizontal=True)\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", x=1, y=-0.08, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_before_university_multiple_choice)}\")\n",
    "fig.update_layout(showlegend=True, legend=dict(xanchor=\"right\", yanchor=\"bottom\", x=1, y=0),\n",
    "                  margin=dict(t=5, b=5, l=0, r=5))\n",
    "# fig.update_traces(marker=dict(color=\"#97F0AA\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Substance Control\n",
    "\n",
    "What drugs are ECE2T2s on??????!!!!!!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"List the recreational controlled substances you have used during university\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_drugs_multiple_choice = df[\"List the recreational controlled substances you have used during university\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_substances = process_multiple_choice_survey(df_drugs_multiple_choice)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_substances[\"substance\"] = all_substances[\"substance\"].str[0:1].str.upper() + all_substances[\"substance\"].str[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_substances[\"substance\"].replace({\"Weed\": \"Cannabis\", \"Marijuana\" : \"Cannabis\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.pie(all_substances, names=\"substance\", color=\"substance\")\n",
    "fig.update_traces(texttemplate=\"%{label}<br>%{value} (%{percent})\",\n",
    "hovertemplate=less_cringe_hovertext_template[\"pie\"][\"hovertemplate\"])\n",
    "add_sample_size_annotation_for_figure(fig, df_drugs_multiple_choice, ref=\"paper\")\n",
    "fig.update_layout(margin=dict(b=0, t=0, l=2, r=2))\n",
    "fig.show(width=PIE_WIDTH, height=PIE_HEIGHT)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How Many Sexual Partners Have you Had?\n",
    "\n",
    "Sad Engineers HAHA (Just kidding guys, the number of sexual partners you've had throughout your life\n",
    "isn't something to brag about... It's just outright draining your soul...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"How many sexual partners have you had\"], check_unique=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_how_many_sexual_partners = df[\"How many sexual partners have you had\"].dropna().astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.histogram(df_how_many_sexual_partners)\n",
    "fig.update_xaxes(title=\"How many sexual partners have you had\")\n",
    "add_percent_labels_for_hist(df_how_many_sexual_partners, fig, horizontal=False)\n",
    "fig.add_annotation(xref=\"paper\", yref=\"paper\", xanchor=\"right\", x=1, y=-0.15, ax=0, ay=0, \n",
    "                   text=f\"Sample Size: {len(df_how_many_sexual_partners)}\")\n",
    "fig.update_layout(margin=dict(t=2,))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Share a story, happy or sad, from your time in ECE\n",
    "\n",
    "\n",
    "\"People encourage me when I'm tired and I think that helped\"\n",
    "\n",
    "\"One time I had an OS lab due, and my friend stayed up (and helped me stay up) until 4am to look over my entire implementation to help fix it. The bug? I forgot to run cmake. Blasting through this program has been one of the most inspiring, exciting, difficult, and worthwhile moments of my life. Each second took half as long as the last. They said it would go fast. It did.\"\n",
    "\n",
    "\"We were all in the computer lab till like 8am working on ECE241, it was awful but we were in it together. That basically sums up engineering for me.\"\n",
    "\n",
    "\"For ECE297, I remember seeing everyone in the lab at 11pm the day before our final milestone. People were trying to get their features completed and all the test cases passing. At the same time, classmates were in and out trying their best to point others in the right direction if they were stuck. It was a sweet memory to remember that everyone was on the same boat -- feeling stressed but determined to finish their work before a big deadline.<br>In ECE344, doing exam preparation on a google document. People were leaving answers and comments on this large master sheet of answers that was curated by students. We were all working collaboratively and helping each other out to get the right answer.\"\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_checker(df[\"1. Share a story, happy or sad, from your time in ECE\"], check_unique=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. What is something you regret over your time at UofT?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values ['Caring less about grades' 'caring too much about what people thought'\n",
      " 'Coming to UofT for my undergrad.' 'Failing a semester.'\n",
      " \"I didn't work on myself enough. I was too good at living on a random number generator.\"\n",
      " 'I regret not doing anything '\n",
      " \"I should've joined more clubs and been less serious about my studies.\"\n",
      " 'I wish I had learned to focus on my own personal issues and then blossom from there, rather than focus hard on external things I want to see change '\n",
      " 'I wish I had taken school a little less seriously and gone out to have fun more :)'\n",
      " 'I wish I had tried an intramural in first year and seen how much fun it would be during the school year. '\n",
      " 'Maybe not doing enough design teams and not spending enough time with friends :( Not participating enough hackathons and events from IEEE etc to grow my technical skills. Relying on school concepts and nothing really else'\n",
      " 'Not dedicating myself more to school. Although I have taken it seriously to an extent, I started very late (start of 4th year) in thinking about how I wanted to apply my schooling towards my career.'\n",
      " 'Not going to libraries to study in the first three years of uni.'\n",
      " \"Not hanging out with friends more!!! Studying is only one part of school and even though I love learning, I should've appreciated spent time with friends even more\"\n",
      " 'Not having more fun hanging out with friends outside. Making more friends in first-year. '\n",
      " 'Not making enough social connections with others outside engineering.'\n",
      " 'Not managing my time better :(\\n'\n",
      " 'Not much tbh. If anything maybe not talking to more people initially.'\n",
      " 'Not playing intramurals and going to more social events in first and second year'\n",
      " 'Not pursuing an international opportunity'\n",
      " 'Not pursuing more relationships '\n",
      " 'Not spending as much time hanging out with friends or enjoying campus'\n",
      " 'Not taking courses out of my zone of familiarity and planning far ahead. There are some courses from other faculties which I think I could have taken, if I had done more research in advance.'\n",
      " 'Picking ECE'\n",
      " 'Picking uoft over waterloo. Overall uoft\\'s community seems less driven to seek jobs and finding like-minded people here is harder. Also PEY is a scam for computer engineers because you \"intern\" for a year but it doesn\\'t count as real experience and it doesn\\'t look as good on your resume as 3-4 separate internships would have looked. I think its still a good program for other disciplines such as civil, and it still found me my first job, but still not good value overall.'\n",
      " nan]\n",
      "Number of NA Values: 29\n"
     ]
    }
   ],
   "source": [
    "data_checker(df[\"2. What is something you regret over your time at UofT?\"], check_unique=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Give a piece of advice to your fellow ECE 2T2 - 2023 grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values ['Although classes are hard, try to absorb as much as possible.'\n",
      " 'best of luck :)'\n",
      " \"Breathe. Ask for help. Don't give up because you are so close to being done. Make an effort to have long-lasting friendships. Work is temporary. Your job is temporary. Friendships in undergrad will help you through the tough times and help you appreciate the good ones too. Lastly, try your best to plan ahead and think ahead. Better to do things early than late. :)\"\n",
      " 'Build good habits and keep them as long as you can. Grades and marks affect your semester but mental illness and chronic anxiety last a lifetime. '\n",
      " 'Do not lose the person that you are in spite of the odds, allow it to nurture and become something better than you could have ever imagined'\n",
      " \"Don't be an asshole, it takes way to much effort\"\n",
      " \"Don't worry about where you start, we have so many years ahead of us in our careers. Your first job doesn't define you.\"\n",
      " 'Find joy in simple things:)'\n",
      " \"Get out there and do what you're passionate about - do what feels right. Don't let yourself or your interpretation of the world be reduced to numbers. We're taught in engineering to make all of our decisions based on data or explicit research. In each of our own ideal lives, the answer may not always be in the numbers. \"\n",
      " 'Have fun and spend time with friends! It goes by faster than you realize'\n",
      " 'Have fun!' 'Hope for the best for everyone 🙂'\n",
      " 'just keep swimming and don’t be afraid to try new things!'\n",
      " 'Live freely, live happily, live for you' 'Live long and prosper'\n",
      " 'Make as many new friends as you can. Memories are what you remember, not marks'\n",
      " \"Never hesitate to step outside your comfort zone when you have a goal that can't be reached within your comfort zone. Just go for it and you definitely won't regret it.\"\n",
      " 'Stay in touch!'\n",
      " \"Stay outta jail, don't kill yourself, and everything else will be fine.\"\n",
      " 'Take care! Work at whatever pace that is required from you, but not necessarily your own pace because it could be slower than a treadmill having a lot of friction  '\n",
      " \"When you find something that gives your life meaning, don't lose it\"\n",
      " 'When you hit your lowest point, you are open to the greatest change.'\n",
      " 'work hard.'\n",
      " \"Y'all are really cool people. Time to show the rest of the world that!\"\n",
      " 'You have your entire life to work, spend your prime years building lifelong connections '\n",
      " nan]\n",
      "Number of NA Values: 29\n"
     ]
    }
   ],
   "source": [
    "data_checker(df[\"3. Give a piece of advice to your fellow ECE 2T2 - 2023 grads\"], check_unique=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Top 5 Questions You Did Not Want to Answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_nulls = df.isna().sum()\n",
    "\n",
    "# df_top_5_values_no_answers = num_nulls.sort_values(ascending=False)\n",
    "df_top_5_values_with_no_answers = pd.DataFrame(num_nulls, columns=[\"Number of Non-Responses\"]).reset_index()\n",
    "df_top_5_values_with_no_answers.rename({\"index\" : \"Question\"}, axis=1, inplace=True)\n",
    "df_top_5_values_with_no_answers[\"Percentage of Non-Responses\"] = \\\n",
    "    (df_top_5_values_with_no_answers[\"Number of Non-Responses\"] / len(df) * 100).round(2).astype(str) + \"%\"\n",
    "\n",
    "df_top_5_values_with_no_answers.sort_values(\"Number of Non-Responses\", ascending=False, inplace=True)\n",
    "df_top_5_values_with_no_answers = df_top_5_values_with_no_answers.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_top_5_values_with_no_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write out num nulls to a file\n",
    "obj = zip(num_nulls.index.tolist(), num_nulls.tolist())\n",
    "nulls_list = list(obj)\n",
    "\n",
    "with open(\"nulls.txt\", mode=\"w\", encoding=\"utf-8\") as f:\n",
    "    f.write(\"\\n\".join([f\"{tup[0]}: {tup[1]}\" for tup in nulls_list]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = go.Figure(go.Table(\n",
    "    header=dict(values=[\"Question\", \"No Response\", \"% No Response\"], align=\"left\", height=130, font_size=30),\n",
    "    cells=dict(values=[df_top_5_values_with_no_answers.Question,\n",
    "                       df_top_5_values_with_no_answers[\"Number of Non-Responses\"],\n",
    "                       df_top_5_values_with_no_answers[\"Percentage of Non-Responses\"]],\n",
    "                       align=\"left\", height=80, font_size=80/4)\n",
    "    )\n",
    ")\n",
    "fig.update_layout(height=800, width=1200, margin_t=20, margin=dict(l=5, r=5))\n",
    "fig.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": [
     "hide_code"
    ]
   },
   "source": [
    "# Test/Archive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Folium Map\n",
    "# school_group = folium.FeatureGroup(name=\"schools\")\n",
    "# for i in range(df_high_school_merged.shape[0]):\n",
    "#     current_data = df_high_school_merged.iloc[i]\n",
    "#     current_name = current_data.name\n",
    "#     number_per_name = current_data[df_high_school_merged.columns[1]]\n",
    "#     html = f'''\n",
    "#     <div style=\"display: flex; justify-content: left; flex-direction: column;\">\n",
    "#         <div style=\"padding:0 10px 10px 0;color:grey\"><b>{current_name}</b></div>\n",
    "#         <div style=\"padding:0 10px 10px 0;font-size:40;font-weight:100;text-align:center\">{number_per_name}</div>\n",
    "#         <div style=\"text-align:center;padding:0 10px 0 0;\">{\"students studied high school here!\"}</div>\n",
    "#     </div>\n",
    "#     '''\n",
    "#     coordinates = current_data[\"Coordinates\"]\n",
    "#     coordinates = (coordinates[0] + np.random.rand() + 0.5,\n",
    "#                    coordinates[1] + np.random.rand() + 0.5)\n",
    "    \n",
    "#     iframe = folium.IFrame(html=html, width=170, height=170)\n",
    "#     icon = folium.Icon(icon=\"school\", color='lightblue', prefix='fa')\n",
    "#     popup = folium.Popup(iframe)\n",
    "#     school_group.add_child(folium.Marker(location=coordinates, popup=popup, icon=icon))\n",
    "#     # print()\n",
    "\n",
    "# _ = school_group.add_to(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from similarity.normalized_levenshtein import NormalizedLevenshtein\n",
    "\n",
    "# levenshtein = NormalizedLevenshtein()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from similarity.cosine import Cosine\n",
    "\n",
    "# cosine = Cosine(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cosine.get_profile(\"ab\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cosine.similarity_profiles(cosine.get_profile(\"a\"), cosine.get_profile(\"ab\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# a = pd.DataFrame([[True, False, False, False], [False, True, True, False], [True, True, False, False]], columns=[\"\"])\n",
    "# a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Correlation Matrix could be fun to unravel some insight\n",
    "# df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# IPyWidgets DOES NOT WORK! Has to communicate between Python and JavaScript as the interactive plots are being generated:\n",
    "#\n",
    "# def generate_profile(x):\n",
    "#     fig = px.histogram(df, x=x, color=x)\n",
    "#     _ = fig.update_layout(xaxis_title=\"\")\n",
    "#     _ = fig.update_traces(hovertemplate=\"%{x}=%{y}\")\n",
    "#     return fig\n",
    "\n",
    "# interact(generate_profile, x=[\"What is your gender?\", \"Are you a domestic or international student?\"])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# from dash import Dash, dcc, html, Input, Output\n",
    "# app = JupyterDash(__name__)\n",
    "# app.layout = html.Div([\n",
    "#     dcc.Dropdown(\n",
    "#                 [\"What is your gender?\", \"Are you a domestic or international student?\"],\n",
    "#                 \"What is your gender?\",\n",
    "#                 id='yaxis-column'\n",
    "#     ),\n",
    "#     dcc.Graph(id=\"output_graph\")\n",
    "# ])\n",
    "\n",
    "# @app.callback(\n",
    "#     Output(\"output_graph\", 'figure'),\n",
    "#     Input(\"yaxis-column\", 'value')\n",
    "# )\n",
    "# def get_graph(yaxis_column_name):\n",
    "#     fig = px.histogram(df, x=yaxis_column_name, color=yaxis_column_name)\n",
    "#     _ = fig.update_traces(hovertemplate=\"%{x}=%{y}\")\n",
    "#     return fig\n",
    "\n",
    "# app.run_server(mode=\"inline\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import plotly.graph_objects as go\n",
    "\n",
    "# df_new = pd.DataFrame(\n",
    "#     {\n",
    "#         \"Date\": [\"2020-01-27\", \"2020-02-27\", \"2020-03-27\"],\n",
    "#         \"A_item\": [2, 8, 0],\n",
    "#         \"B_item\": [1, 7, 10],\n",
    "#         \"C_item\": [9, 2, 9],\n",
    "#         \"Channel_type\": [\"Channel_1\", \"Channel_1\", \"Channel_2\"],\n",
    "#     }\n",
    "# )\n",
    "\n",
    "# fig = go.Figure(go.Table(header={\"values\": df_new.columns}, cells={\"values\": df_new.T.values}))\n",
    "# fig.update_layout(\n",
    "#     updatemenus=[\n",
    "#         {\n",
    "#             \"buttons\": [\n",
    "#                 {\n",
    "#                     \"label\": c,\n",
    "#                     \"method\": \"update\",\n",
    "#                     \"args\": [\n",
    "#                         {\n",
    "#                             \"cells\": {\n",
    "#                                 \"values\": df_new.T.values\n",
    "#                                 if c == \"All\"\n",
    "#                                 else df_new.loc[df_new[\"Channel_type\"].eq(c)].T.values\n",
    "#                             }\n",
    "#                         }\n",
    "#                     ],\n",
    "#                 }\n",
    "#                 for c in [\"All\"] + df_new[\"Channel_type\"].unique().tolist()\n",
    "#             ]\n",
    "#         }\n",
    "#     ]\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_new = pd.DataFrame(\n",
    "#     {\n",
    "#         \"Date\": [\"2020-01-27\", \"2020-02-27\", \"2020-03-27\", \"2020-01-27\"],\n",
    "#         \"A_item\": [2, 8, 0, 3],\n",
    "#         \"B_item\": [1, 7, 10, 4],\n",
    "#         \"C_item\": [9, 2, 9, 5],\n",
    "#         \"Channel_type\": [\"Channel_1\", \"Channel_1\", \"Channel_2\", \"Channel_2\"],\n",
    "#     }\n",
    "# )\n",
    "\n",
    "# fig = go.Figure(go.Table(header={\"values\": df_new.columns}, cells={\"values\": df_new.T.values}))\n",
    "# fig.update_layout(\n",
    "#     updatemenus=[\n",
    "#         {\n",
    "#             \"y\": 1 - (i / 5),\n",
    "#             \"buttons\": [\n",
    "#                 {\n",
    "#                     \"label\": c,\n",
    "#                     \"method\": \"restyle\",\n",
    "#                     \"args\": [\n",
    "#                         {\n",
    "#                             \"cells\": {\n",
    "#                                 \"values\": df_new.T.values\n",
    "#                                 if c == \"All\"\n",
    "#                                 else df_new.loc[df_new[menu].eq(c)].T.values\n",
    "#                             }\n",
    "#                         }\n",
    "#                     ],\n",
    "#                 }\n",
    "#                 for c in [\"All\"] + df_new[menu].unique().tolist()\n",
    "#             ],\n",
    "#         }\n",
    "#         for i, menu in enumerate([\"Channel_type\", \"Date\"])\n",
    "#     ]\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig = px.histogram(df, x=\"What is your gender?\", color=\"What is your gender?\")\n",
    "# fig.update_layout(xaxis_title=\"\")\n",
    "# fig.update_traces(hovertemplate=\"%{x}=%{y}\")\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fig = px.histogram(df, x=\"Are you a domestic or international student?\", color=\"Are you a domestic or international student?\")\n",
    "# fig.update_layout(xaxis_title=\"\")\n",
    "# fig.update_traces(hovertemplate=\"%{x}=%{y}\")\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# fig = px.pie(df, names=\"What is your sexual orientation?\", facet_col=\"What is your gender?\", \n",
    "#                    facet_col_spacing=0.04,\n",
    "#                    color=\"What is your sexual orientation?\")\n",
    "# fig.for_each_annotation(lambda a: a.update(text=a.text.split(\"=\")[-1]))\n",
    "# fig.update_traces(hovertemplate=\"<i>%{label}</i><br>Count: %{value}\")\n",
    "# fig.update_layout(showlegend=False)\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Globe Map\n",
    "# df_traffic = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2011_february_us_airport_traffic.csv')\n",
    "# df_traffic['text'] = df_traffic['airport'] + '' + df_traffic['city'] + ', ' + df_traffic['state'] + '' + 'Arrivals: ' + df_traffic['cnt'].astype(str)\n",
    "\n",
    "# fig = go.Figure(data=go.Scattergeo(\n",
    "#         lon = df_traffic['long'],\n",
    "#         lat = df_traffic['lat'],\n",
    "#         text = df_traffic['text'],\n",
    "#         mode = 'markers',\n",
    "#         marker_color = df_traffic['cnt'],\n",
    "#         ))\n",
    "\n",
    "\n",
    "# fig.update_geos(projection_type=\"orthographic\")\n",
    "# fig.update_layout(height=600,\n",
    "#                   title = 'Most trafficked US airports<br>(Hover for airport names)',\n",
    "#                   )\n",
    "# # fig.update_layout(\n",
    "\n",
    "# # )\n",
    "# fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chloropleth Maps\n",
    "\n",
    "# import requests\n",
    "# import json\n",
    "\n",
    "# first_year_locations_geojson = json.loads(\n",
    "#     requests.get(\n",
    "#         \"https://nominatim.openstreetmap.org/search?q=Chestnut Residence&format=geojson&polygon_geojson=1&limit=1\"\n",
    "#     ).text\n",
    "# )\n",
    "\n",
    "# first_year_locations_geojson[\"features\"][0][\"properties\"][\"location\"] = \"Chestnut\"\n",
    "# ## Create initial GeoJSON Query. Add new polygons to first_year_locations_geojson[\"features\"]\n",
    "# location_queries = [\n",
    "#     (\"Innis college toronto\", \"Innis\"),\n",
    "#     (\"University College Ontario\", \"University College\"),\n",
    "#     (\"Victoria College\", \"Victoria\"),\n",
    "#     (\"Elmsley Hall\", \"St. Michaels College\")\n",
    "# ]\n",
    "# for query, location in location_queries:\n",
    "#     response = requests.get(\n",
    "#         f\"https://nominatim.openstreetmap.org/search?q={query}&format=geojson&polygon_geojson=1&limit=1\"\n",
    "#     )\n",
    "#     try:\n",
    "#         response_json = json.loads(response.text)\n",
    "#         response_json_polygon = response_json[\"features\"].pop()\n",
    "#         response_json_polygon[\"properties\"][\"location\"] = location\n",
    "#         first_year_locations_geojson[\"features\"].append(response_json_polygon)\n",
    "#     except Exception as e:\n",
    "#         print(e)\n",
    "# df_first_year_locations\n",
    "# df = px.data.election()\n",
    "# geojson = px.data.election_geojson()\n",
    "# fig = px.choropleth_mapbox(df_first_year_locations, geojson=first_year_locations_geojson,\n",
    "#                            color=\"In first year, where did you live?\", \n",
    "#                            locations=\"index\", featureidkey=\"properties.location\",\n",
    "#                            center={\"lat\":  43.65992116412125, \"lon\": -79.39048393551872},\n",
    "#                            mapbox_style=\"carto-positron\", zoom=14)\n",
    "# fig.update_layout(margin={\"r\":0,\"t\":0,\"l\":0,\"b\":0})\n",
    "# fig.update_traces(colorscale=\n",
    "#                   [[0, \"#d2f2d4\"], [0.1, \"#7be382\"], [0.2, \"#26cc00\"],\n",
    "#                    [0.3, \"#22b600\"], [1, \"#009c1a\"]])\n",
    "# # fig.update_traces(tickmode=\"array\", tickval=df_first_year_locations[\"index\"], ticks=\"outside\")\n",
    "# fig.show(renderer=\"svg\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}