Content_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import load_embedding, parse_tup, cos_sim, cos_dist\n",
    "from kaleido.scopes.plotly import PlotlyScope\n",
    "from tqdm.auto import tqdm\n",
    "from datetime import datetime\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "scope = PlotlyScope()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from utils import CANDIDATE_SUBS\n",
    "TIME_FRAME = \"monthly\"\n",
    "left_candidates = [\"JoeBiden\",\"SandersForPresident\",\"BaemyKlobaechar\",\"ElizabethWarren\",\"Pete_Buttigieg\",\"YangForPresidentHQ\"]\n",
    "right_candidates = [\"The_Donald\"]\n",
    "cand_names = [\"joe\",\"biden\",\"bernie\",\"sanders\",\"sander\",\"amy\",\"klobuchar\",\"elizabeth\",\"warren\",\"pete\",\"buttigieg\",\"andrew\",\"yang\",\"submission\"]\n",
    "mapping = {}\n",
    "for a,b in zip(left_candidates+right_candidates,px.colors.qualitative.Plotly):\n",
    "    mapping[a] = b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init(\"/h/224/cameron/spark-3.0.0-preview2-bin-hadoop2.7\")\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import StructType, StructField, IntegerType,StringType\n",
    "\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "spark.sparkContext.getConf().getAll()\n",
    "cols = [\"created_utc\",\"author\",\"subreddit\",\"body\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments = spark.read.load(\"/comments_2019.parquet\").select(*cols)\n",
    "pulled_comments = spark.read.parquet(\"dataframes/2020_comments/*-2020_comments.parquet\").select(*cols)\n",
    "comments = comments.union(pulled_comments)\n",
    "comments = comments.where((comments['body'] != \"[removed]\")\n",
    "                          & (comments['body'] != \"[deleted]\")\n",
    "                          & (comments[\"author\"] != \"AutoModerator\")\n",
    "                          & (comments[\"author\"] != \"[deleted]\")\n",
    "                          & ~comments[\"body\"].contains(\"www.reddit.com\"))\n",
    "\n",
    "to_udf = udf(lambda ts: datetime.fromtimestamp(ts).strftime(\"%Y-%m-%d %H:00:00\"))\n",
    "comments = comments.withColumn(\"timestamp\", to_udf(comments[\"created_utc\"]))\n",
    "comments = comments.withColumn('date', date_format(col('timestamp'), 'd/M/yyyy'))\n",
    "comments = comments.withColumn('year',date_format(col('timestamp'), 'yyyy'))\n",
    "comments = comments.withColumn('month',date_format(col('timestamp'), 'M/yyyy'))\n",
    "\n",
    "comments = comments.drop(*[\"created_utc\",\"timestamp\"])\n",
    "\n",
    "comments.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# comments.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comment Flows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import sum as _sum\n",
    "from utils import coalese_csvs\n",
    "from os.path import join\n",
    "\n",
    "temp_comments = None\n",
    "try:\n",
    "    temp_comments = pd.read_csv(\"dataframes/2020_comments/2020_cand_counts.csv\",index_col=0)\n",
    "except:\n",
    "    temp_comments = comments.groupBy([\"subreddit\",\"date\",\"author\"]).count().toPandas()\n",
    "    temp_comments['date'] = pd.to_datetime(temp_comments['date'],format=\"%d/%m/%Y\")\n",
    "    temp_comments = temp_comments.sort_values(['date','subreddit'])\n",
    "    temp_comments.to_csv(\"dataframes/2020_comments/2020_cand_sub_counts.csv\")\n",
    "\n",
    "    sub_counts = temp_comments.groupby([\"subreddit\",\"date\"]).sum()\n",
    "    sub_counts.to_csv(\"dataframes/2020_comments/2020_cand_sub_counts.csv\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subreddit PMI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "co_occurence_matrix = temp_comments.groupby(['subreddit','author']).agg({'count':'sum'}).reset_index()\n",
    "co_occurence_matrix['count'] = 1\n",
    "co_occurence_matrix = pd.pivot_table(co_occurence_matrix, values='count', index=['author'], columns=['subreddit']).fillna(0)\n",
    "print(co_occurence_matrix.sum(axis = 0))\n",
    "labels = co_occurence_matrix.columns\n",
    "co_occurence_matrix = co_occurence_matrix.astype(int).to_numpy()\n",
    "co_occurence_matrix = co_occurence_matrix.T.dot(co_occurence_matrix)\n",
    "\n",
    "co_occurence_matrix\n",
    "\n",
    "def pmi(df, positive=True):\n",
    "    col_totals = df.sum(axis=0)\n",
    "    total = col_totals.sum()\n",
    "    row_totals = df.sum(axis=1)\n",
    "    expected = np.outer(row_totals, col_totals) / total\n",
    "    df = df / expected\n",
    "    # Silence distracting warnings about log(0):\n",
    "    with np.errstate(divide='ignore'):\n",
    "        df = np.log(df)\n",
    "    df[np.isinf(df)] = 0.0  # log(0) = 0\n",
    "    if positive:\n",
    "        df[df < 0] = 0.0\n",
    "    return df\n",
    "pmi_matrix = pmi(co_occurence_matrix,False)\n",
    "\n",
    "for i in range(len(pmi_matrix)):\n",
    "    pmi_matrix[i,i] = None\n",
    "\n",
    "data = go.Heatmap(z=pmi_matrix,y=labels,x=labels)\n",
    "\n",
    "layout = {\n",
    "    \"font\":{\"size\": 23},\n",
    "    \"height\": 600,\n",
    "    \"width\": 1000,    \n",
    "    \"template\": \"simple_white\",\n",
    "    \"xaxis\": {\"title\": \"Subreddit\"},\n",
    "    \"yaxis\": {\"title\": \"Subreddit\"},\n",
    "}\n",
    "fig = go.Figure(data=data,layout=layout)\n",
    "\n",
    "# with open(\"visualizations/comment_PMI.pdf\", \"wb\") as f:\n",
    "#     f.write(scope.transform(fig, format=\"pdf\"))\n",
    "# fig.write_html(\"visualizations/comment_PMI.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flow Chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2019-01-01</th>\n",
       "      <th>2019-02-01</th>\n",
       "      <th>2019-03-01</th>\n",
       "      <th>2019-04-01</th>\n",
       "      <th>2019-05-01</th>\n",
       "      <th>2019-06-01</th>\n",
       "      <th>2019-07-01</th>\n",
       "      <th>2019-08-01</th>\n",
       "      <th>2019-09-01</th>\n",
       "      <th>2019-10-01</th>\n",
       "      <th>2019-11-01</th>\n",
       "      <th>2019-12-01</th>\n",
       "      <th>2020-01-01</th>\n",
       "      <th>2020-02-01</th>\n",
       "      <th>2020-03-01</th>\n",
       "      <th>2020-04-01</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1: BaemyKlobaechar</td>\n",
       "      <td>2: BaemyKlobaechar</td>\n",
       "      <td>3: BaemyKlobaechar</td>\n",
       "      <td>4: BaemyKlobaechar</td>\n",
       "      <td>5: BaemyKlobaechar</td>\n",
       "      <td>6: BaemyKlobaechar</td>\n",
       "      <td>7: BaemyKlobaechar</td>\n",
       "      <td>8: BaemyKlobaechar</td>\n",
       "      <td>9: BaemyKlobaechar</td>\n",
       "      <td>10: BaemyKlobaechar</td>\n",
       "      <td>11: No_Comment</td>\n",
       "      <td>12: No_Comment</td>\n",
       "      <td>13: No_Comment</td>\n",
       "      <td>14: BaemyKlobaechar</td>\n",
       "      <td>15: BaemyKlobaechar</td>\n",
       "      <td>16: JoeBiden</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1: BaemyKlobaechar</td>\n",
       "      <td>2: BaemyKlobaechar</td>\n",
       "      <td>3: BaemyKlobaechar</td>\n",
       "      <td>4: No_Comment</td>\n",
       "      <td>5: No_Comment</td>\n",
       "      <td>6: No_Comment</td>\n",
       "      <td>7: No_Comment</td>\n",
       "      <td>8: No_Comment</td>\n",
       "      <td>9: No_Comment</td>\n",
       "      <td>10: No_Comment</td>\n",
       "      <td>11: No_Comment</td>\n",
       "      <td>12: BaemyKlobaechar</td>\n",
       "      <td>13: No_Comment</td>\n",
       "      <td>14: BaemyKlobaechar</td>\n",
       "      <td>15: JoeBiden</td>\n",
       "      <td>16: JoeBiden</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1: BaemyKlobaechar</td>\n",
       "      <td>2: BaemyKlobaechar</td>\n",
       "      <td>3: No_Comment</td>\n",
       "      <td>4: BaemyKlobaechar</td>\n",
       "      <td>5: No_Comment</td>\n",
       "      <td>6: BaemyKlobaechar</td>\n",
       "      <td>7: No_Comment</td>\n",
       "      <td>8: BaemyKlobaechar</td>\n",
       "      <td>9: BaemyKlobaechar</td>\n",
       "      <td>10: No_Comment</td>\n",
       "      <td>11: No_Comment</td>\n",
       "      <td>12: No_Comment</td>\n",
       "      <td>13: No_Comment</td>\n",
       "      <td>14: No_Comment</td>\n",
       "      <td>15: No_Comment</td>\n",
       "      <td>16: No_Comment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1: BaemyKlobaechar</td>\n",
       "      <td>2: BaemyKlobaechar</td>\n",
       "      <td>3: No_Comment</td>\n",
       "      <td>4: BaemyKlobaechar</td>\n",
       "      <td>5: Pete_Buttigieg</td>\n",
       "      <td>6: No_Comment</td>\n",
       "      <td>7: No_Comment</td>\n",
       "      <td>8: No_Comment</td>\n",
       "      <td>9: SandersForPresident</td>\n",
       "      <td>10: No_Comment</td>\n",
       "      <td>11: No_Comment</td>\n",
       "      <td>12: No_Comment</td>\n",
       "      <td>13: YangForPresidentHQ</td>\n",
       "      <td>14: Pete_Buttigieg</td>\n",
       "      <td>15: SandersForPresident</td>\n",
       "      <td>16: No_Comment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1: BaemyKlobaechar</td>\n",
       "      <td>2: BaemyKlobaechar</td>\n",
       "      <td>3: No_Comment</td>\n",
       "      <td>4: JoeBiden</td>\n",
       "      <td>5: No_Comment</td>\n",
       "      <td>6: No_Comment</td>\n",
       "      <td>7: No_Comment</td>\n",
       "      <td>8: No_Comment</td>\n",
       "      <td>9: JoeBiden</td>\n",
       "      <td>10: No_Comment</td>\n",
       "      <td>11: Pete_Buttigieg</td>\n",
       "      <td>12: Pete_Buttigieg</td>\n",
       "      <td>13: No_Comment</td>\n",
       "      <td>14: Pete_Buttigieg</td>\n",
       "      <td>15: Pete_Buttigieg</td>\n",
       "      <td>16: No_Comment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           2019-01-01          2019-02-01          2019-03-01  \\\n",
       "0  1: BaemyKlobaechar  2: BaemyKlobaechar  3: BaemyKlobaechar   \n",
       "1  1: BaemyKlobaechar  2: BaemyKlobaechar  3: BaemyKlobaechar   \n",
       "2  1: BaemyKlobaechar  2: BaemyKlobaechar       3: No_Comment   \n",
       "3  1: BaemyKlobaechar  2: BaemyKlobaechar       3: No_Comment   \n",
       "4  1: BaemyKlobaechar  2: BaemyKlobaechar       3: No_Comment   \n",
       "\n",
       "           2019-04-01          2019-05-01          2019-06-01  \\\n",
       "0  4: BaemyKlobaechar  5: BaemyKlobaechar  6: BaemyKlobaechar   \n",
       "1       4: No_Comment       5: No_Comment       6: No_Comment   \n",
       "2  4: BaemyKlobaechar       5: No_Comment  6: BaemyKlobaechar   \n",
       "3  4: BaemyKlobaechar   5: Pete_Buttigieg       6: No_Comment   \n",
       "4         4: JoeBiden       5: No_Comment       6: No_Comment   \n",
       "\n",
       "           2019-07-01          2019-08-01              2019-09-01  \\\n",
       "0  7: BaemyKlobaechar  8: BaemyKlobaechar      9: BaemyKlobaechar   \n",
       "1       7: No_Comment       8: No_Comment           9: No_Comment   \n",
       "2       7: No_Comment  8: BaemyKlobaechar      9: BaemyKlobaechar   \n",
       "3       7: No_Comment       8: No_Comment  9: SandersForPresident   \n",
       "4       7: No_Comment       8: No_Comment             9: JoeBiden   \n",
       "\n",
       "            2019-10-01          2019-11-01           2019-12-01  \\\n",
       "0  10: BaemyKlobaechar      11: No_Comment       12: No_Comment   \n",
       "1       10: No_Comment      11: No_Comment  12: BaemyKlobaechar   \n",
       "2       10: No_Comment      11: No_Comment       12: No_Comment   \n",
       "3       10: No_Comment      11: No_Comment       12: No_Comment   \n",
       "4       10: No_Comment  11: Pete_Buttigieg   12: Pete_Buttigieg   \n",
       "\n",
       "               2020-01-01           2020-02-01               2020-03-01  \\\n",
       "0          13: No_Comment  14: BaemyKlobaechar      15: BaemyKlobaechar   \n",
       "1          13: No_Comment  14: BaemyKlobaechar             15: JoeBiden   \n",
       "2          13: No_Comment       14: No_Comment           15: No_Comment   \n",
       "3  13: YangForPresidentHQ   14: Pete_Buttigieg  15: SandersForPresident   \n",
       "4          13: No_Comment   14: Pete_Buttigieg       15: Pete_Buttigieg   \n",
       "\n",
       "       2020-04-01  count  \n",
       "0    16: JoeBiden      1  \n",
       "1    16: JoeBiden      1  \n",
       "2  16: No_Comment      1  \n",
       "3  16: No_Comment      1  \n",
       "4  16: No_Comment      1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uc_pivot = pd.pivot_table(temp_comments.sort_values(['date']), values='count', index=['date','author'], columns=['subreddit']).fillna(0)\n",
    "uc_pivot[\"max_sub\"] = uc_pivot.idxmax(axis=1)\n",
    "uc_pivot[\"count\"] = uc_pivot.max(axis=1)\n",
    "uc_pivot = uc_pivot[[\"max_sub\",\"count\"]].reset_index()\n",
    "\n",
    "flow = pd.pivot_table(uc_pivot,index='author',columns='date',values='max_sub',aggfunc=np.sum).reset_index().fillna(\"No_Comment\")\n",
    "flow = flow.drop(\"author\",axis=1)\n",
    "# Count identical journeys (which is why you group by every column)\n",
    "flow = flow.groupby(list(flow.columns)).size().to_frame().rename({0: 'count'},axis=1).reset_index()\n",
    "for i, col in enumerate(flow.columns[:-1]):\n",
    "    flow[col] = flow[col].apply(lambda x : \"{}: {}\".format(i+1,x))\n",
    "flow.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>count</th>\n",
       "      <th>sourceID</th>\n",
       "      <th>targetID</th>\n",
       "      <th>color</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>1: SandersForPresident</td>\n",
       "      <td>2: SandersForPresident</td>\n",
       "      <td>7.643483</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>rgba(44, 160, 44,0.4)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>1: SandersForPresident</td>\n",
       "      <td>2: No_Comment</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>rgba(192,192,192,0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>1: No_Comment</td>\n",
       "      <td>2: SandersForPresident</td>\n",
       "      <td>9.152605</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>rgba(192,192,192,0.2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>1: No_Comment</td>\n",
       "      <td>2: BaemyKlobaechar</td>\n",
       "      <td>4.276666</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>rgba(192,192,192,0.2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>1: No_Comment</td>\n",
       "      <td>2: ElizabethWarren</td>\n",
       "      <td>4.418841</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>rgba(192,192,192,0.2)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     source                  target     count  sourceID  \\\n",
       "289  1: SandersForPresident  2: SandersForPresident  7.643483         1   \n",
       "288  1: SandersForPresident           2: No_Comment  0.100000         1   \n",
       "283           1: No_Comment  2: SandersForPresident  9.152605         6   \n",
       "278           1: No_Comment      2: BaemyKlobaechar  4.276666         6   \n",
       "279           1: No_Comment      2: ElizabethWarren  4.418841         6   \n",
       "\n",
       "     targetID                  color  \n",
       "289         8  rgba(44, 160, 44,0.4)  \n",
       "288        13    rgba(192,192,192,0)  \n",
       "283         8  rgba(192,192,192,0.2)  \n",
       "278         9  rgba(192,192,192,0.2)  \n",
       "279        10  rgba(192,192,192,0.2)  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform df into a source-target pair\n",
    "f_cols = list(flow.columns)\n",
    "\n",
    "label_list = [\"{}: {}\".format(i+1,cand) for i in range(len(f_cols)-1) for cand in left_candidates+[\"No_Comment\"]]\n",
    "color_list = [mapping.get(c,\"rgba(192,192,192,0)\") for _ in range(len(f_cols)) for c in left_candidates+[\"No_Comment\"]]\n",
    "source_target_df =  pd.DataFrame(columns=['source','target','count'])\n",
    "for i in range(len(f_cols)-2):\n",
    "    temp_df = flow[[f_cols[i],f_cols[i+1],'count']]\n",
    "    temp_df.columns = ['source','target','count']\n",
    "    source_target_df = pd.concat([source_target_df,temp_df])\n",
    "    source_target_df = source_target_df.groupby(['source','target']).agg({'count':'sum'}).reset_index()\n",
    "\n",
    "# add index for source-target pair\n",
    "source_target_df['sourceID'] = source_target_df['source'].apply(lambda x: label_list.index(x))\n",
    "source_target_df['targetID'] = source_target_df['target'].apply(lambda x: label_list.index(x))\n",
    "source_target_df = source_target_df[source_target_df[\"count\"] > 60]\n",
    "c = lambda x ,t: 0.1 if \"No_Comment\" in t else np.log(x)\n",
    "source_target_df[\"count\"] = source_target_df[[\"count\",\"target\"]].apply(lambda x : c(*x),axis=1) \n",
    "source_target_df = source_target_df.sort_values([\"sourceID\",\"targetID\"])\n",
    "def color(row):\n",
    "    s,t  = row[\"source\"], row[\"target\"]\n",
    "    # Get the name after the colon\n",
    "    s,t = s.split(\":\",1)[1].strip(),t.split(\":\",1)[1].strip()\n",
    "    if t == \"No_Comment\":\n",
    "        return \"rgba(192,192,192,0)\"\n",
    "    if s == \"No_Comment\":\n",
    "        return \"rgba(192,192,192,0.2)\"\n",
    "    return \"rgba(44, 160, 44,0.4)\" if s==t else \"rgba(0,0,160,0.4)\"\n",
    "source_target_df[\"color\"] = source_target_df[[\"source\",\"target\"]].apply(color,axis=1)\n",
    "source_target_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_map = {str((i+1)): (i+0.01)/16 for i in range(17)}\n",
    "y_map = {\n",
    "    \"No_Comment\": 0.01,\n",
    "    \"SandersForPresident\": 0.23,\n",
    "    \"YangForPresidentHQ\": 0.41,\n",
    "    \"Pete_Buttigieg\": 0.58,\n",
    "    \"BaemyKlobaechar\": 1,\n",
    "    \"ElizabethWarren\": 0.73,\n",
    "    \"JoeBiden\": 0.9,\n",
    "}\n",
    "to_position = set(list(source_target_df[\"source\"].unique())+list(source_target_df[\"target\"].unique()))\n",
    "x_tup = [(x_map[x.split(\": \",1)[0]],x) for x in label_list]\n",
    "y_tup = [(y_map[x.split(\": \",1)[1]],x) for x in label_list]\n",
    "x,y = [x for x,y in x_tup if y in to_position],[x for x,y in y_tup if y in to_position]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "link": {
          "color": [
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(0,0,160,0.4)",
           "rgba(44, 160, 44,0.4)",
           "rgba(192,192,192,0)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0.2)",
           "rgba(192,192,192,0)"
          ],
          "hovertemplate": "10^%{value} users switched subreddits<extra></extra>",
          "source": [
           1,
           1,
           6,
           6,
           6,
           6,
           6,
           8,
           8,
           8,
           9,
           10,
           12,
           12,
           13,
           13,
           13,
           13,
           13,
           15,
           15,
           15,
           15,
           18,
           18,
           19,
           19,
           20,
           20,
           20,
           20,
           20,
           20,
           21,
           22,
           22,
           24,
           24,
           25,
           25,
           25,
           26,
           26,
           27,
           27,
           27,
           27,
           27,
           27,
           28,
           29,
           29,
           29,
           31,
           31,
           32,
           32,
           33,
           33,
           34,
           34,
           34,
           34,
           34,
           34,
           35,
           36,
           36,
           36,
           36,
           38,
           38,
           39,
           39,
           40,
           40,
           40,
           41,
           41,
           41,
           41,
           41,
           41,
           42,
           43,
           43,
           43,
           45,
           45,
           45,
           46,
           46,
           47,
           47,
           47,
           48,
           48,
           48,
           48,
           48,
           48,
           49,
           49,
           50,
           50,
           50,
           50,
           52,
           52,
           53,
           53,
           54,
           54,
           54,
           55,
           55,
           55,
           55,
           55,
           55,
           56,
           56,
           57,
           57,
           57,
           57,
           59,
           59,
           60,
           60,
           61,
           61,
           61,
           62,
           62,
           62,
           62,
           62,
           62,
           63,
           63,
           64,
           64,
           64,
           64,
           66,
           66,
           66,
           67,
           67,
           68,
           68,
           68,
           69,
           69,
           69,
           69,
           69,
           69,
           70,
           70,
           71,
           71,
           71,
           71,
           73,
           73,
           73,
           74,
           74,
           74,
           75,
           75,
           75,
           76,
           76,
           76,
           76,
           76,
           76,
           77,
           77,
           78,
           78,
           78,
           78,
           80,
           80,
           80,
           81,
           81,
           81,
           81,
           82,
           82,
           82,
           82,
           83,
           83,
           83,
           83,
           83,
           83,
           84,
           84,
           85,
           85,
           85,
           85,
           85,
           87,
           87,
           87,
           87,
           87,
           88,
           88,
           88,
           89,
           89,
           89,
           89,
           90,
           90,
           90,
           90,
           90,
           90,
           90,
           91,
           91,
           92,
           92,
           92,
           92,
           92,
           92,
           93,
           94,
           94,
           94,
           94,
           94,
           95,
           95,
           95,
           95,
           95,
           95,
           96,
           96,
           96,
           96,
           96,
           96,
           97,
           97,
           97,
           97,
           97,
           97,
           98,
           98,
           98,
           99,
           99,
           99,
           99,
           99,
           100,
           101,
           101,
           101,
           101,
           102,
           102,
           102,
           102,
           103,
           103,
           103,
           104,
           104,
           104,
           104,
           104,
           104
          ],
          "target": [
           8,
           13,
           8,
           9,
           10,
           12,
           13,
           15,
           19,
           20,
           20,
           20,
           19,
           20,
           15,
           17,
           18,
           19,
           20,
           22,
           25,
           26,
           27,
           25,
           27,
           26,
           27,
           21,
           22,
           24,
           25,
           26,
           27,
           34,
           29,
           34,
           31,
           34,
           29,
           32,
           34,
           33,
           34,
           28,
           29,
           31,
           32,
           33,
           34,
           41,
           36,
           40,
           41,
           38,
           41,
           39,
           41,
           40,
           41,
           35,
           36,
           38,
           39,
           40,
           41,
           48,
           43,
           45,
           47,
           48,
           45,
           48,
           46,
           48,
           43,
           47,
           48,
           42,
           43,
           45,
           46,
           47,
           48,
           49,
           50,
           54,
           55,
           50,
           52,
           55,
           53,
           55,
           50,
           54,
           55,
           49,
           50,
           52,
           53,
           54,
           55,
           56,
           62,
           57,
           59,
           61,
           62,
           59,
           62,
           60,
           62,
           57,
           61,
           62,
           56,
           57,
           59,
           60,
           61,
           62,
           63,
           69,
           64,
           66,
           68,
           69,
           66,
           69,
           67,
           69,
           64,
           68,
           69,
           63,
           64,
           66,
           67,
           68,
           69,
           70,
           76,
           71,
           73,
           75,
           76,
           71,
           73,
           76,
           74,
           76,
           71,
           75,
           76,
           70,
           71,
           73,
           74,
           75,
           76,
           77,
           83,
           78,
           81,
           82,
           83,
           78,
           80,
           83,
           78,
           81,
           83,
           78,
           82,
           83,
           77,
           78,
           80,
           81,
           82,
           83,
           84,
           90,
           85,
           87,
           89,
           90,
           85,
           87,
           90,
           85,
           87,
           88,
           90,
           85,
           87,
           89,
           90,
           84,
           85,
           87,
           88,
           89,
           90,
           91,
           97,
           92,
           94,
           95,
           96,
           97,
           92,
           94,
           95,
           96,
           97,
           92,
           95,
           97,
           92,
           95,
           96,
           97,
           91,
           92,
           93,
           94,
           95,
           96,
           97,
           98,
           104,
           98,
           99,
           101,
           102,
           103,
           104,
           104,
           98,
           99,
           101,
           102,
           104,
           98,
           99,
           101,
           102,
           103,
           104,
           98,
           99,
           101,
           102,
           103,
           104,
           98,
           99,
           101,
           102,
           103,
           104,
           105,
           106,
           111,
           105,
           106,
           108,
           110,
           111,
           111,
           105,
           106,
           108,
           111,
           105,
           106,
           109,
           111,
           106,
           110,
           111,
           105,
           106,
           108,
           109,
           110,
           111
          ],
          "value": [
           7.643482907077201,
           0.1,
           9.152605321324945,
           4.276666119016055,
           4.418840607796598,
           6.244166900663736,
           0.1,
           8.237214703349489,
           4.584967478670572,
           0.1,
           0.1,
           0.1,
           5.762051382780177,
           0.1,
           8.917310693197807,
           4.1588830833596715,
           7.395721608602045,
           8.12355783506165,
           0.1,
           8.197263371414335,
           5.017279836814924,
           4.406719247264253,
           0.1,
           6.959398512133975,
           0.1,
           7.318539548567902,
           0.1,
           4.727387818712341,
           8.682368589375221,
           5.602118820879701,
           7.999342952713282,
           7.666690200080086,
           0.1,
           0.1,
           7.946971357693591,
           0.1,
           4.890349128221754,
           0.1,
           4.465908118654584,
           7.254177846456518,
           0.1,
           7.350516171833998,
           0.1,
           4.762173934797756,
           8.340933226000878,
           6.135564891081739,
           6.872128101338986,
           7.236339342754344,
           0.1,
           0.1,
           7.874358824729881,
           4.276666119016055,
           0.1,
           5.60947179518496,
           0.1,
           7.018401799069201,
           0.1,
           7.456454555176209,
           0.1,
           4.882801922586371,
           9.57171425857138,
           6.714170529909472,
           7.282073658093465,
           8.153349757998892,
           0.1,
           0.1,
           8.173575486634153,
           4.31748811353631,
           5.402677381872279,
           0.1,
           6.040254711277414,
           0.1,
           7.122059881629142,
           0.1,
           4.1588830833596715,
           7.925157512224703,
           0.1,
           4.204692619390966,
           8.905444318789714,
           6.478509642208569,
           7.204892510204673,
           8.477412321404392,
           0.1,
           4.127134385045092,
           8.302513718514158,
           5.407171771460119,
           0.1,
           4.219507705176107,
           6.129050210060545,
           0.1,
           7.072421900537371,
           0.1,
           5.19295685089021,
           8.267192185932146,
           0.1,
           5.0238805208462765,
           9.56640512978922,
           6.593044534142437,
           6.890609120147166,
           9.081256218564645,
           0.1,
           4.605170185988092,
           0.1,
           8.481151420068972,
           4.30406509320417,
           5.793013608384144,
           0.1,
           6.342121418721152,
           0.1,
           6.902742737158593,
           0.1,
           5.631211781821365,
           8.703506769479729,
           0.1,
           4.77912349311153,
           9.228965836099349,
           6.926577033222725,
           6.728628613084702,
           8.907747686788648,
           0.1,
           4.499809670330265,
           0.1,
           8.403352374992478,
           4.727387818712341,
           5.605802066295998,
           0.1,
           6.628041376179533,
           0.1,
           6.90875477931522,
           0.1,
           5.4510384535657,
           8.797246236703527,
           0.1,
           4.189654742026425,
           9.155461710466193,
           6.99117688712121,
           6.954638864880987,
           8.854236693405742,
           0.1,
           4.394449154672439,
           0.1,
           8.39931015075952,
           4.553876891600541,
           5.327876168789581,
           0.1,
           4.330733340286331,
           6.693323668269949,
           0.1,
           7.1372784372603855,
           0.1,
           5.262690188904886,
           8.782015969721936,
           0.1,
           4.6913478822291435,
           8.913011922472597,
           6.790097235513905,
           7.268920128193722,
           8.760139370026627,
           0.1,
           4.634728988229636,
           0.1,
           8.469682208745185,
           4.174387269895637,
           5.442417710521793,
           0.1,
           4.762173934797756,
           6.400257445308821,
           0.1,
           4.1588830833596715,
           7.301147805856032,
           0.1,
           5.356586274672012,
           8.874447967219979,
           0.1,
           5.2832037287379885,
           9.508591395777199,
           6.437751649736401,
           7.330405211844402,
           9.146654820290049,
           0.1,
           4.976733742420574,
           0.1,
           8.845201135339584,
           5.480638923341991,
           5.583496308781699,
           0.1,
           4.127134385045092,
           6.448889394146858,
           0.1,
           4.672828834461906,
           4.634728988229636,
           7.266827347520591,
           0.1,
           6.175867270105761,
           4.248495242049359,
           8.969160133175821,
           0.1,
           5.501258210544727,
           9.963547308963792,
           7.779466967458324,
           7.00397413672268,
           8.806723554725636,
           0.1,
           5.14166355650266,
           0.1,
           9.460086905084927,
           5.123963979403259,
           5.846438775057725,
           6.289715570908998,
           0.1,
           6.163314804034641,
           7.002155954403621,
           5.303304908059076,
           4.574710978503383,
           0.1,
           4.499809670330265,
           7.54274354536855,
           0.1,
           6.651571873589727,
           5.075173815233827,
           8.890548245606167,
           0.1,
           5.902633333401366,
           10.719648508991218,
           4.6443908991413725,
           7.664815785285735,
           8.675734219544788,
           9.189831495344642,
           0.1,
           5.726847747587197,
           0.1,
           5.971261839790462,
           9.989711119284745,
           6.284134161070802,
           6.18826412308259,
           5.6937321388027,
           0.1,
           0.1,
           4.248495242049359,
           5.736572297479192,
           7.2196420401307355,
           4.2626798770413155,
           0.1,
           6.385194398997726,
           6.452048954437226,
           5.501258210544727,
           7.83002808253384,
           4.3694478524670215,
           0.1,
           4.969813299576001,
           7.2640301428995295,
           4.9344739331306915,
           5.062595033026967,
           8.28045768658256,
           0.1,
           7.983098940710892,
           10.749870708994932,
           7.906178840394815,
           7.828038032125829,
           7.9617188159813645,
           0.1,
           7.02108396428914,
           5.420534999272286,
           0.1,
           5.902633333401366,
           9.372374183823416,
           4.394449154672439,
           5.236441962829949,
           0.1,
           0.1,
           4.709530201312334,
           5.438079308923196,
           6.329720905522696,
           0.1,
           5.3230099791384085,
           5.497168225293202,
           6.366470447731438,
           0.1,
           5.564520407322694,
           7.295735072749282,
           0.1,
           7.2682230211595655,
           9.77996306625063,
           5.5254529391317835,
           5.365976015021851,
           7.046647277848756,
           0.1
          ]
         },
         "node": {
          "color": [
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)",
           "#636EFA",
           "#EF553B",
           "#00CC96",
           "#AB63FA",
           "#FFA15A",
           "#19D3F3",
           "rgba(192,192,192,0)"
          ],
          "hovertemplate": "10^%{value} users predominantly commented on %{label}<extra></extra>",
          "label": [
           " ",
           "Sanders",
           " ",
           " ",
           " ",
           " ",
           "No Comment",
           " ",
           " ",
           "Klobuchar",
           "Warren",
           " ",
           "Yang",
           " ",
           " ",
           " ",
           " ",
           " ",
           "Buttigieg",
           " ",
           " ",
           "Biden",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " ",
           " "
          ],
          "line": {
           "color": "rgba(192,192,192,0)"
          },
          "pad": 10,
          "thickness": 10,
          "x": [
           0.000625,
           0.000625,
           0.063125,
           0.063125,
           0.063125,
           0.063125,
           0.063125,
           0.125625,
           0.125625,
           0.125625,
           0.125625,
           0.125625,
           0.188125,
           0.188125,
           0.188125,
           0.188125,
           0.188125,
           0.188125,
           0.250625,
           0.250625,
           0.250625,
           0.250625,
           0.250625,
           0.250625,
           0.313125,
           0.313125,
           0.313125,
           0.313125,
           0.313125,
           0.313125,
           0.375625,
           0.375625,
           0.375625,
           0.375625,
           0.375625,
           0.375625,
           0.438125,
           0.438125,
           0.438125,
           0.438125,
           0.438125,
           0.438125,
           0.500625,
           0.500625,
           0.500625,
           0.500625,
           0.500625,
           0.500625,
           0.563125,
           0.563125,
           0.563125,
           0.563125,
           0.563125,
           0.563125,
           0.625625,
           0.625625,
           0.625625,
           0.625625,
           0.625625,
           0.625625,
           0.688125,
           0.688125,
           0.688125,
           0.688125,
           0.688125,
           0.688125,
           0.750625,
           0.750625,
           0.750625,
           0.750625,
           0.750625,
           0.750625,
           0.813125,
           0.813125,
           0.813125,
           0.813125,
           0.813125,
           0.813125,
           0.813125,
           0.875625,
           0.875625,
           0.875625,
           0.875625,
           0.875625,
           0.875625,
           0.875625,
           0.938125,
           0.938125,
           0.938125,
           0.938125,
           0.938125,
           0.938125
          ],
          "y": [
           0.23,
           0.01,
           0.23,
           1,
           0.73,
           0.41,
           0.01,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           1,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           1,
           0.73,
           0.58,
           0.41,
           0.01,
           0.9,
           0.23,
           0.73,
           0.58,
           0.41,
           0.01
          ]
         },
         "type": "sankey"
        }
       ],
       "layout": {
        "annotations": [
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Jan 2019",
          "x": 0.0003125,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Feb 2019",
          "x": 0.06412499999999999,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Mar 2019",
          "x": 0.12793749999999998,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Apr 2019",
          "x": 0.19175,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "May 2019",
          "x": 0.25556249999999997,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Jun 2019",
          "x": 0.319375,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Jul 2019",
          "x": 0.3831875,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Aug 2019",
          "x": 0.447,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Sep 2019",
          "x": 0.5108125,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Oct 2019",
          "x": 0.574625,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Nov 2019",
          "x": 0.6384375000000001,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Dec 2019",
          "x": 0.70225,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Jan 2020",
          "x": 0.7660625000000001,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Feb 2020",
          "x": 0.829875,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Mar 2020",
          "x": 0.8936875000000001,
          "y": -0.1
         },
         {
          "bgcolor": "rgba(0,0,0,0)",
          "font": {
           "size": 18
          },
          "showarrow": false,
          "text": "Apr 2020",
          "x": 0.9575,
          "y": -0.1
         }
        ],
        "font": {
         "size": 23
        },
        "height": 500,
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 2000
       }
      },
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"3bb33409-1443-4f2f-a7e3-bc3639dc16ad\" class=\"plotly-graph-div\" style=\"height:500px; width:2000px;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"3bb33409-1443-4f2f-a7e3-bc3639dc16ad\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '3bb33409-1443-4f2f-a7e3-bc3639dc16ad',\n",
       "                        [{\"link\": {\"color\": [\"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(0,0,160,0.4)\", \"rgba(44, 160, 44,0.4)\", \"rgba(192,192,192,0)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0.2)\", \"rgba(192,192,192,0)\"], \"hovertemplate\": \"10^%{value} users switched subreddits<extra></extra>\", \"source\": [1, 1, 6, 6, 6, 6, 6, 8, 8, 8, 9, 10, 12, 12, 13, 13, 13, 13, 13, 15, 15, 15, 15, 18, 18, 19, 19, 20, 20, 20, 20, 20, 20, 21, 22, 22, 24, 24, 25, 25, 25, 26, 26, 27, 27, 27, 27, 27, 27, 28, 29, 29, 29, 31, 31, 32, 32, 33, 33, 34, 34, 34, 34, 34, 34, 35, 36, 36, 36, 36, 38, 38, 39, 39, 40, 40, 40, 41, 41, 41, 41, 41, 41, 42, 43, 43, 43, 45, 45, 45, 46, 46, 47, 47, 47, 48, 48, 48, 48, 48, 48, 49, 49, 50, 50, 50, 50, 52, 52, 53, 53, 54, 54, 54, 55, 55, 55, 55, 55, 55, 56, 56, 57, 57, 57, 57, 59, 59, 60, 60, 61, 61, 61, 62, 62, 62, 62, 62, 62, 63, 63, 64, 64, 64, 64, 66, 66, 66, 67, 67, 68, 68, 68, 69, 69, 69, 69, 69, 69, 70, 70, 71, 71, 71, 71, 73, 73, 73, 74, 74, 74, 75, 75, 75, 76, 76, 76, 76, 76, 76, 77, 77, 78, 78, 78, 78, 80, 80, 80, 81, 81, 81, 81, 82, 82, 82, 82, 83, 83, 83, 83, 83, 83, 84, 84, 85, 85, 85, 85, 85, 87, 87, 87, 87, 87, 88, 88, 88, 89, 89, 89, 89, 90, 90, 90, 90, 90, 90, 90, 91, 91, 92, 92, 92, 92, 92, 92, 93, 94, 94, 94, 94, 94, 95, 95, 95, 95, 95, 95, 96, 96, 96, 96, 96, 96, 97, 97, 97, 97, 97, 97, 98, 98, 98, 99, 99, 99, 99, 99, 100, 101, 101, 101, 101, 102, 102, 102, 102, 103, 103, 103, 104, 104, 104, 104, 104, 104], \"target\": [8, 13, 8, 9, 10, 12, 13, 15, 19, 20, 20, 20, 19, 20, 15, 17, 18, 19, 20, 22, 25, 26, 27, 25, 27, 26, 27, 21, 22, 24, 25, 26, 27, 34, 29, 34, 31, 34, 29, 32, 34, 33, 34, 28, 29, 31, 32, 33, 34, 41, 36, 40, 41, 38, 41, 39, 41, 40, 41, 35, 36, 38, 39, 40, 41, 48, 43, 45, 47, 48, 45, 48, 46, 48, 43, 47, 48, 42, 43, 45, 46, 47, 48, 49, 50, 54, 55, 50, 52, 55, 53, 55, 50, 54, 55, 49, 50, 52, 53, 54, 55, 56, 62, 57, 59, 61, 62, 59, 62, 60, 62, 57, 61, 62, 56, 57, 59, 60, 61, 62, 63, 69, 64, 66, 68, 69, 66, 69, 67, 69, 64, 68, 69, 63, 64, 66, 67, 68, 69, 70, 76, 71, 73, 75, 76, 71, 73, 76, 74, 76, 71, 75, 76, 70, 71, 73, 74, 75, 76, 77, 83, 78, 81, 82, 83, 78, 80, 83, 78, 81, 83, 78, 82, 83, 77, 78, 80, 81, 82, 83, 84, 90, 85, 87, 89, 90, 85, 87, 90, 85, 87, 88, 90, 85, 87, 89, 90, 84, 85, 87, 88, 89, 90, 91, 97, 92, 94, 95, 96, 97, 92, 94, 95, 96, 97, 92, 95, 97, 92, 95, 96, 97, 91, 92, 93, 94, 95, 96, 97, 98, 104, 98, 99, 101, 102, 103, 104, 104, 98, 99, 101, 102, 104, 98, 99, 101, 102, 103, 104, 98, 99, 101, 102, 103, 104, 98, 99, 101, 102, 103, 104, 105, 106, 111, 105, 106, 108, 110, 111, 111, 105, 106, 108, 111, 105, 106, 109, 111, 106, 110, 111, 105, 106, 108, 109, 110, 111], \"value\": [7.643482907077201, 0.1, 9.152605321324945, 4.276666119016055, 4.418840607796598, 6.244166900663736, 0.1, 8.237214703349489, 4.584967478670572, 0.1, 0.1, 0.1, 5.762051382780177, 0.1, 8.917310693197807, 4.1588830833596715, 7.395721608602045, 8.12355783506165, 0.1, 8.197263371414335, 5.017279836814924, 4.406719247264253, 0.1, 6.959398512133975, 0.1, 7.318539548567902, 0.1, 4.727387818712341, 8.682368589375221, 5.602118820879701, 7.999342952713282, 7.666690200080086, 0.1, 0.1, 7.946971357693591, 0.1, 4.890349128221754, 0.1, 4.465908118654584, 7.254177846456518, 0.1, 7.350516171833998, 0.1, 4.762173934797756, 8.340933226000878, 6.135564891081739, 6.872128101338986, 7.236339342754344, 0.1, 0.1, 7.874358824729881, 4.276666119016055, 0.1, 5.60947179518496, 0.1, 7.018401799069201, 0.1, 7.456454555176209, 0.1, 4.882801922586371, 9.57171425857138, 6.714170529909472, 7.282073658093465, 8.153349757998892, 0.1, 0.1, 8.173575486634153, 4.31748811353631, 5.402677381872279, 0.1, 6.040254711277414, 0.1, 7.122059881629142, 0.1, 4.1588830833596715, 7.925157512224703, 0.1, 4.204692619390966, 8.905444318789714, 6.478509642208569, 7.204892510204673, 8.477412321404392, 0.1, 4.127134385045092, 8.302513718514158, 5.407171771460119, 0.1, 4.219507705176107, 6.129050210060545, 0.1, 7.072421900537371, 0.1, 5.19295685089021, 8.267192185932146, 0.1, 5.0238805208462765, 9.56640512978922, 6.593044534142437, 6.890609120147166, 9.081256218564645, 0.1, 4.605170185988092, 0.1, 8.481151420068972, 4.30406509320417, 5.793013608384144, 0.1, 6.342121418721152, 0.1, 6.902742737158593, 0.1, 5.631211781821365, 8.703506769479729, 0.1, 4.77912349311153, 9.228965836099349, 6.926577033222725, 6.728628613084702, 8.907747686788648, 0.1, 4.499809670330265, 0.1, 8.403352374992478, 4.727387818712341, 5.605802066295998, 0.1, 6.628041376179533, 0.1, 6.90875477931522, 0.1, 5.4510384535657, 8.797246236703527, 0.1, 4.189654742026425, 9.155461710466193, 6.99117688712121, 6.954638864880987, 8.854236693405742, 0.1, 4.394449154672439, 0.1, 8.39931015075952, 4.553876891600541, 5.327876168789581, 0.1, 4.330733340286331, 6.693323668269949, 0.1, 7.1372784372603855, 0.1, 5.262690188904886, 8.782015969721936, 0.1, 4.6913478822291435, 8.913011922472597, 6.790097235513905, 7.268920128193722, 8.760139370026627, 0.1, 4.634728988229636, 0.1, 8.469682208745185, 4.174387269895637, 5.442417710521793, 0.1, 4.762173934797756, 6.400257445308821, 0.1, 4.1588830833596715, 7.301147805856032, 0.1, 5.356586274672012, 8.874447967219979, 0.1, 5.2832037287379885, 9.508591395777199, 6.437751649736401, 7.330405211844402, 9.146654820290049, 0.1, 4.976733742420574, 0.1, 8.845201135339584, 5.480638923341991, 5.583496308781699, 0.1, 4.127134385045092, 6.448889394146858, 0.1, 4.672828834461906, 4.634728988229636, 7.266827347520591, 0.1, 6.175867270105761, 4.248495242049359, 8.969160133175821, 0.1, 5.501258210544727, 9.963547308963792, 7.779466967458324, 7.00397413672268, 8.806723554725636, 0.1, 5.14166355650266, 0.1, 9.460086905084927, 5.123963979403259, 5.846438775057725, 6.289715570908998, 0.1, 6.163314804034641, 7.002155954403621, 5.303304908059076, 4.574710978503383, 0.1, 4.499809670330265, 7.54274354536855, 0.1, 6.651571873589727, 5.075173815233827, 8.890548245606167, 0.1, 5.902633333401366, 10.719648508991218, 4.6443908991413725, 7.664815785285735, 8.675734219544788, 9.189831495344642, 0.1, 5.726847747587197, 0.1, 5.971261839790462, 9.989711119284745, 6.284134161070802, 6.18826412308259, 5.6937321388027, 0.1, 0.1, 4.248495242049359, 5.736572297479192, 7.2196420401307355, 4.2626798770413155, 0.1, 6.385194398997726, 6.452048954437226, 5.501258210544727, 7.83002808253384, 4.3694478524670215, 0.1, 4.969813299576001, 7.2640301428995295, 4.9344739331306915, 5.062595033026967, 8.28045768658256, 0.1, 7.983098940710892, 10.749870708994932, 7.906178840394815, 7.828038032125829, 7.9617188159813645, 0.1, 7.02108396428914, 5.420534999272286, 0.1, 5.902633333401366, 9.372374183823416, 4.394449154672439, 5.236441962829949, 0.1, 0.1, 4.709530201312334, 5.438079308923196, 6.329720905522696, 0.1, 5.3230099791384085, 5.497168225293202, 6.366470447731438, 0.1, 5.564520407322694, 7.295735072749282, 0.1, 7.2682230211595655, 9.77996306625063, 5.5254529391317835, 5.365976015021851, 7.046647277848756, 0.1]}, \"node\": {\"color\": [\"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\", \"#636EFA\", \"#EF553B\", \"#00CC96\", \"#AB63FA\", \"#FFA15A\", \"#19D3F3\", \"rgba(192,192,192,0)\"], \"hovertemplate\": \"10^%{value} users predominantly commented on %{label}<extra></extra>\", \"label\": [\" \", \"Sanders\", \" \", \" \", \" \", \" \", \"No Comment\", \" \", \" \", \"Klobuchar\", \"Warren\", \" \", \"Yang\", \" \", \" \", \" \", \" \", \" \", \"Buttigieg\", \" \", \" \", \"Biden\", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \", \" \"], \"line\": {\"color\": \"rgba(192,192,192,0)\"}, \"pad\": 10, \"thickness\": 10, \"x\": [0.000625, 0.000625, 0.063125, 0.063125, 0.063125, 0.063125, 0.063125, 0.125625, 0.125625, 0.125625, 0.125625, 0.125625, 0.188125, 0.188125, 0.188125, 0.188125, 0.188125, 0.188125, 0.250625, 0.250625, 0.250625, 0.250625, 0.250625, 0.250625, 0.313125, 0.313125, 0.313125, 0.313125, 0.313125, 0.313125, 0.375625, 0.375625, 0.375625, 0.375625, 0.375625, 0.375625, 0.438125, 0.438125, 0.438125, 0.438125, 0.438125, 0.438125, 0.500625, 0.500625, 0.500625, 0.500625, 0.500625, 0.500625, 0.563125, 0.563125, 0.563125, 0.563125, 0.563125, 0.563125, 0.625625, 0.625625, 0.625625, 0.625625, 0.625625, 0.625625, 0.688125, 0.688125, 0.688125, 0.688125, 0.688125, 0.688125, 0.750625, 0.750625, 0.750625, 0.750625, 0.750625, 0.750625, 0.813125, 0.813125, 0.813125, 0.813125, 0.813125, 0.813125, 0.813125, 0.875625, 0.875625, 0.875625, 0.875625, 0.875625, 0.875625, 0.875625, 0.938125, 0.938125, 0.938125, 0.938125, 0.938125, 0.938125], \"y\": [0.23, 0.01, 0.23, 1, 0.73, 0.41, 0.01, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 1, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 1, 0.73, 0.58, 0.41, 0.01, 0.9, 0.23, 0.73, 0.58, 0.41, 0.01]}, \"type\": \"sankey\"}],\n",
       "                        {\"annotations\": [{\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Jan 2019\", \"x\": 0.0003125, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Feb 2019\", \"x\": 0.06412499999999999, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Mar 2019\", \"x\": 0.12793749999999998, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Apr 2019\", \"x\": 0.19175, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"May 2019\", \"x\": 0.25556249999999997, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Jun 2019\", \"x\": 0.319375, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Jul 2019\", \"x\": 0.3831875, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Aug 2019\", \"x\": 0.447, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Sep 2019\", \"x\": 0.5108125, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Oct 2019\", \"x\": 0.574625, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Nov 2019\", \"x\": 0.6384375000000001, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Dec 2019\", \"x\": 0.70225, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Jan 2020\", \"x\": 0.7660625000000001, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Feb 2020\", \"x\": 0.829875, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Mar 2020\", \"x\": 0.8936875000000001, \"y\": -0.1}, {\"bgcolor\": \"rgba(0,0,0,0)\", \"font\": {\"size\": 18}, \"showarrow\": false, \"text\": \"Apr 2020\", \"x\": 0.9575, \"y\": -0.1}], \"font\": {\"size\": 23}, \"height\": 500, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"width\": 2000},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('3bb33409-1443-4f2f-a7e3-bc3639dc16ad');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "name_map = {\n",
    "    \"No_Comment\": \"No Comment\",\n",
    "    \"SandersForPresident\": \"Sanders\",\n",
    "    \"ElizabethWarren\": \"Warren\",\n",
    "    \"BaemyKlobaechar\": \"Klobuchar\",\n",
    "    \"JoeBiden\": \"Biden\",\n",
    "    \"Pete_Buttigieg\": \"Buttigieg\",\n",
    "    \"YangForPresidentHQ\": \"Yang\"\n",
    "}\n",
    "pretty_labels = []\n",
    "for cand in label_list:\n",
    "    l = name_map[cand.split(\": \",1)[1]]\n",
    "    if l in pretty_labels or (not cand in to_position):\n",
    "        pretty_labels.append(\" \")\n",
    "    else:\n",
    "        pretty_labels.append(l)\n",
    "\n",
    "data = [go.Sankey(\n",
    "            node = dict(\n",
    "              pad = 10,\n",
    "              thickness = 10,\n",
    "              line = dict(color = \"rgba(192,192,192,0)\"),\n",
    "              label =pretty_labels,\n",
    "              color = color_list,\n",
    "              x=x,\n",
    "              y=y,\n",
    "              hovertemplate='10^%{value} users predominantly commented on %{label}<extra></extra>',  \n",
    "            ),\n",
    "            link = dict(\n",
    "                source = source_target_df['sourceID'],\n",
    "                target = source_target_df['targetID'],\n",
    "                value = source_target_df['count'],\n",
    "                hovertemplate='10^%{value} users switched subreddits<extra></extra>',\n",
    "                color =  source_target_df['color']\n",
    "            )\n",
    "        )]\n",
    "\n",
    "dates = [datetime.strptime(date, '%Y-%m-%d').strftime(\"%b %Y\") for date in uc_pivot[\"date\"].unique()]\n",
    "percs = [(i+0.005+i*0.021)/16 for i in range(17)]\n",
    "annotations = [dict(x=perc,y=-0.1,\n",
    "                    text=date,\n",
    "                    showarrow=False,\n",
    "                    font=dict(size=18),\n",
    "                    bgcolor=\"rgba(0,0,0,0)\") for perc,date in zip(percs,dates)]\n",
    "\n",
    "layout = dict(\n",
    "        height=500,\n",
    "        width=2000,\n",
    "#         margin=dict(t=40, l=50, r=100, b=100),\n",
    "        font=dict(size=23),\n",
    "        annotations=annotations\n",
    "    )\n",
    "\n",
    "fig = go.Figure(data=data,layout=layout)\n",
    "with open(\"visualizations/temporal/subreddit_flow.pdf\", \"wb\") as f:\n",
    "    f.write(scope.transform(fig, format=\"pdf\"))\n",
    "# fig.write_html(\"visualizations/temporal/subreddit_flow.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import StringType, ArrayType \n",
    "from pyspark.sql.functions import udf, col, split \n",
    "from utils import clean_text,get_wordnet_pos\n",
    "\n",
    "udf_clean = udf(clean_text,ArrayType(StringType()))\n",
    "# clean_text(\"Amy Klobuchar was running and eating at same time. She has bad habit of swimming after playing long hours in the Sun.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments = comments.withColumn(\"clean_body_tokenized\", udf_clean(comments[\"body\"]))\n",
    "comments = comments.withColumn(\"len\", size(\"clean_body_tokenized\"))\n",
    "# Drop comments that result in an empty string \"\" after cleaning. These are comments like \"ok\", \"who?\" etc. \n",
    "comments = comments.filter(\"len > 0\")\n",
    "comments.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word Distributions\n",
    "\n",
    "Probability of a word $w_i$ appearing a subreddit $R$ is \n",
    "\n",
    "$P(w_i|R) = \\frac{f(w_i,R)}{|R|}$\n",
    "\n",
    "Aka what I've been putting off for too long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from operator import add\n",
    "import pyspark.sql.functions as f\n",
    "from pyspark.ml.feature import CountVectorizer \n",
    "import os\n",
    "\n",
    "\n",
    "def filter_dict(a_dict,thresh=0.0001,n=None):\n",
    "    if n:\n",
    "        return {key: val for key,val in a_dict.items() if val/n > thresh}\n",
    "    else:\n",
    "        return {key: val for key,val in a_dict.items() if val > thresh}\n",
    "\n",
    "def count_dict(dataframe,cand=None,thresh=0.0001):\n",
    "    iso = dataframe.where(dataframe[\"subreddit\"] == cand) if cand else dataframe\n",
    "    counts = iso.select(f.explode('clean_body_tokenized').alias('col')).groupBy('col').count()\n",
    "    n = counts.agg(f.sum(\"count\")).collect()[0][0]\n",
    "    print(counts.show(),n)\n",
    "    counts = counts.where(f\"(count/{n})>{thresh}\").toPandas()\n",
    "    print(counts.head())\n",
    "    counts.columns = [\"word\",\"count\"]\n",
    "    counts = counts.set_index(\"word\")\n",
    "    counts = pd.Series(counts[\"count\"],name=cand)\n",
    "    return counts,n\n",
    "\n",
    "def count_vec(dataframe,fp,normalized_fp,columns,include_total=False):\n",
    "    count_df = pd.DataFrame()\n",
    "    perc_df = pd.DataFrame()\n",
    "    try:\n",
    "        count_df = pd.read_csv(fp,index_col=0)\n",
    "        perc_df = pd.read_csv(normalized_fp,index_col=0)\n",
    "    except:\n",
    "        pass\n",
    "    count_df.index.name = \"word\"\n",
    "    perc_df.index.name = \"word\"\n",
    "\n",
    "    for column in tqdm(columns):\n",
    "        n = None\n",
    "        if not column in count_df:\n",
    "            count_vec,n = count_dict(dataframe,column)\n",
    "            count_df = pd.concat([count_df,count_vec], axis=1) \n",
    "            count_df.to_csv(fp)\n",
    "        if not column in perc_df:\n",
    "            n = n if n else dataframe.where(dataframe[\"subreddit\"] == column).select(f.sum(\"len\")).collect()[0][0]\n",
    "            perc_df = pd.concat([perc_df,count_df[column]/n], axis=1) \n",
    "            perc_df.to_csv(normalized_fp)\n",
    "    if include_total and not \"total\" in count_df:\n",
    "        count_df[\"total\"] = count_df.sum(axis=1)\n",
    "    if include_total and not \"total\" in perc_df:\n",
    "        n = dataframe.select(f.sum(\"len\")).collect()[0][0]\n",
    "        perc_df[\"total\"] = count_df[\"total\"]/n\n",
    "    count_df = count_df.fillna(0)\n",
    "    perc_df = perc_df.fillna(0)\n",
    "    count_df.to_csv(fp)\n",
    "    perc_df.to_csv(normalized_fp)\n",
    "    return count_df, perc_df\n",
    "\n",
    "\n",
    "to_odds = lambda x : (x/(1-x+0.001))\n",
    "log_odds_ratio = lambda o1,o2 : np.log((o1+0.001)/(o2+0.001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d49b607372dc433e9974302b7ef3a2da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>JoeBiden</th>\n",
       "      <th>BaemyKlobaechar</th>\n",
       "      <th>SandersForPresident</th>\n",
       "      <th>ElizabethWarren</th>\n",
       "      <th>Pete_Buttigieg</th>\n",
       "      <th>YangForPresidentHQ</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>0.010252</td>\n",
       "      <td>0.011152</td>\n",
       "      <td>0.009426</td>\n",
       "      <td>0.009869</td>\n",
       "      <td>0.010495</td>\n",
       "      <td>0.010606</td>\n",
       "      <td>0.010091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.010045</td>\n",
       "      <td>0.007489</td>\n",
       "      <td>0.012278</td>\n",
       "      <td>0.010473</td>\n",
       "      <td>0.010355</td>\n",
       "      <td>0.013181</td>\n",
       "      <td>0.012071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>would</th>\n",
       "      <td>0.009590</td>\n",
       "      <td>0.009779</td>\n",
       "      <td>0.008738</td>\n",
       "      <td>0.009892</td>\n",
       "      <td>0.008349</td>\n",
       "      <td>0.010203</td>\n",
       "      <td>0.009287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trump</th>\n",
       "      <td>0.009569</td>\n",
       "      <td>0.008459</td>\n",
       "      <td>0.006323</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.003913</td>\n",
       "      <td>0.004635</td>\n",
       "      <td>0.005220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vote</th>\n",
       "      <td>0.009532</td>\n",
       "      <td>0.007058</td>\n",
       "      <td>0.009480</td>\n",
       "      <td>0.006602</td>\n",
       "      <td>0.004527</td>\n",
       "      <td>0.005016</td>\n",
       "      <td>0.006791</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        JoeBiden  BaemyKlobaechar  SandersForPresident  ElizabethWarren  \\\n",
       "word                                                                      \n",
       "like    0.010252         0.011152             0.009426         0.009869   \n",
       "people  0.010045         0.007489             0.012278         0.010473   \n",
       "would   0.009590         0.009779             0.008738         0.009892   \n",
       "trump   0.009569         0.008459             0.006323         0.004320   \n",
       "vote    0.009532         0.007058             0.009480         0.006602   \n",
       "\n",
       "        Pete_Buttigieg  YangForPresidentHQ     total  \n",
       "word                                                  \n",
       "like          0.010495            0.010606  0.010091  \n",
       "people        0.010355            0.013181  0.012071  \n",
       "would         0.008349            0.010203  0.009287  \n",
       "trump         0.003913            0.004635  0.005220  \n",
       "vote          0.004527            0.005016  0.006791  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fp = \"dataframes/word_freq/comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-comment_counts.csv\"\n",
    "\n",
    "cand_count_df, cand_perc_df = count_vec(None,fp,normalized_fp,left_candidates)\n",
    "cand_count_df,cand_perc_df = cand_count_df[~cand_count_df.index.isin(cand_names)],cand_perc_df[~cand_perc_df.index.isin(cand_names)]\n",
    "\n",
    "cand_perc_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Odds Ratio\n",
    "\n",
    "Source: ```Monroe, Burt L., Michael P. Colaresi, and Kevin M. Quinn. \"Fightin'words: Lexical feature selection and evaluation for identifying the content of political conflict.\" Political Analysis 16, no. 4 (2008): 372-403.```\n",
    "\n",
    "Odds for a word $w$ in a class of documents $i$ is $O^{(i)}_{w}=\\frac{f^{(i)}_{w}}{(1-f^{(i)}_{w})}$\n",
    "\n",
    "Odds ratio for two classes (candidates) would be $\\theta^{(i_1 - i_2)}_{w} = \\frac{O^{(i^1)}_{w}}{O^{(i^2)}_{w}}$\n",
    "\n",
    "Log odds ratio would then be $log(\\theta^{(i_1 - i_2)}_{w})$\n",
    "\n",
    "By logging the odds ratio we can compare the odds ratio of words across classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from itertools import combinations\n",
    "from numpy.random import randint\n",
    "\n",
    "# For each n choose 2 combinations of candidates\n",
    "for cand1,cand2 in list(combinations(left_candidates,2)):\n",
    "    # Calculate log odds ratio\n",
    "    isolated_perc = cand_perc_df[[cand1,cand2]]\n",
    "    word_odds = isolated_perc.applymap(lambda x : to_odds(x))\n",
    "    word_odds[\"log_odds\"] = word_odds.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "    # get absolute frequency of word usage\n",
    "    word_odds[\"use\"] = cand_count_df[cand1] + cand_count_df[cand2]\n",
    "    word_odds = word_odds[np.isfinite(word_odds).all(1)]\n",
    "    word_odds = word_odds.sort_values(\"log_odds\",ascending=False)\n",
    "    # visualize\n",
    "    fig = px.scatter(word_odds,\n",
    "                     x=\"use\",\n",
    "                     y=\"log_odds\",\n",
    "                     hover_data=[word_odds.index],\n",
    "                      labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                     opacity=0.5)\n",
    "    # annotations\n",
    "    head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "    annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                        y = row[\"log_odds\"],\n",
    "                        ay = randint(-20,20),\n",
    "                        ax = randint(-20,20),\n",
    "                        text = row.name) \n",
    "                   for _,row in \n",
    "                   word_odds.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "    xmax = word_odds[\"use\"].max()*0.9\n",
    "    ymax,ymin = word_odds[\"log_odds\"].max()*1.2, word_odds[\"log_odds\"].min()\n",
    "    annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=f\"<b>{cand1}</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=f\"<b>{cand2}</b>\")]\n",
    "    args = {\n",
    "        \"font\":{\"size\": 20},\n",
    "        \"template\": \"simple_white\",\n",
    "        \"title\": \"Log Odds Ratio of Word Usage (/r/{}, /r/{})\".format(cand1,cand2),\n",
    "        \"annotations\": annotations\n",
    "    }\n",
    "    fig.update_layout(**args)\n",
    "    fig.update_xaxes(type=\"log\")\n",
    "#     fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(cand1,cand2))\n",
    "#     fig.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Master Distribution\n",
    "\n",
    "Total comments and word frequencies on reddit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import clean_text\n",
    "\n",
    "total_comments = pd.read_csv(\"/misinfo/all1grams.csv\",header=None)\n",
    "total_comments.columns = [\"word\",\"total\"]\n",
    "total_comments[\"word\"] = total_comments[\"word\"].apply(clean_text)\n",
    "total_comments = total_comments[total_comments[\"word\"].map(lambda x : len(x)>0)]\n",
    "total_comments[\"word\"] = total_comments[\"word\"].apply(lambda x : x[0])\n",
    "# merge duplicate words \n",
    "total_comments = total_comments.groupby([\"word\"],as_index=False).agg({'total': 'sum'})\n",
    "total_comments = total_comments.dropna().set_index(\"word\")\n",
    "\n",
    "total_comment_perc = total_comments.copy()\n",
    "n = total_comment_perc.sum()\n",
    "total_comment_perc = total_comment_perc/n\n",
    "total_comment_perc, total_comments = total_comment_perc[total_comment_perc > 0.00001].dropna(), total_comments[total_comment_perc > 0.00001].dropna()\n",
    "# total_comment_perc\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Against Master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# cand_perc_df,cand_count_df = cand_perc_df.drop(\"total\",axis=1),cand_count_df.drop(\"total\",axis=1)\n",
    "cand_perc_df = pd.concat([cand_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "cand_count_df = pd.concat([cand_count_df,total_comments], axis=1).fillna(0)\n",
    "cand_count_df = cand_count_df.loc[:,~cand_count_df.columns.duplicated()]\n",
    "cand_perc_df = cand_perc_df.loc[:,~cand_perc_df.columns.duplicated()]\n",
    "cand_count_df.index.name, cand_perc_df.index.name = \"word\",\"word\"\n",
    "\n",
    "master_log_odds = cand_perc_df.copy().applymap(lambda x : to_odds(x))\n",
    "for cand in left_candidates:\n",
    "    master_log_odds[cand] = master_log_odds.apply(lambda x : log_odds_ratio(x[cand],x[\"total\"]),axis=1)\n",
    "master_log_odds = master_log_odds.loc[~(master_log_odds==0).any(axis=1)]\n",
    "master_log_odds = master_log_odds.reset_index()\n",
    "master_log_odds = master_log_odds.melt(id_vars=[\"word\"], value_vars=left_candidates,var_name=\"subreddit\",value_name=\"log_odds\")\n",
    "\n",
    "master_log_odds = master_log_odds.merge(cand_count_df[\"total\"],how=\"left\",left_on=\"word\",right_index=True)\n",
    "annotations = []\n",
    "for cand in left_candidates:\n",
    "    iso = master_log_odds[master_log_odds[\"subreddit\"] == cand].sort_values(\"log_odds\",ascending=False)\n",
    "    annotations += [dict(x = np.log10(row[\"total\"]),\n",
    "                        y = row[\"log_odds\"],\n",
    "                        ay = randint(0,30),\n",
    "                        ax = randint(-30,0),\n",
    "                        text = row[\"word\"]) \n",
    "                   for _,row in \n",
    "                   iso.head(3).iterrows() if row[\"total\"]>0]\n",
    "xmax = master_log_odds[\"total\"].max()*0.9\n",
    "ymax,ymin = master_log_odds[\"log_odds\"].max()*1.1, master_log_odds[\"log_odds\"].min()*1.1\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Subreddit Specific</b>\"),\n",
    "                dict(x= np.log10(xmax),y=0,showarrow=False,text=\"<b>General Usage</b>\"),\n",
    "                dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Other Subreddits</b>\")]\n",
    "    \n",
    "\n",
    "fig = px.scatter(master_log_odds,\n",
    "                 x=\"total\",\n",
    "                 y=\"log_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"total\": \"Word Frequency (All Subreddits)\",\n",
    "                     \"log_odds\": \"Log Odds Ratio Against All Subreddits\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "args = {\n",
    "    \"font\":{\"size\": 23},\n",
    "    \"height\": 1200,\n",
    "    \"width\": 1000,\n",
    "    \"template\": \"simple_white\",\n",
    "    \"annotations\": annotations,\n",
    "    \"legend\":{\"orientation\":\"h\"}\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "# fig.write_html(\"visualizations/content_analysis/all_subs-log_odds_ratio.html\")\n",
    "fig.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "n = 20\n",
    "top_words = pd.DataFrame()\n",
    "for cand in left_candidates:\n",
    "    iso = master_log_odds[master_log_odds[\"subreddit\"] == cand].sort_values(\"log_odds\",ascending=False)\n",
    "    print(iso.head(n))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word clouds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "odds_dict = master_log_odds.to_dict()\n",
    "for cand in left_candidates:\n",
    "    iso = master_log_odds[master_log_odds[\"subreddit\"] == cand]\n",
    "    word_cloud_dict = {}\n",
    "    for row in iso.iterrows():\n",
    "        word_cloud_dict[row[1].word] = row[1].log_odds\n",
    "\n",
    "    wordcloud = WordCloud(background_color=\"white\",\n",
    "                          colormap=\"nipy_spectral\").generate_from_frequencies(word_cloud_dict)\n",
    " \n",
    "    plt.figure(figsize=(12,6))\n",
    "    plt.imshow(wordcloud, interpolation=\"hermite\")\n",
    "#     plt.title(\"{} Subreddit Word Cloud\\n\".format(cand))\n",
    "    plt.axis('off')\n",
    "    plt.tight_layout(pad=0)\n",
    "#     plt.savefig(\"visualizations/content_analysis/word_clouds/{}_subreddit_word_cloud.pdf\".format(cand))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Log Odds Against Democratic Debates\n",
    "\n",
    "How different are the comments on candidate subs from the words of the candidates themselves "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import clean_text\n",
    "left_candidates = [\"JoeBiden\",\"SandersForPresident\",\"BaemyKlobaechar\",\"ElizabethWarren\",\"Pete_Buttigieg\",\"YangForPresidentHQ\"]\n",
    "cands = [\"Joe Biden\",\"Bernie Sanders\",\"Amy Klobuchar\",\"Elizabeth Warren\",\"Pete Buttigieg\",\"Andrew Yang\"]\n",
    "cand_speeches = pd.read_csv(\"external_datasets/candidate_speeches/debates.csv\")\n",
    "cand_speeches = cand_speeches[cand_speeches[\"speaker\"].isin(cands)].dropna(subset=[\"speech\"])\n",
    "cand_speeches[\"clean_text\"] = cand_speeches[\"speech\"].apply(clean_text)\n",
    "cand_speeches = cand_speeches[[\"speaker\",\"gender\",\"speech\",\"clean_text\"]].set_index(\"speaker\")\n",
    "# cand_speeches.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Joe Biden           14044.0\n",
      "Bernie Sanders      12319.0\n",
      "Amy Klobuchar        9762.0\n",
      "Elizabeth Warren    10160.0\n",
      "Pete Buttigieg       9734.0\n",
      "Andrew Yang          4490.0\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Joe Biden</th>\n",
       "      <th>Bernie Sanders</th>\n",
       "      <th>Amy Klobuchar</th>\n",
       "      <th>Elizabeth Warren</th>\n",
       "      <th>Pete Buttigieg</th>\n",
       "      <th>Andrew Yang</th>\n",
       "      <th>total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>make</th>\n",
       "      <td>0.018102</td>\n",
       "      <td>0.008561</td>\n",
       "      <td>0.014136</td>\n",
       "      <td>0.013679</td>\n",
       "      <td>0.010591</td>\n",
       "      <td>0.006904</td>\n",
       "      <td>0.013502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>go</th>\n",
       "      <td>0.017503</td>\n",
       "      <td>0.011743</td>\n",
       "      <td>0.015571</td>\n",
       "      <td>0.013036</td>\n",
       "      <td>0.011241</td>\n",
       "      <td>0.010468</td>\n",
       "      <td>0.014543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fact</th>\n",
       "      <td>0.015573</td>\n",
       "      <td>0.003182</td>\n",
       "      <td>0.000922</td>\n",
       "      <td>0.000734</td>\n",
       "      <td>0.002415</td>\n",
       "      <td>0.004900</td>\n",
       "      <td>0.005636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>would</th>\n",
       "      <td>0.013044</td>\n",
       "      <td>0.003258</td>\n",
       "      <td>0.012395</td>\n",
       "      <td>0.002295</td>\n",
       "      <td>0.005481</td>\n",
       "      <td>0.012249</td>\n",
       "      <td>0.008247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>0.012844</td>\n",
       "      <td>0.025002</td>\n",
       "      <td>0.017107</td>\n",
       "      <td>0.020105</td>\n",
       "      <td>0.011148</td>\n",
       "      <td>0.012249</td>\n",
       "      <td>0.017915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>needs</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>0.000017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>endemic</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>0.000017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>heed</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>0.000017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fantasize</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>0.000017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>paine</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000223</td>\n",
       "      <td>0.000017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3529 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           Joe Biden  Bernie Sanders  Amy Klobuchar  Elizabeth Warren  \\\n",
       "make        0.018102        0.008561       0.014136          0.013679   \n",
       "go          0.017503        0.011743       0.015571          0.013036   \n",
       "fact        0.015573        0.003182       0.000922          0.000734   \n",
       "would       0.013044        0.003258       0.012395          0.002295   \n",
       "people      0.012844        0.025002       0.017107          0.020105   \n",
       "...              ...             ...            ...               ...   \n",
       "needs       0.000000        0.000000       0.000000          0.000000   \n",
       "endemic     0.000000        0.000000       0.000000          0.000000   \n",
       "heed        0.000000        0.000000       0.000000          0.000000   \n",
       "fantasize   0.000000        0.000000       0.000000          0.000000   \n",
       "paine       0.000000        0.000000       0.000000          0.000000   \n",
       "\n",
       "           Pete Buttigieg  Andrew Yang     total  \n",
       "make             0.010591     0.006904  0.013502  \n",
       "go               0.011241     0.010468  0.014543  \n",
       "fact             0.002415     0.004900  0.005636  \n",
       "would            0.005481     0.012249  0.008247  \n",
       "people           0.011148     0.012249  0.017915  \n",
       "...                   ...          ...       ...  \n",
       "needs            0.000000     0.000223  0.000017  \n",
       "endemic          0.000000     0.000223  0.000017  \n",
       "heed             0.000000     0.000223  0.000017  \n",
       "fantasize        0.000000     0.000223  0.000017  \n",
       "paine            0.000000     0.000223  0.000017  \n",
       "\n",
       "[3529 rows x 7 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debate_count = {}\n",
    "debate_prob = {}\n",
    "for cand in cands:\n",
    "    iso = cand_speeches.loc[cand]\n",
    "    word_count = pd.Series(np.concatenate([x for x in iso[\"clean_text\"]])).value_counts()\n",
    "    n = word_count.sum()\n",
    "    debate_count[cand] = filter_dict(word_count.to_dict(),n=n)\n",
    "    word_prob = filter_dict((word_count/n).to_dict())\n",
    "    debate_prob[cand] = word_prob\n",
    "\n",
    "debate_count = pd.DataFrame.from_dict(debate_count).fillna(0)\n",
    "print(debate_count.sum(axis=0))\n",
    "debate_count[\"total\"] = debate_count.sum(axis=1)\n",
    "debate_prob = pd.DataFrame.from_dict(debate_prob).fillna(0)\n",
    "debate_prob[\"total\"] = debate_count[\"total\"]/debate_count['total'].sum()\n",
    "debate_count,debate_prob = debate_count[~debate_count.index.isin(cand_names)],debate_prob[~debate_prob.index.isin(cand_names)]\n",
    "\n",
    "debate_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "colorbar": {
          "title": {
           "text": "r2"
          }
         },
         "type": "heatmap",
         "x": [
          "Joe Biden",
          "Bernie Sanders",
          "Amy Klobuchar",
          "Elizabeth Warren",
          "Pete Buttigieg",
          "Andrew Yang"
         ],
         "y": [
          "JoeBiden",
          "SandersForPresident",
          "BaemyKlobaechar",
          "ElizabethWarren",
          "Pete_Buttigieg",
          "YangForPresidentHQ"
         ],
         "z": [
          [
           0.005915432020781708,
           0.0007238470946436815,
           0.026590128661010515,
           0.011392870171388347,
           0.002503104750605667,
           0.01594413073275147
          ],
          [
           0.002318301178939275,
           0.027451700605683693,
           0.005287790670458148,
           0.0022072705207313303,
           0.021783421990097307,
           0.016314859441054264
          ],
          [
           0.0005414109590708982,
           0.006089192398711307,
           0.053098054309279163,
           0.003315864081798736,
           0.008682006010290455,
           0.003015092963708832
          ],
          [
           0.0027643099720096043,
           0.003150084858670077,
           0.009365260059796432,
           3.179655500562539e-05,
           0.0008512501451627408,
           0.02543928388933614
          ],
          [
           0.003982637121745847,
           0.020524883199156796,
           0.030801303269496594,
           0.002264137027012225,
           0.049069248622638284,
           0.00509755531535494
          ],
          [
           0.00014963276272272896,
           0.0016682289249402293,
           0.0051785671844272185,
           9.56469870705412e-06,
           0.00012170347253281921,
           0.04460832304698259
          ]
         ]
        }
       ],
       "layout": {
        "font": {
         "size": 23
        },
        "height": 600,
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "rgb(36,36,36)"
            },
            "error_y": {
             "color": "rgb(36,36,36)"
            },
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "baxis": {
             "endlinecolor": "rgb(36,36,36)",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "rgb(36,36,36)"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "line": {
              "color": "white",
              "width": 0.6
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 1,
              "tickcolor": "rgb(36,36,36)",
              "ticks": "outside"
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 1,
             "tickcolor": "rgb(36,36,36)",
             "ticks": "outside"
            },
            "colorscale": [
             [
              0,
              "#440154"
             ],
             [
              0.1111111111111111,
              "#482878"
             ],
             [
              0.2222222222222222,
              "#3e4989"
             ],
             [
              0.3333333333333333,
              "#31688e"
             ],
             [
              0.4444444444444444,
              "#26828e"
             ],
             [
              0.5555555555555556,
              "#1f9e89"
             ],
             [
              0.6666666666666666,
              "#35b779"
             ],
             [
              0.7777777777777778,
              "#6ece58"
             ],
             [
              0.8888888888888888,
              "#b5de2b"
             ],
             [
              1,
              "#fde725"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "rgb(237,237,237)"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "rgb(217,217,217)"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 1,
            "tickcolor": "rgb(36,36,36)",
            "ticks": "outside"
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "rgb(103,0,31)"
            ],
            [
             0.1,
             "rgb(178,24,43)"
            ],
            [
             0.2,
             "rgb(214,96,77)"
            ],
            [
             0.3,
             "rgb(244,165,130)"
            ],
            [
             0.4,
             "rgb(253,219,199)"
            ],
            [
             0.5,
             "rgb(247,247,247)"
            ],
            [
             0.6,
             "rgb(209,229,240)"
            ],
            [
             0.7,
             "rgb(146,197,222)"
            ],
            [
             0.8,
             "rgb(67,147,195)"
            ],
            [
             0.9,
             "rgb(33,102,172)"
            ],
            [
             1,
             "rgb(5,48,97)"
            ]
           ],
           "sequential": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#440154"
            ],
            [
             0.1111111111111111,
             "#482878"
            ],
            [
             0.2222222222222222,
             "#3e4989"
            ],
            [
             0.3333333333333333,
             "#31688e"
            ],
            [
             0.4444444444444444,
             "#26828e"
            ],
            [
             0.5555555555555556,
             "#1f9e89"
            ],
            [
             0.6666666666666666,
             "#35b779"
            ],
            [
             0.7777777777777778,
             "#6ece58"
            ],
            [
             0.8888888888888888,
             "#b5de2b"
            ],
            [
             1,
             "#fde725"
            ]
           ]
          },
          "colorway": [
           "#1F77B4",
           "#FF7F0E",
           "#2CA02C",
           "#D62728",
           "#9467BD",
           "#8C564B",
           "#E377C2",
           "#7F7F7F",
           "#BCBD22",
           "#17BECF"
          ],
          "font": {
           "color": "rgb(36,36,36)"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "white",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "white",
          "polar": {
           "angularaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "radialaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "yaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           },
           "zaxis": {
            "backgroundcolor": "white",
            "gridcolor": "rgb(232,232,232)",
            "gridwidth": 2,
            "linecolor": "rgb(36,36,36)",
            "showbackground": true,
            "showgrid": false,
            "showline": true,
            "ticks": "outside",
            "zeroline": false,
            "zerolinecolor": "rgb(36,36,36)"
           }
          },
          "shapedefaults": {
           "fillcolor": "black",
           "line": {
            "width": 0
           },
           "opacity": 0.3
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "baxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           },
           "bgcolor": "white",
           "caxis": {
            "gridcolor": "rgb(232,232,232)",
            "linecolor": "rgb(36,36,36)",
            "showgrid": false,
            "showline": true,
            "ticks": "outside"
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "rgb(232,232,232)",
           "linecolor": "rgb(36,36,36)",
           "showgrid": false,
           "showline": true,
           "ticks": "outside",
           "title": {
            "standoff": 15
           },
           "zeroline": false,
           "zerolinecolor": "rgb(36,36,36)"
          }
         }
        },
        "width": 750,
        "xaxis": {
         "title": {
          "text": "Debate"
         }
        },
        "yaxis": {
         "title": {
          "text": "Subreddit"
         }
        }
       }
      },
      "text/html": [
       "<div>\n",
       "        \n",
       "        \n",
       "            <div id=\"b4f1e165-d427-4255-8062-3c4f3f7d7519\" class=\"plotly-graph-div\" style=\"height:600px; width:750px;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                require([\"plotly\"], function(Plotly) {\n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"b4f1e165-d427-4255-8062-3c4f3f7d7519\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'b4f1e165-d427-4255-8062-3c4f3f7d7519',\n",
       "                        [{\"colorbar\": {\"title\": {\"text\": \"r2\"}}, \"type\": \"heatmap\", \"x\": [\"Joe Biden\", \"Bernie Sanders\", \"Amy Klobuchar\", \"Elizabeth Warren\", \"Pete Buttigieg\", \"Andrew Yang\"], \"y\": [\"JoeBiden\", \"SandersForPresident\", \"BaemyKlobaechar\", \"ElizabethWarren\", \"Pete_Buttigieg\", \"YangForPresidentHQ\"], \"z\": [[0.005915432020781708, 0.0007238470946436815, 0.026590128661010515, 0.011392870171388347, 0.002503104750605667, 0.01594413073275147], [0.002318301178939275, 0.027451700605683693, 0.005287790670458148, 0.0022072705207313303, 0.021783421990097307, 0.016314859441054264], [0.0005414109590708982, 0.006089192398711307, 0.053098054309279163, 0.003315864081798736, 0.008682006010290455, 0.003015092963708832], [0.0027643099720096043, 0.003150084858670077, 0.009365260059796432, 3.179655500562539e-05, 0.0008512501451627408, 0.02543928388933614], [0.003982637121745847, 0.020524883199156796, 0.030801303269496594, 0.002264137027012225, 0.049069248622638284, 0.00509755531535494], [0.00014963276272272896, 0.0016682289249402293, 0.0051785671844272185, 9.56469870705412e-06, 0.00012170347253281921, 0.04460832304698259]]}],\n",
       "                        {\"font\": {\"size\": 23}, \"height\": 600, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"rgb(36,36,36)\"}, \"error_y\": {\"color\": \"rgb(36,36,36)\"}, \"marker\": {\"line\": {\"color\": \"white\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"white\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"rgb(36,36,36)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(36,36,36)\"}, \"baxis\": {\"endlinecolor\": \"rgb(36,36,36)\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"rgb(36,36,36)\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"colorscale\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"colorscale\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"colorscale\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"line\": {\"color\": \"white\", \"width\": 0.6}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"colorscale\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"colorscale\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}, \"colorscale\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"rgb(237,237,237)\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"rgb(217,217,217)\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 1, \"tickcolor\": \"rgb(36,36,36)\", \"ticks\": \"outside\"}}, \"colorscale\": {\"diverging\": [[0.0, \"rgb(103,0,31)\"], [0.1, \"rgb(178,24,43)\"], [0.2, \"rgb(214,96,77)\"], [0.3, \"rgb(244,165,130)\"], [0.4, \"rgb(253,219,199)\"], [0.5, \"rgb(247,247,247)\"], [0.6, \"rgb(209,229,240)\"], [0.7, \"rgb(146,197,222)\"], [0.8, \"rgb(67,147,195)\"], [0.9, \"rgb(33,102,172)\"], [1.0, \"rgb(5,48,97)\"]], \"sequential\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]], \"sequentialminus\": [[0.0, \"#440154\"], [0.1111111111111111, \"#482878\"], [0.2222222222222222, \"#3e4989\"], [0.3333333333333333, \"#31688e\"], [0.4444444444444444, \"#26828e\"], [0.5555555555555556, \"#1f9e89\"], [0.6666666666666666, \"#35b779\"], [0.7777777777777778, \"#6ece58\"], [0.8888888888888888, \"#b5de2b\"], [1.0, \"#fde725\"]]}, \"colorway\": [\"#1F77B4\", \"#FF7F0E\", \"#2CA02C\", \"#D62728\", \"#9467BD\", \"#8C564B\", \"#E377C2\", \"#7F7F7F\", \"#BCBD22\", \"#17BECF\"], \"font\": {\"color\": \"rgb(36,36,36)\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"white\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"white\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\"}, \"bgcolor\": \"white\", \"radialaxis\": {\"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"white\", \"gridcolor\": \"rgb(232,232,232)\", \"gridwidth\": 2, \"linecolor\": \"rgb(36,36,36)\", \"showbackground\": true, \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\", \"zeroline\": false, \"zerolinecolor\": \"rgb(36,36,36)\"}, \"yaxis\": {\"backgroundcolor\": \"white\", \"gridcolor\": \"rgb(232,232,232)\", \"gridwidth\": 2, \"linecolor\": \"rgb(36,36,36)\", \"showbackground\": true, \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\", \"zeroline\": false, \"zerolinecolor\": \"rgb(36,36,36)\"}, \"zaxis\": {\"backgroundcolor\": \"white\", \"gridcolor\": \"rgb(232,232,232)\", \"gridwidth\": 2, \"linecolor\": \"rgb(36,36,36)\", \"showbackground\": true, \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\", \"zeroline\": false, \"zerolinecolor\": \"rgb(36,36,36)\"}}, \"shapedefaults\": {\"fillcolor\": \"black\", \"line\": {\"width\": 0}, \"opacity\": 0.3}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\"}, \"baxis\": {\"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\"}, \"bgcolor\": \"white\", \"caxis\": {\"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\", \"title\": {\"standoff\": 15}, \"zeroline\": false, \"zerolinecolor\": \"rgb(36,36,36)\"}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"rgb(232,232,232)\", \"linecolor\": \"rgb(36,36,36)\", \"showgrid\": false, \"showline\": true, \"ticks\": \"outside\", \"title\": {\"standoff\": 15}, \"zeroline\": false, \"zerolinecolor\": \"rgb(36,36,36)\"}}}, \"width\": 750, \"xaxis\": {\"title\": {\"text\": \"Debate\"}}, \"yaxis\": {\"title\": {\"text\": \"Subreddit\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('b4f1e165-d427-4255-8062-3c4f3f7d7519');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                });\n",
       "            </script>\n",
       "        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from itertools import product\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "debate_odds = debate_prob.applymap(lambda x : to_odds(x))\n",
    "subreddit_odds = cand_perc_df.applymap(lambda x : to_odds(x))\n",
    "correlation = []\n",
    "for subreddit,candidate in list(product(left_candidates,cands)):\n",
    "    deb = debate_odds[[candidate,\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "    sub = subreddit_odds[[subreddit,\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "    merged = pd.merge(pd.Series(sub,name=subreddit),pd.Series(deb,name=candidate),left_index=True,right_index=True,how=\"outer\").fillna(0)\n",
    "    model = np.polyfit(merged[subreddit], merged[candidate], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(merged[candidate], predict(merged[subreddit]))\n",
    "    correlation.append(r2)\n",
    "\n",
    "correlation = np.array(correlation).reshape(len(left_candidates),len(cands))\n",
    "data = go.Heatmap(z=correlation,colorbar=dict(title=\"r2\"),y=left_candidates,x=cands)\n",
    "\n",
    "layout = {\n",
    "    \"font\":{\"size\": 23},\n",
    "    \"height\": 600,\n",
    "    \"width\": 750,    \n",
    "    \"template\": \"simple_white\",\n",
    "#     \"title\": \"Subreddit-Debate Log-Odds Ratio Correlation\",\n",
    "    \"yaxis\": {\"title\": \"Subreddit\"},\n",
    "    \"xaxis\": {\"title\": \"Debate\"},\n",
    "}\n",
    "fig = go.Figure(data=data,layout=layout)\n",
    "with open(\"visualizations/content_analysis/subreddit-debate_correlation.pdf\", \"wb\") as f:\n",
    "    f.write(scope.transform(fig, format=\"pdf\"))\n",
    "# fig.write_html(\"visualizations/content_analysis/subreddit-debate_correlation.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize Democratic Debates Word Freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for subreddit,candidate in zip(left_candidates,cands):\n",
    "    deb = debate_odds[[candidate,\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "    sub = subreddit_odds[[subreddit,\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "    merged = pd.merge(pd.Series(sub,name=subreddit),pd.Series(deb,name=candidate),left_index=True,right_index=True,how=\"outer\").fillna(0)\n",
    "    # Drop any row that contains a 0 (avoid divide by 0 error)\n",
    "    merged = merged.loc[~(merged==0).any(axis=1)]\n",
    "    merged[\"log_odds\"] = merged.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "    merged = merged.sort_values(\"log_odds\")\n",
    "    model = np.polyfit(merged[subreddit], merged[candidate], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(merged[candidate], predict(merged[subreddit]))\n",
    "    print(subreddit,candidate,r2)\n",
    "    x=np.linspace(merged[candidate].min(),merged[candidate].max(),2)\n",
    "   \n",
    "    fig = px.scatter(merged,\n",
    "                     x=candidate,\n",
    "                     y=subreddit,\n",
    "                     hover_data=[merged.index],\n",
    "                     opacity=0.5,\n",
    "                     labels={\n",
    "                     subreddit: \"Subreddit Usage\",\n",
    "                     candidate: \"Debate Usage\",\n",
    "                     \"word\": \"Word\"\n",
    "                     })\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    name=r2,\n",
    "                    mode='lines')\n",
    "    args = {\n",
    "        \"font\":{\"size\": 20},\n",
    "        \"template\": \"simple_white\",\n",
    "        \"title\": \"{} Log Odds Ratio Comparison (Debates/Subreddit)\".format(candidate),\n",
    "    }\n",
    "    fig.update_layout(**args)\n",
    "#     fig.write_html(\"visualizations/content_analysis/{}_debate_sub-log_odds.html\".format(subreddit))\n",
    "    fig.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Content Analysis Using Political Dimension Subreddits\n",
    "\n",
    "We know that our political dimensions from the embeddings line up with survey data. But is this mirrored in the content/word usage of candidate subreddits compared to explicitly conservative/progressive subreddits?\n",
    "\n",
    "\n",
    "**Validate against master distribution**\n",
    "This way we have some valid way of comparing what words disproportionately occur in the partisan subreddits/political subreddits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Progressive Subreddits\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "ideological_subs =[(\"Conservative\",\"progressive\"),\n",
    "                   (\"Republican\",\"democrats\"),\n",
    "                   (\"conservatives\",\"SocialDemocracy\"),\n",
    "                   (\"TheNewRight\",\"WeAreNotAsking\"),\n",
    "                   (\"neoliberal\",\"dsa\"),\n",
    "                   (\"Capitalism\",\"capitalism_in_decay\"),\n",
    "                   (\"walkaway\",\"LateStageImperialism\")]\n",
    "prog_subs, con_subs = [i for _,i in ideological_subs],[i for i,_ in ideological_subs]\n",
    "\n",
    "assign_sub = lambda x : \"progressive_sub\" if x in prog_subs else \"con_subs\" if x in con_subs else x\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "\n",
    "pol_comments = comments.withColumn(\"sub_type\",udf_assign_sub(comments[\"subreddit\"]))\n",
    "pol_comments = pol_comments.drop('subreddit').withColumnRenamed('sub_type', 'subreddit')\n",
    "\n",
    "# pol_comments.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/ideology_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-ideology_comment_counts.csv\"\n",
    "\n",
    "ideology_count_df,ideology_perc_df = count_vec(pol_comments,fp,normalized_fp,[\"progressive_sub\",\"con_subs\"],include_total=False)\n",
    "\n",
    "ideology_perc_df = pd.concat([ideology_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "ideology_count_df = pd.concat([ideology_count_df,total_comments], axis=1).fillna(0)\n",
    "ideology_count_df = ideology_count_df.loc[:,~ideology_count_df.columns.duplicated()]\n",
    "ideology_perc_df = ideology_perc_df.loc[:,~ideology_perc_df.columns.duplicated()]\n",
    "ideology_count_df.index.name, ideology_perc_df.index.name = \"word\",\"word\"\n",
    "\n",
    "ideology_count_df[left_candidates],ideology_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from numpy.random import randint\n",
    "\n",
    "isolated_ideology = ideology_perc_df[[\"con_subs\",\"progressive_sub\"]]\n",
    "isolated_ideology = isolated_ideology.applymap(lambda x : to_odds(x))\n",
    "\n",
    "isolated_ideology[\"log_odds\"] = isolated_ideology.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "# get absolute frequency of word usage\n",
    "isolated_ideology[\"use\"] = ideology_count_df[\"con_subs\"] + ideology_count_df[\"progressive_sub\"]\n",
    "isolated_ideology = isolated_ideology[np.isfinite(isolated_ideology).all(1)]\n",
    "isolated_ideology = isolated_ideology.sort_values(\"log_odds\")\n",
    "# visualize\n",
    "fig = px.scatter(isolated_ideology,\n",
    "                 x=\"use\",\n",
    "                 y=\"log_odds\",\n",
    "                 hover_data=[isolated_ideology.index],\n",
    "                 labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                 opacity=0.5)\n",
    "# annotations\n",
    "head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                    y = row[\"log_odds\"],\n",
    "                    ay = randint(-20,20),\n",
    "                    ax = randint(-20,20),\n",
    "                    text = row.name) \n",
    "               for _,row in \n",
    "               isolated_ideology.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "xmax = isolated_ideology[\"use\"].max()*0.9\n",
    "ymax,ymin = isolated_ideology[\"log_odds\"].max()*1.2, isolated_ideology[\"log_odds\"].min()\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Conservative Subreddits</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Progressive Subreddits</b>\")]\n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Log Odds Ratio of Word Usage ({}, {})\".format(\"Conservative Subreddits\",\"Progressive Subreddits\"),\n",
    "    \"annotations\": annotations\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "# fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(\"con_subs\",\"progressive_subs\"))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "# merged = pd.merge(cand_perc_df[left_candidates].applymap(lambda x : to_odds(x)),ideology_perc_df[[\"progressive_sub\"]],left_index=True,right_index=True,how=\"inner\").fillna(0)\n",
    "melted = ideology_perc_df.reset_index()\n",
    "melted = melted.melt(value_vars=left_candidates,id_vars=[\"word\"],var_name=\"subreddit\",value_name=\"word_odds\").set_index(\"word\",drop=True)\n",
    "melted = pd.merge(ideology_perc_df,melted,left_index=True,right_index=True,how=\"right\").fillna(0)\n",
    "melted = melted[[\"subreddit\",\"word_odds\",\"progressive_sub\",\"con_subs\",\"total\"]]\n",
    "melted[[\"word_odds\",\"progressive_sub\",\"total\"]] = melted[[\"word_odds\",\"progressive_sub\",\"total\"]].applymap(lambda x : to_odds(x))\n",
    "melted[\"word_odds\"] = melted[[\"word_odds\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"progressive_sub\"] = melted[[\"progressive_sub\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"con_subs\"] =  melted[[\"con_subs\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted = melted.reset_index()\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"progressive_sub\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    prog_r2 = r2_score(iso[\"progressive_sub\"], predict(iso[\"word_odds\"]))\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"con_subs\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    con_r2 = r2_score(iso[\"con_subs\"], predict(iso[\"word_odds\"]))\n",
    "    print(f\"{cand}:\\n\\tprogressive alignment:{prog_r2}\\n\\tconservative alignment:{con_r2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "fig = px.scatter(melted,\n",
    "                 x=\"progressive_sub\",\n",
    "                 y=\"word_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"progressive_sub\": \"Log Odds Ratio (Progressive Subreddits)\",\n",
    "                     \"word_odds\": \"Log Odds Ratio (Candidate Subreddit)\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "xmin,xmax = melted[\"progressive_sub\"].min(),melted[\"progressive_sub\"].max()\n",
    "x = np.linspace(xmin,xmax,num=2)\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"progressive_sub\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(iso[\"progressive_sub\"], predict(iso[\"word_odds\"]))\n",
    "    print(cand,r2)\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    mode='lines',\n",
    "                    line=dict(dash=\"dash\",width=2,color=mapping[cand]),\n",
    "                    name=f\"{cand} Linear Regression (r<sup>2</sup>={r2:.2f})\",\n",
    "                    legendgroup=cand,\n",
    "                    showlegend=False\n",
    "                   )\n",
    "    \n",
    "args = {\n",
    "    \"font\":{\"size\": 23},\n",
    "    \"height\": 600,\n",
    "    \"width\": 1000,    \n",
    "     \"legend\": {\"orientation\":\"h\",\n",
    "              \"yanchor\":\"bottom\",\n",
    "                \"y\":1.02,\n",
    "                \"xanchor\":\"right\",\n",
    "                \"x\":1},\n",
    "    \"template\": \"simple_white\",\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "# fig.write_html(\"visualizations/content_analysis/political_dimension_comparisons/ideology.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Age Subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_subs = [(\"RedditForGrownups\",\"teenagers\"),\n",
    "            (\"RedditForGrownups\",\"teenagersnew\")]\n",
    "\n",
    "young_subs, old_subs = [i for _,i in age_subs],[i for i,_ in age_subs]\n",
    "\n",
    "assign_sub = lambda x : \"old_sub\" if x in old_subs else \"young_sub\" if x in young_subs else x\n",
    "\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "# Drop comments that result in an empty string \"\" after cleaning. These are comments like \"ok\", \"who?\" etc. \n",
    "age_comments = comments.withColumn(\"sub_type\",udf_assign_sub(comments[\"subreddit\"]))\n",
    "age_comments = age_comments.drop('subreddit').withColumnRenamed('sub_type', 'subreddit')\n",
    "\n",
    "# age_comments.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/age_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-age_comment_counts.csv\"\n",
    "\n",
    "age_count_df,age_perc_df = count_vec(age_comments,fp,normalized_fp,[\"old_sub\",\"young_sub\"],include_total=False)\n",
    "age_perc_df = pd.concat([age_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "age_count_df = pd.concat([age_count_df,total_comments], axis=1).fillna(0)\n",
    "age_count_df = age_count_df.loc[:,~age_count_df.columns.duplicated()]\n",
    "age_perc_df = age_perc_df.loc[:,~age_perc_df.columns.duplicated()]\n",
    "\n",
    "age_perc_df.index.name,age_count_df.index.name = \"word\",\"word\"\n",
    "age_count_df[left_candidates],age_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n",
    "age_count_df.to_csv(fp),age_perc_df.to_csv(normalized_fp)\n",
    "\n",
    "age_perc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import randint\n",
    "\n",
    "isolated_age = age_perc_df[[\"old_sub\",\"young_sub\"]]\n",
    "isolated_age = isolated_age.applymap(lambda x : to_odds(x))\n",
    "\n",
    "isolated_age[\"log_odds\"] = isolated_age.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "# get absolute frequency of word usage\n",
    "isolated_age[\"use\"] = age_count_df[\"old_sub\"] + age_count_df[\"young_sub\"]\n",
    "isolated_age = isolated_age[np.isfinite(isolated_age).all(1)]\n",
    "isolated_age = isolated_age.sort_values(\"log_odds\")\n",
    "# visualize\n",
    "fig = px.scatter(isolated_age,\n",
    "                 x=\"use\",\n",
    "                 y=\"log_odds\",\n",
    "                 hover_data=[isolated_age.index],\n",
    "                 labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                 opacity=0.5)\n",
    "# annotations\n",
    "head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                    y = row[\"log_odds\"],\n",
    "                    ay = randint(-20,20),\n",
    "                    ax = randint(-20,20),\n",
    "                    text = row.name) \n",
    "               for _,row in \n",
    "               isolated_age.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "xmax = isolated_age[\"use\"].max()*0.9\n",
    "ymax,ymin = isolated_age[\"log_odds\"].max()*1.2, isolated_age[\"log_odds\"].min()\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Old Subreddits</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Young Subreddits</b>\")]\n",
    "args = {\n",
    "    \"font\":{\"size\": 16},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Log Odds Ratio of Word Usage ({}, {})\".format(\"Old Subreddits\",\"Young Subreddits\"),\n",
    "    \"annotations\": annotations\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "# fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(\"old_sub\",\"young_subs\"))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merged = pd.merge(cand_perc_df[left_candidates].applymap(lambda x : to_odds(x)),age_perc_df[[\"young_sub\"]],left_index=True,right_index=True,how=\"inner\").fillna(0)\n",
    "melted = age_perc_df.reset_index()\n",
    "melted = melted.melt(value_vars=left_candidates,id_vars=[\"word\"],var_name=\"subreddit\",value_name=\"word_odds\").set_index(\"word\",drop=True)\n",
    "melted = pd.merge(age_perc_df,melted,left_index=True,right_index=True,how=\"right\").fillna(0)\n",
    "melted = melted[[\"subreddit\",\"word_odds\",\"young_sub\",\"old_sub\",\"total\"]]\n",
    "melted[[\"word_odds\",\"young_sub\",\"old_sub\",\"total\"]] = melted[[\"word_odds\",\"young_sub\",\"old_sub\",\"total\"]].applymap(lambda x : to_odds(x))\n",
    "melted[\"word_odds\"] = melted[[\"word_odds\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"young_sub\"] = melted[[\"young_sub\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"old_sub\"] = melted[[\"old_sub\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted = melted.reset_index()\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"young_sub\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    prog_r2 = r2_score(iso[\"young_sub\"], predict(iso[\"word_odds\"]))\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"old_sub\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    con_r2 = r2_score(iso[\"old_sub\"], predict(iso[\"word_odds\"]))\n",
    "    print(f\"{cand}:\\n\\tyoung alignment:{prog_r2}\\n\\tolder alignment:{con_r2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "fig = px.scatter(melted,\n",
    "                 x=\"young_sub\",\n",
    "                 y=\"word_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"young_sub\": \"Log Odds Ratio For Young Subreddits\",\n",
    "                     \"word_odds\": \"Log Odds Ratio For Candidate Subreddit\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "xmin,xmax = melted[\"young_sub\"].min(),melted[\"young_sub\"].max()\n",
    "x = np.linspace(xmin,xmax,num=2)\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"young_sub\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(iso[\"young_sub\"], predict(iso[\"word_odds\"]))\n",
    "    print(cand,r2)\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    mode='lines',\n",
    "                    line=dict(dash=\"dash\",width=2,color=mapping[cand]),\n",
    "                    name=f\"{cand} Linear Regression (r<sup>2</sup>={r2:.2f})\",\n",
    "                    legendgroup=cand,\n",
    "                   )\n",
    "    \n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Candidate Word Distribution Relative to Young Subreddits\",\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.write_html(\"visualizations/content_analysis/political_dimension_comparisons/age.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gun Control Subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anti_gcs, pro_gcs = [\"progun\"],[\"GunsAreCool\"]\n",
    "\n",
    "assign_sub = lambda x : \"pro_gc\" if x in pro_gcs else \"anti_gc\" if x in anti_gcs else x\n",
    "\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "gc_comments = comments.withColumn(\"sub_type\",udf_assign_sub(comments[\"subreddit\"]))\n",
    "gc_comments = gc_comments.drop('subreddit').withColumnRenamed('sub_type', 'subreddit')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/gc_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-gc_comment_counts.csv\"\n",
    "\n",
    "gc_count_df,gc_perc_df = count_vec(gc_comments,fp,normalized_fp,[\"pro_gc\",\"anti_gc\"],include_total=False)\n",
    "\n",
    "gc_perc_df = pd.concat([gc_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "gc_count_df = pd.concat([gc_count_df,total_comments], axis=1).fillna(0)\n",
    "gc_count_df = gc_count_df.loc[:,~gc_count_df.columns.duplicated()]\n",
    "gc_perc_df = gc_perc_df.loc[:,~gc_perc_df.columns.duplicated()]\n",
    "\n",
    "gc_perc_df.index.name,gc_count_df.index.name = \"word\",\"word\"\n",
    "gc_count_df[left_candidates],gc_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n",
    "gc_count_df.to_csv(fp),gc_perc_df.to_csv(normalized_fp)\n",
    "\n",
    "gc_perc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import randint\n",
    "\n",
    "isolated_gc = gc_perc_df[[\"anti_gc\",\"pro_gc\"]]\n",
    "isolated_gc = isolated_gc.applymap(lambda x : to_odds(x))\n",
    "\n",
    "isolated_gc[\"log_odds\"] = isolated_gc.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "# get absolute frequency of word usgc\n",
    "isolated_gc[\"use\"] = gc_count_df[\"anti_gc\"] + gc_count_df[\"pro_gc\"]\n",
    "isolated_gc = isolated_gc[np.isfinite(isolated_gc).all(1)]\n",
    "isolated_gc = isolated_gc.sort_values(\"log_odds\")\n",
    "# visualize\n",
    "fig = px.scatter(isolated_gc,\n",
    "                 x=\"use\",\n",
    "                 y=\"log_odds\",\n",
    "                 hover_data=[isolated_gc.index],\n",
    "                 labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                 opacity=0.5)\n",
    "# annotations\n",
    "head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                    y = row[\"log_odds\"],\n",
    "                    ay = randint(-20,20),\n",
    "                    ax = randint(-20,20),\n",
    "                    text = row.name) \n",
    "               for _,row in \n",
    "               isolated_gc.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "xmax = isolated_gc[\"use\"].max()*0.9\n",
    "ymax,ymin = isolated_gc[\"log_odds\"].max()*1.2, isolated_gc[\"log_odds\"].min()\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Anti Gun Control Subreddits</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Pro Gun Control Subreddits</b>\")]\n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Log Odds Ratio of Word Usage ({})\".format(\"Gun Control Subreddits\"),\n",
    "    \"annotations\": annotations\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(\"anti_gc\",\"pro_gc\"))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merged = pd.merge(cand_perc_df[left_candidates].applymap(lambda x : to_odds(x)),gc_perc_df[[\"pro_gc\"]],left_index=True,right_index=True,how=\"inner\").fillna(0)\n",
    "melted = gc_perc_df.reset_index()\n",
    "melted = melted.melt(value_vars=left_candidates,id_vars=[\"word\"],var_name=\"subreddit\",value_name=\"word_odds\").set_index(\"word\",drop=True)\n",
    "melted = pd.merge(gc_perc_df,melted,left_index=True,right_index=True,how=\"right\").fillna(0)\n",
    "melted = melted[[\"subreddit\",\"word_odds\",\"pro_gc\",\"anti_gc\",\"total\"]]\n",
    "melted[[\"word_odds\",\"pro_gc\",\"anti_gc\",\"total\"]] = melted[[\"word_odds\",\"pro_gc\",\"anti_gc\",\"total\"]].applymap(lambda x : to_odds(x))\n",
    "melted[\"word_odds\"] = melted[[\"word_odds\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"pro_gc\"] = melted[[\"pro_gc\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"anti_gc\"] = melted[[\"anti_gc\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted = melted.reset_index()\n",
    "\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"pro_gc\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    prog_r2 = r2_score(iso[\"pro_gc\"], predict(iso[\"word_odds\"]))\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"anti_gc\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    con_r2 = r2_score(iso[\"anti_gc\"], predict(iso[\"word_odds\"]))\n",
    "    print(f\"{cand}:\\n\\tpro gun control alignment:{prog_r2}\\n\\tanti_gc alignment:{con_r2}\")\n",
    "# melted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "fig = px.scatter(melted,\n",
    "                 x=\"pro_gc\",\n",
    "                 y=\"word_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"pro_gc\": \"Log Odds Ratio For Pro Gun Control Subreddits\",\n",
    "                     \"word_odds\": \"Log Odds Ratio For Candidate Subreddit\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "xmin,xmax = melted[\"pro_gc\"].min(),melted[\"pro_gc\"].max()\n",
    "x = np.linspace(xmin,xmax,num=2)\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"pro_gc\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(iso[\"pro_gc\"], predict(iso[\"word_odds\"]))\n",
    "    print(cand,r2)\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    mode='lines',\n",
    "                    line=dict(dash=\"dash\",width=2,color=mapping[cand]),\n",
    "                    name=f\"{cand} Linear Regression (r<sup>2</sup>={r2:.2f})\",\n",
    "                    legendgroup=cand,\n",
    "                   )\n",
    "    \n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Candidate Word Distribution Relative to Pro Gun Control Subreddits\",\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.write_html(\"visualizations/content_analysis/political_dimension_comparisons/gun_control.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### War Subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hawkish, pascifistic = [\"CredibleDefense\",\"WarCollege\",\"Intelligence\"], [\"EndlessWar\"]\n",
    "assign_sub = lambda x : \"hawkish\" if x in hawkish else \"pascifistic\" if x in pascifistic else x\n",
    "\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "\n",
    "war_comments = comments.withColumn(\"sub_type\",udf_assign_sub(comments[\"subreddit\"]))\n",
    "war_comments = war_comments.drop('subreddit').withColumnRenamed('sub_type', 'subreddit')\n",
    "# war_comments.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/war_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-war_comment_counts.csv\"\n",
    "\n",
    "war_count_df,war_perc_df = count_vec(war_comments,fp,normalized_fp,[\"hawkish\",\"pascifistic\"],include_total=False)\n",
    "\n",
    "war_perc_df = pd.concat([war_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "war_count_df = pd.concat([war_count_df,total_comments], axis=1).fillna(0)\n",
    "war_count_df = war_count_df.loc[:,~war_count_df.columns.duplicated()]\n",
    "war_perc_df = war_perc_df.loc[:,~war_perc_df.columns.duplicated()]\n",
    "\n",
    "war_perc_df.index.name,war_count_df.index.name = \"word\",\"word\"\n",
    "war_count_df[left_candidates],gc_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n",
    "war_count_df.to_csv(fp),war_perc_df.to_csv(normalized_fp)\n",
    "\n",
    "# war_perc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import randint\n",
    "\n",
    "isolated_war = war_perc_df[[\"hawkish\",\"pascifistic\"]]\n",
    "isolated_war = isolated_war.applymap(lambda x : to_odds(x))\n",
    "\n",
    "isolated_war[\"log_odds\"] = isolated_war.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "# get absolute frequency of word uswar\n",
    "isolated_war[\"use\"] = war_count_df[\"hawkish\"] + war_count_df[\"pascifistic\"]\n",
    "isolated_war = isolated_war[np.isfinite(isolated_war).all(1)]\n",
    "isolated_war = isolated_war.sort_values(\"log_odds\")\n",
    "# visualize\n",
    "fig = px.scatter(isolated_war,\n",
    "                 x=\"use\",\n",
    "                 y=\"log_odds\",\n",
    "                 hover_data=[isolated_war.index],\n",
    "                 labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                 opacity=0.5)\n",
    "# annotations\n",
    "head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                    y = row[\"log_odds\"],\n",
    "                    ay = randint(-20,20),\n",
    "                    ax = randint(-20,20),\n",
    "                    text = row.name) \n",
    "               for _,row in \n",
    "               isolated_war.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "xmax = isolated_war[\"use\"].max()*0.9\n",
    "ymax,ymin = isolated_war[\"log_odds\"].max()*1.2, isolated_war[\"log_odds\"].min()\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Hawkish Subreddits</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Pascifistic Subreddits</b>\")]\n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Log Odds Ratio of Word Usage ({}, {})\".format(\"Hawkish Subreddits\",\"Pascifistic Subreddits\"),\n",
    "    \"annotations\": annotations\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(\"hawkish\",\"pascifistic\"))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merged = pd.merge(cand_perc_df[left_candidates].applymap(lambda x : to_odds(x)),war_perc_df[[\"pascifistic\"]],left_index=True,right_index=True,how=\"inner\").fillna(0)\n",
    "melted = war_perc_df.reset_index()\n",
    "melted = melted.melt(value_vars=left_candidates,id_vars=[\"word\"],var_name=\"subreddit\",value_name=\"word_odds\").set_index(\"word\",drop=True)\n",
    "melted = pd.merge(war_perc_df,melted,left_index=True,right_index=True,how=\"right\").fillna(0)\n",
    "melted = melted[[\"subreddit\",\"word_odds\",\"pascifistic\",\"hawkish\",\"total\"]]\n",
    "melted[[\"word_odds\",\"pascifistic\",\"hawkish\",\"total\"]] = melted[[\"word_odds\",\"pascifistic\",\"hawkish\",\"total\"]].applymap(lambda x : to_odds(x))\n",
    "melted[\"word_odds\"] = melted[[\"word_odds\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"pascifistic\"] = melted[[\"pascifistic\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"hawkish\"] = melted[[\"hawkish\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted = melted.reset_index()\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"pascifistic\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    prog_r2 = r2_score(iso[\"pascifistic\"], predict(iso[\"word_odds\"]))\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"hawkish\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    con_r2 = r2_score(iso[\"hawkish\"], predict(iso[\"word_odds\"]))\n",
    "    print(f\"{cand}:\\n\\tpascifistic alignment:{prog_r2}\\n\\thawkish alignment:{con_r2}\")\n",
    "# melted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "fig = px.scatter(melted,\n",
    "                 x=\"pascifistic\",\n",
    "                 y=\"word_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"pascifistic\": \"Log Odds Ratio For Pro Gun Control Subreddits\",\n",
    "                     \"word_odds\": \"Log Odds Ratio For Candidate Subreddit\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "xmin,xmax = melted[\"pascifistic\"].min(),melted[\"pascifistic\"].max()\n",
    "x = np.linspace(xmin,xmax,num=2)\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"pascifistic\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(iso[\"pascifistic\"], predict(iso[\"word_odds\"]))\n",
    "    print(cand,r2)\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    mode='lines',\n",
    "                    line=dict(dash=\"dash\",width=2,color=mapping[cand]),\n",
    "                    name=f\"{cand} Linear Regression (r<sup>2</sup>={r2:.2f})\",\n",
    "                    legendgroup=cand,\n",
    "                   )\n",
    "    \n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Candidate Word Distribution Relative to Pascifistic Subreddits\",\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.write_html(\"visualizations/content_analysis/political_dimension_comparisons/war.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trade/Globalism Subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glob_subs = [(\"neoliberal\",\"LateStageImperialism\"),\n",
    "             (\"Economics\",\"capitalism_in_decay\"),\n",
    "             (\"Libertarian\",\"EnoughLibertarianSpam\")\n",
    "            ]\n",
    "\n",
    "anti_trade, pro_trade = [i for _,i in glob_subs],[i for i,_ in glob_subs]\n",
    "\n",
    "assign_sub = lambda x : \"pro_trade\" if x in pro_trade else \"anti_trade\" if x in anti_trade else x\n",
    "\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "# Drop comments that result in an empty string \"\" after cleaning. These are comments like \"ok\", \"who?\" etc. \n",
    "trade_comments = comments.withColumn(\"clean_body_tokenized\", udf_clean(comments[\"body\"]))\n",
    "trade_comments = trade_comments.withColumn(\"sub_type\",udf_assign_sub(trade_comments[\"subreddit\"]))\n",
    "\n",
    "# trade_comments.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/trade_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-trade_comment_counts.csv\"\n",
    "\n",
    "\n",
    "trade_count_df,trade_perc_df = count_vec(trade_comments,fp,normalized_fp,[\"anti_trade\", \"pro_trade\"],include_total=False)\n",
    "\n",
    "trade_perc_df = pd.concat([trade_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "trade_count_df = pd.concat([trade_count_df,total_comments], axis=1).fillna(0)\n",
    "trade_count_df = trade_count_df.loc[:,~trade_count_df.columns.duplicated()]\n",
    "trade_perc_df = trade_perc_df.loc[:,~trade_perc_df.columns.duplicated()]\n",
    "\n",
    "trade_perc_df.index.name,trade_count_df.index.name = \"word\",\"word\"\n",
    "trade_count_df[left_candidates],trade_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n",
    "trade_count_df.to_csv(fp),trade_perc_df.to_csv(normalized_fp)\n",
    "\n",
    "trade_perc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import randint\n",
    "\n",
    "isolated_trade = trade_perc_df[[\"anti_trade\", \"pro_trade\"]]\n",
    "isolated_trade = isolated_trade.applymap(lambda x : to_odds(x))\n",
    "\n",
    "isolated_trade[\"log_odds\"] = isolated_trade.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "# get absolute frequency of word ustrade\n",
    "isolated_trade[\"use\"] = trade_count_df[\"anti_trade\"] + trade_count_df[\"pro_trade\"]\n",
    "isolated_trade = isolated_trade[np.isfinite(isolated_trade).all(1)]\n",
    "isolated_trade = isolated_trade.sort_values(\"log_odds\")\n",
    "# visualize\n",
    "fig = px.scatter(isolated_trade,\n",
    "                 x=\"use\",\n",
    "                 y=\"log_odds\",\n",
    "                 hover_data=[isolated_trade.index],\n",
    "                 labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                 opacity=0.5)\n",
    "# annotations\n",
    "head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                    y = row[\"log_odds\"],\n",
    "                    ay = randint(-20,20),\n",
    "                    ax = randint(-20,20),\n",
    "                    text = row.name) \n",
    "               for _,row in \n",
    "               isolated_trade.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "xmax = isolated_trade[\"use\"].max()*0.9\n",
    "ymax,ymin = isolated_trade[\"log_odds\"].max()*1.2, isolated_trade[\"log_odds\"].min()\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Anti-Trade Subreddits</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Pro-Trade Subreddits</b>\")]\n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Log Odds Ratio of Word Usage ({}, {})\".format(\"Anti-Trade Subreddits\",\"Pro-Trade Subreddits\"),\n",
    "    \"annotations\": annotations\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(\"anti_trade\",\"pro_trade\"))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "melted = trade_perc_df.reset_index()\n",
    "melted = melted.melt(value_vars=left_candidates,id_vars=[\"word\"],var_name=\"subreddit\",value_name=\"word_odds\").set_index(\"word\",drop=True)\n",
    "melted = pd.merge(trade_perc_df,melted,left_index=True,right_index=True,how=\"right\").fillna(0)\n",
    "melted = melted[[\"subreddit\",\"word_odds\",\"anti_trade\",\"pro_trade\",\"total\"]]\n",
    "melted[[\"word_odds\",\"anti_trade\",\"pro_trade\",\"total\"]] = melted[[\"word_odds\",\"anti_trade\",\"pro_trade\",\"total\"]].applymap(lambda x : to_odds(x))\n",
    "melted[\"word_odds\"] = melted[[\"word_odds\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"anti_trade\"] = melted[[\"anti_trade\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"pro_trade\"] = melted[[\"pro_trade\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted = melted.reset_index()\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"anti_trade\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    prog_r2 = r2_score(iso[\"anti_trade\"], predict(iso[\"word_odds\"]))\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"pro_trade\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    con_r2 = r2_score(iso[\"pro_trade\"], predict(iso[\"word_odds\"]))\n",
    "    print(f\"{cand}:\\n\\tanti_trade alignment:{prog_r2}\\n\\tpro_tradealignment:{con_r2}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "fig = px.scatter(melted,\n",
    "                 x=\"anti_trade\",\n",
    "                 y=\"word_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"anti_trade\": \"Log Odds Ratio For Pro Gun Control Subreddits\",\n",
    "                     \"word_odds\": \"Log Odds Ratio For Candidate Subreddit\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "xmin,xmax = melted[\"anti_trade\"].min(),melted[\"anti_trade\"].max()\n",
    "x = np.linspace(xmin,xmax,num=2)\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"anti_trade\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(iso[\"anti_trade\"], predict(iso[\"word_odds\"]))\n",
    "    print(cand,r2)\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    mode='lines',\n",
    "                    line=dict(dash=\"dash\",width=2,color=mapping[cand]),\n",
    "                    name=f\"{cand} Linear Regression (r<sup>2</sup>={r2:.2f})\",\n",
    "                    legendgroup=cand,\n",
    "                   )\n",
    "    \n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Candidate Word Distribution Relative to Anti-Trade Subreddits\",\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.write_html(\"visualizations/content_analysis/political_dimension_comparisons/trade-glob.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Birth Control/Feminism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bc_subs = [(\"prolife\",\"prochoice\"),\n",
    "           (\"prolife\",\"birthcontrol\")]\n",
    "\n",
    "pro_bc, anti_bc = [i for _,i in bc_subs],[i for i,_ in bc_subs]\n",
    "\n",
    "assign_sub = lambda x : \"pro_bc\" if x in pro_bc else \"anti_bc\" if x in anti_bc else x\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "bc_comments = comments.withColumn(\"sub_type\",udf_assign_sub(comments[\"subreddit\"]))\n",
    "bc_comments = bc_comments.drop('subreddit').withColumnRenamed('sub_type', 'subreddit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/bc_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-bc_comment_counts.csv\"\n",
    "\n",
    "bc_count_df,bc_perc_df = count_vec(bc_comments,fp,normalized_fp,[\"anti_bc\", \"pro_bc\"],include_total=False)\n",
    "\n",
    "bc_perc_df = pd.concat([bc_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "bc_count_df = pd.concat([bc_count_df,total_comments], axis=1).fillna(0)\n",
    "bc_count_df = bc_count_df.loc[:,~bc_count_df.columns.duplicated()]\n",
    "bc_perc_df = bc_perc_df.loc[:,~bc_perc_df.columns.duplicated()]\n",
    "\n",
    "bc_perc_df.index.name, bc_count_df.index.name = \"word\",\"word\"\n",
    "bc_count_df[left_candidates],bc_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n",
    "bc_count_df.to_csv(fp),bc_perc_df.to_csv(normalized_fp)\n",
    "\n",
    "bc_perc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy.random import randint\n",
    "\n",
    "isolated_bc = bc_perc_df[[\"anti_bc\", \"pro_bc\"]]\n",
    "isolated_bc = isolated_bc.applymap(lambda x : to_odds(x))\n",
    "\n",
    "isolated_bc[\"log_odds\"] = isolated_bc.apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "# get absolute frequency of word usbc\n",
    "isolated_bc[\"use\"] = bc_count_df[\"anti_bc\"] + bc_count_df[\"pro_bc\"]\n",
    "isolated_bc = isolated_bc[np.isfinite(isolated_bc).all(1)]\n",
    "isolated_bc = isolated_bc.sort_values(\"log_odds\")\n",
    "# visualize\n",
    "fig = px.scatter(isolated_bc,\n",
    "                 x=\"use\",\n",
    "                 y=\"log_odds\",\n",
    "                 hover_data=[isolated_bc.index],\n",
    "                 labels={\"use\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                 opacity=0.5)\n",
    "# annotations\n",
    "head_tail_slice = list(range(7))+list(range(-7,0))\n",
    "annotations = [dict(x = np.log10(row[\"use\"]),\n",
    "                    y = row[\"log_odds\"],\n",
    "                    ay = randint(-20,20),\n",
    "                    ax = randint(-20,20),\n",
    "                    text = row.name) \n",
    "               for _,row in \n",
    "               isolated_bc.iloc[head_tail_slice][[\"log_odds\",\"use\"]].iterrows()]\n",
    "xmax = isolated_bc[\"use\"].max()*0.9\n",
    "ymax,ymin = isolated_bc[\"log_odds\"].max()*1.2, isolated_bc[\"log_odds\"].min()\n",
    "annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=\"<b>Anti Birth Control Subreddits</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=\"<b>Pro Birth Control Subreddits</b>\")]\n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Log Odds Ratio of Word Usage ({}, {})\".format(\"Anti Birth Control Subreddits\",\"Pro Birth Control Subreddits\"),\n",
    "    \"annotations\": annotations\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.update_xaxes(type=\"log\")\n",
    "fig.write_html(\"visualizations/content_analysis/{}_{}-log_odds_ratio.html\".format(\"anti_bc\",\"pro_bc\"))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "melted = bc_perc_df.reset_index()\n",
    "melted = melted.melt(value_vars=left_candidates,id_vars=[\"word\"],var_name=\"subreddit\",value_name=\"word_odds\").set_index(\"word\",drop=True)\n",
    "melted = pd.merge(bc_perc_df,melted,left_index=True,right_index=True,how=\"right\").fillna(0)\n",
    "melted = melted[[\"subreddit\",\"word_odds\",\"anti_bc\",\"pro_bc\",\"total\"]]\n",
    "melted[[\"word_odds\",\"anti_bc\",\"pro_bc\",\"total\"]] = melted[[\"word_odds\",\"anti_bc\",\"pro_bc\",\"total\"]].applymap(lambda x : to_odds(x))\n",
    "melted[\"word_odds\"] = melted[[\"word_odds\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"anti_bc\"] = melted[[\"anti_bc\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted[\"pro_bc\"] = melted[[\"pro_bc\",\"total\"]].apply(lambda x : log_odds_ratio(*x),axis=1)\n",
    "melted = melted.reset_index()\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"pro_bc\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    prog_r2 = r2_score(iso[\"pro_bc\"], predict(iso[\"word_odds\"]))\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"anti_bc\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    con_r2 = r2_score(iso[\"anti_bc\"], predict(iso[\"word_odds\"]))\n",
    "    print(f\"{cand}:\\n\\tapro_bc alignment:{prog_r2}\\n\\tanti_bc alignment:{con_r2}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "fig = px.scatter(melted,\n",
    "                 x=\"anti_bc\",\n",
    "                 y=\"word_odds\",\n",
    "                 color=\"subreddit\",\n",
    "                 hover_data=['word'],\n",
    "                 opacity=0.5,\n",
    "                 color_discrete_map=mapping,\n",
    "                 labels={\n",
    "                     \"anti_bc\": \"Log Odds Ratio For Pro Gun Control Subreddits\",\n",
    "                     \"word_odds\": \"Log Odds Ratio For Candidate Subreddit\",\n",
    "                     \"subreddit\": \"Subreddit\",\n",
    "                     \"word\": \"Word\"\n",
    "                 })\n",
    "xmin,xmax = melted[\"anti_bc\"].min(),melted[\"anti_bc\"].max()\n",
    "x = np.linspace(xmin,xmax,num=2)\n",
    "\n",
    "for cand in left_candidates:\n",
    "    iso = melted[melted[\"subreddit\"]==cand]\n",
    "    # Fit a linear regression to our model and its results\n",
    "    model = np.polyfit(iso[\"word_odds\"], iso[\"anti_bc\"], 1)\n",
    "    predict = np.poly1d(model)\n",
    "    r2 = r2_score(iso[\"anti_bc\"], predict(iso[\"word_odds\"]))\n",
    "    print(cand,r2)\n",
    "    fig.add_scatter(x=x,\n",
    "                    y=predict(x),\n",
    "                    mode='lines',\n",
    "                    line=dict(dash=\"dash\",width=2,color=mapping[cand]),\n",
    "                    name=f\"{cand} Linear Regression (r<sup>2</sup>={r2:.2f})\",\n",
    "                    legendgroup=cand,\n",
    "                   )\n",
    "    \n",
    "args = {\n",
    "    \"font\":{\"size\": 20},\n",
    "    \"template\": \"simple_white\",\n",
    "    \"title\": \"Candidate Word Distribution Relative to Anti Birth Control Subreddits\",\n",
    "}\n",
    "fig.update_layout(**args)\n",
    "fig.write_html(\"visualizations/content_analysis/political_dimension_comparisons/birth_control.html\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Religion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "religious_subs = [(\"Christianity\",\"exchristian\"),\n",
    "                  (\"TraditionalCatholics\",\"excatholic\"),\n",
    "                  (\"lds\",\"exmormon\"),(\"mormon\",\"exmormon\"),(\"islam\",\"exmuslim\")]\n",
    "\n",
    "pro_religious, anti_religious = [i for _,i in religious_subs],[i for i,_ in religious_subs]\n",
    "\n",
    "assign_sub = lambda x : \"pro_religious\" if x in pro_religious else \"anti_religious\" if x in anti_religious else x\n",
    "udf_assign_sub = udf(assign_sub,StringType())\n",
    "religious_comments = comments.withColumn(\"sub_type\",udf_assign_sub(comments[\"subreddit\"]))\n",
    "religious_comments = religious_comments.drop('subreddit').withColumnRenamed('sub_type', 'subreddit')\n",
    "religious_comments.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = \"dataframes/word_freq/religious_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-religious_comment_counts.csv\"\n",
    "\n",
    "religious_count_df\n",
    "\n",
    "religious_count_df,religious_perc_df = count_vec(religious_comments,fp,normalized_fp,[\"anti_religious\", \"pro_religious\"],include_total=False)\n",
    "\n",
    "religious_perc_df = pd.concat([religious_perc_df,total_comment_perc], axis=1).fillna(0)\n",
    "religious_count_df = pd.concat([religious_count_df,total_comments], axis=1).fillna(0)\n",
    "religious_count_df = religious_count_df.loc[:,~religious_count_df.columns.duplicated()]\n",
    "religious_perc_df = religious_perc_df.loc[:,~religious_perc_df.columns.duplicated()]\n",
    "\n",
    "religious_perc_df.index.name, religious_count_df.index.name = \"word\",\"word\"\n",
    "religious_count_df[left_candidates],religious_perc_df[left_candidates] = cand_count_df[left_candidates], cand_perc_df[left_candidates]\n",
    "religious_count_df.to_csv(fp),religious_perc_df.to_csv(normalized_fp)\n",
    "\n",
    "religious_perc_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Temporal Content Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_comments = comments.withColumn('subreddit', concat(lit('('),col('subreddit'),lit(','),col('year'),lit(')')))\n",
    "fp = \"dataframes/word_freq/temporal_comment_counts.csv\"\n",
    "normalized_fp = \"dataframes/word_freq/normalized-temporal_comment_counts.csv\"\n",
    "cols = np.array([[f\"({c},2019)\",f\"({c},2020)\"] for c in left_candidates]).flatten()\n",
    "\n",
    "temporal_count,temporal_prob = count_vec(temp_comments,fp,normalized_fp,cols,include_total=False)\n",
    "temporal_count,temporal_prob = temporal_count[cols],temporal_prob[cols]\n",
    "temporal_prob = pd.concat([temporal_prob,total_comment_perc], axis=1).fillna(0)\n",
    "temporal_count = pd.concat([temporal_count,total_comments], axis=1).fillna(0)\n",
    "temporal_count = temporal_count.loc[:,~temporal_count.columns.duplicated()]\n",
    "temporal_prob = temporal_prob.loc[:,~temporal_prob.columns.duplicated()]\n",
    "temporal_count.index.name, temporal_prob.index.name = \"word\",\"word\"\n",
    "# temporal_count.to_csv(fp),temporal_prob.to_csv(normalized_fp)\n",
    "temporal_count,temporal_prob = temporal_count[~temporal_count.index.isin(cand_names)],temporal_prob[~temporal_prob.index.isin(cand_names)]\n",
    "\n",
    "temporal_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "top_bottom = {}\n",
    "for cand in left_candidates:\n",
    "    cols = [f\"({cand},2019)\",f\"({cand},2020)\"]\n",
    "    iso_count,iso_prob = temporal_count[cols],temporal_prob[cols]\n",
    "    iso_prob.applymap(lambda x : to_odds(x))\n",
    "    iso_prob[\"log_odds\"] = iso_prob.apply(lambda x : log_odds_ratio(x[cols[1]],x[cols[0]]),axis=1)\n",
    "    iso_prob = iso_prob[np.isfinite(iso_prob).all(1)]\n",
    "    iso_prob[\"Total\"] = iso_count.sum(axis=1)\n",
    "    iso_prob = iso_prob.sort_values(\"log_odds\")\n",
    "    fig = px.scatter(iso_prob,\n",
    "                     x=\"Total\",\n",
    "                     y=\"log_odds\",\n",
    "                     hover_data=[iso_prob.index],\n",
    "                    labels={\"Total\":\"Word Frequency\",\n",
    "                         \"index\": \"Word\",\n",
    "                         \"log_odds\": \"Log Odds Ratio\"},\n",
    "                     opacity=0.5)\n",
    "    # annotations\n",
    "    head_tail_slice = list(range(3))\n",
    "    top_bottom[cand] = [row.name for _,row in iso_prob.iloc[head_tail_slice].iterrows()]\n",
    "    head_tail_slice = list(range(10))+list(range(-10,0))\n",
    "    print(iso_prob.iloc[head_tail_slice])\n",
    "    annotations = [dict(x = np.log10(row[\"Total\"]),\n",
    "                        y = row[\"log_odds\"],\n",
    "#                         ay = randint(-20,20),\n",
    "#                         ax = randint(-20,20),\n",
    "                        text = row.name) \n",
    "                   for _,row in \n",
    "                   iso_prob.iloc[head_tail_slice][[\"log_odds\",\"Total\"]].iterrows()]\n",
    "    xmax = iso_prob[\"Total\"].max()*0.9\n",
    "    ymax,ymin = iso_prob[\"log_odds\"].max()*1.2, iso_prob[\"log_odds\"].min()\n",
    "    annotations += [dict(x= np.log10(xmax),y=ymax,showarrow=False,text=f\"<b>2020</b>\"),dict(x= np.log10(xmax),y=ymin,showarrow=False,text=f\"<b>2019</b>\")]\n",
    "    args = {\n",
    "        \"font\":{\"size\": 16},\n",
    "        \"template\": \"simple_white\",\n",
    "        \"title\": f\"{cand} Log Odds Ratio of Word Usage (2019/2020)\",\n",
    "        \"annotations\": annotations\n",
    "    }\n",
    "    fig.update_layout(**args)\n",
    "    fig.update_xaxes(type=\"log\")\n",
    "#     fig.write_html(f\"visualizations/content_analysis/temporal/{cand}-SS-log_odds_ratio.html\")\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}