UF_combo_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from textacy import text_stats, make_spacy_doc\n",
    "from langdetect import detect, DetectorFactory\n",
    "from scipy.stats import pointbiserialr\n",
    "from scipy.stats import zscore, gmean\n",
    "from datasets import Dataset, load_dataset\n",
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import torch\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sample = pd.DataFrame(load_dataset(\"argilla/ultrafeedback-binarized-preferences-cleaned\")[\"train\"]).sample(n=15000, random_state=42)\n",
    "df_clean = df_sample.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>prompt</th>\n",
       "      <th>chosen</th>\n",
       "      <th>chosen-rating</th>\n",
       "      <th>chosen-model</th>\n",
       "      <th>rejected</th>\n",
       "      <th>rejected-rating</th>\n",
       "      <th>rejected-model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Topics: Wound management for general practitio...</td>\n",
       "      <td>[{'content': 'Topics: Wound management for gen...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>[{'content': 'Topics: Wound management for gen...</td>\n",
       "      <td>3.75</td>\n",
       "      <td>ultralm-13b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Part 1. Definition\\nIn this task, you are give...</td>\n",
       "      <td>[{'content': 'Part 1. Definition\n",
       "In this task,...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>[{'content': 'Part 1. Definition\n",
       "In this task,...</td>\n",
       "      <td>3.25</td>\n",
       "      <td>vicuna-33b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>You will act as an voice changer. You will cha...</td>\n",
       "      <td>[{'content': 'You will act as an voice changer...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>[{'content': 'You will act as an voice changer...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>ultralm-13b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Write a well-researched paper on the physiolog...</td>\n",
       "      <td>[{'content': 'Write a well-researched paper on...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>[{'content': 'Write a well-researched paper on...</td>\n",
       "      <td>3.00</td>\n",
       "      <td>ultralm-13b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Create a step-by-step recipe for making fluffy...</td>\n",
       "      <td>[{'content': 'Create a step-by-step recipe for...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>[{'content': 'Create a step-by-step recipe for...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>falcon-40b-instruct</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995</th>\n",
       "      <td>flan_v2_cot</td>\n",
       "      <td>[QUESTION] If \"Two kids stepping stones over t...</td>\n",
       "      <td>[{'content': '[QUESTION] If \"Two kids stepping...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>[{'content': '[QUESTION] If \"Two kids stepping...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14996</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Instructions: Given an object and a part, deci...</td>\n",
       "      <td>[{'content': 'Instructions: Given an object an...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>vicuna-33b</td>\n",
       "      <td>[{'content': 'Instructions: Given an object an...</td>\n",
       "      <td>3.50</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14997</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Write me a letter asking my friend Dan what he...</td>\n",
       "      <td>[{'content': 'Write me a letter asking my frie...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>[{'content': 'Write me a letter asking my frie...</td>\n",
       "      <td>1.25</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14998</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>You will be given a definition of a task first...</td>\n",
       "      <td>[{'content': 'You will be given a definition o...</td>\n",
       "      <td>2.00</td>\n",
       "      <td>llama-2-70b-chat</td>\n",
       "      <td>[{'content': 'You will be given a definition o...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>vicuna-33b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14999</th>\n",
       "      <td>false_qa</td>\n",
       "      <td>What's the resolution of a cat?</td>\n",
       "      <td>[{'content': 'What's the resolution of a cat?'...</td>\n",
       "      <td>4.25</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>[{'content': 'What's the resolution of a cat?'...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15000 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             source                                             prompt  \\\n",
       "0          sharegpt  Topics: Wound management for general practitio...   \n",
       "1      flan_v2_niv2  Part 1. Definition\\nIn this task, you are give...   \n",
       "2          sharegpt  You will act as an voice changer. You will cha...   \n",
       "3         ultrachat  Write a well-researched paper on the physiolog...   \n",
       "4         ultrachat  Create a step-by-step recipe for making fluffy...   \n",
       "...             ...                                                ...   \n",
       "14995   flan_v2_cot  [QUESTION] If \"Two kids stepping stones over t...   \n",
       "14996  flan_v2_niv2  Instructions: Given an object and a part, deci...   \n",
       "14997      sharegpt  Write me a letter asking my friend Dan what he...   \n",
       "14998  flan_v2_niv2  You will be given a definition of a task first...   \n",
       "14999      false_qa                    What's the resolution of a cat?   \n",
       "\n",
       "                                                  chosen  chosen-rating  \\\n",
       "0      [{'content': 'Topics: Wound management for gen...           5.00   \n",
       "1      [{'content': 'Part 1. Definition\n",
       "In this task,...           4.50   \n",
       "2      [{'content': 'You will act as an voice changer...           4.50   \n",
       "3      [{'content': 'Write a well-researched paper on...           4.75   \n",
       "4      [{'content': 'Create a step-by-step recipe for...           5.00   \n",
       "...                                                  ...            ...   \n",
       "14995  [{'content': '[QUESTION] If \"Two kids stepping...           4.75   \n",
       "14996  [{'content': 'Instructions: Given an object an...           4.50   \n",
       "14997  [{'content': 'Write me a letter asking my frie...           4.50   \n",
       "14998  [{'content': 'You will be given a definition o...           2.00   \n",
       "14999  [{'content': 'What's the resolution of a cat?'...           4.25   \n",
       "\n",
       "           chosen-model                                           rejected  \\\n",
       "0          wizardlm-70b  [{'content': 'Topics: Wound management for gen...   \n",
       "1          wizardlm-70b  [{'content': 'Part 1. Definition\n",
       "In this task,...   \n",
       "2          wizardlm-13b  [{'content': 'You will act as an voice changer...   \n",
       "3          wizardlm-13b  [{'content': 'Write a well-researched paper on...   \n",
       "4       llama-2-7b-chat  [{'content': 'Create a step-by-step recipe for...   \n",
       "...                 ...                                                ...   \n",
       "14995     gpt-3.5-turbo  [{'content': '[QUESTION] If \"Two kids stepping...   \n",
       "14996        vicuna-33b  [{'content': 'Instructions: Given an object an...   \n",
       "14997       ultralm-13b  [{'content': 'Write me a letter asking my frie...   \n",
       "14998  llama-2-70b-chat  [{'content': 'You will be given a definition o...   \n",
       "14999             gpt-4  [{'content': 'What's the resolution of a cat?'...   \n",
       "\n",
       "       rejected-rating       rejected-model  \n",
       "0                 3.75          ultralm-13b  \n",
       "1                 3.25           vicuna-33b  \n",
       "2                 1.75          ultralm-13b  \n",
       "3                 3.00          ultralm-13b  \n",
       "4                 4.50  falcon-40b-instruct  \n",
       "...                ...                  ...  \n",
       "14995             1.75      llama-2-7b-chat  \n",
       "14996             3.50      llama-2-7b-chat  \n",
       "14997             1.25      llama-2-7b-chat  \n",
       "14998             1.00           vicuna-33b  \n",
       "14999             4.00         wizardlm-70b  \n",
       "\n",
       "[15000 rows x 8 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean = df_clean.reset_index(drop=True)\n",
    "df_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_clean['chosen'] = df_clean['chosen'].apply(lambda x: x[1]['content'])\n",
    "df_clean['rejected'] = df_clean['rejected'].apply(lambda x: x[1]['content'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Need to bring in the docs to filter before organizing and running stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "topics = pd.read_csv(\"UF10k_mixedbread_topics.csv\")\n",
    "docs = pd.read_csv(\"UF_mixedbread_docs.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cleaning\n",
    "docs['topic_id'] = docs['topic_id'].str.extract(r'(\\d+)').astype(int)\n",
    "\n",
    "docs['x_cord'] = docs['x_cord'].str.extract(r'(\\d+.\\d+)').astype(float)\n",
    "docs['y_cord'] = docs['y_cord'].str.extract(r'(\\d+.\\d+)').astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merging to get all the data in one place\n",
    "docs_and_topics = docs.merge(topics.drop(columns=['doc_id', 'topic_id']), on='content')\n",
    "docs_and_topics = docs_and_topics.drop(columns=['term_id', 'doc_id']).rename(columns={'10': 'metadata', 'content': 'prompt'})\n",
    "df_clean = df_clean.merge(docs_and_topics, on='prompt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>prompt</th>\n",
       "      <th>chosen</th>\n",
       "      <th>chosen-rating</th>\n",
       "      <th>chosen-model</th>\n",
       "      <th>rejected</th>\n",
       "      <th>rejected-rating</th>\n",
       "      <th>rejected-model</th>\n",
       "      <th>x_cord</th>\n",
       "      <th>y_cord</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>metadata</th>\n",
       "      <th>topic_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Topics: Wound management for general practitio...</td>\n",
       "      <td>General practitioners play a vital role in the...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>Wound management is a crucial aspect of genera...</td>\n",
       "      <td>3.75</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>0.915843</td>\n",
       "      <td>0.482842</td>\n",
       "      <td>22</td>\n",
       "      <td>[0.20282909274101257, -0.05036671459674835, -0...</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Childhood Stress and Health</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Part 1. Definition\\nIn this task, you are give...</td>\n",
       "      <td>Step 1: Identify the language of the post. The...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>Non-hateful\\n\\nExplanation: This post seems to...</td>\n",
       "      <td>3.25</td>\n",
       "      <td>vicuna-33b</td>\n",
       "      <td>2.740021</td>\n",
       "      <td>4.862818</td>\n",
       "      <td>14</td>\n",
       "      <td>[0.06639007478952408, 0.23666216433048248, -0....</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_niv2', 'ratin...</td>\n",
       "      <td>Analyzing Product Sentiment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>You will act as an voice changer. You will cha...</td>\n",
       "      <td>An umbrella has been brought by her on the way.</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>Alright, just let me know which form of voice ...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>2.045614</td>\n",
       "      <td>4.058925</td>\n",
       "      <td>15</td>\n",
       "      <td>[-0.4921853840351105, -0.1649392992258072, -0....</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Language Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Write a well-researched paper on the physiolog...</td>\n",
       "      <td>Title: The Physiological and Psychological Imp...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>Title: The Physiological and Psychological Imp...</td>\n",
       "      <td>3.00</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>1.698340</td>\n",
       "      <td>0.061782</td>\n",
       "      <td>8</td>\n",
       "      <td>[-0.30388227105140686, 0.15756076574325562, 0....</td>\n",
       "      <td>('metadata', {'source': 'ultrachat', 'rating':...</td>\n",
       "      <td>Healthy Lifestyle Exploration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Create a step-by-step recipe for making fluffy...</td>\n",
       "      <td>Of course, I'd be happy to help you with that!...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>Here is a step-by-step recipe for making fluff...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>falcon-40b-instruct</td>\n",
       "      <td>3.702293</td>\n",
       "      <td>1.716949</td>\n",
       "      <td>6</td>\n",
       "      <td>[-0.05435372143983841, 0.28580591082572937, -0...</td>\n",
       "      <td>('metadata', {'source': 'ultrachat', 'rating':...</td>\n",
       "      <td>Culinary Creations</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Transform is an emerging brand that delivers m...</td>\n",
       "      <td>Welcome to Transform, your go-to source for in...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>Sure, here's a possible description for the \"A...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>llama-2-70b-chat</td>\n",
       "      <td>0.732869</td>\n",
       "      <td>1.959356</td>\n",
       "      <td>21</td>\n",
       "      <td>[-0.4771421253681183, -0.0352528840303421, -0....</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Marketing Strategy Platform</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10204</th>\n",
       "      <td>evol_instruct</td>\n",
       "      <td>Using Python and the scikit-learn library, how...</td>\n",
       "      <td>Detecting sarcasm in text is a challenging tas...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>Hello! As a helpful and respectful assistant, ...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>llama-2-13b-chat</td>\n",
       "      <td>2.532706</td>\n",
       "      <td>5.040441</td>\n",
       "      <td>14</td>\n",
       "      <td>[0.3401801884174347, 0.21816378831863403, 0.02...</td>\n",
       "      <td>('metadata', {'source': 'evol_instruct', 'rati...</td>\n",
       "      <td>Analyzing Product Sentiment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10205</th>\n",
       "      <td>flan_v2_cot</td>\n",
       "      <td>[QUESTION] If \"Two kids stepping stones over t...</td>\n",
       "      <td>Yes, the premise mentions pink plaid pants and...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>Confidence: 80%\\n\\nYou have provided several s...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>3.797116</td>\n",
       "      <td>0.511288</td>\n",
       "      <td>18</td>\n",
       "      <td>[0.39390242099761963, 0.10773289203643799, -0....</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_cot', 'rating...</td>\n",
       "      <td>Gender and Clothing Perceptions</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10206</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Write me a letter asking my friend Dan what he...</td>\n",
       "      <td>Dear Dan,\\n\\nWe hope this letter finds you wel...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>I cannot fulfill your request as it goes again...</td>\n",
       "      <td>1.25</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>0.873350</td>\n",
       "      <td>3.242940</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.7983337640762329, -0.37210243940353394, 0.5...</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Cultural Expression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10207</th>\n",
       "      <td>false_qa</td>\n",
       "      <td>What's the resolution of a cat?</td>\n",
       "      <td>As an AI, I assume you are asking about a cat'...</td>\n",
       "      <td>4.25</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>As an AI, I don't have the capability to deter...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>0.756816</td>\n",
       "      <td>1.959702</td>\n",
       "      <td>10</td>\n",
       "      <td>[0.114014632999897, -0.06832381337881088, 0.15...</td>\n",
       "      <td>('metadata', {'source': 'false_qa', 'rating': ...</td>\n",
       "      <td>Creative Writing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10208 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              source                                             prompt  \\\n",
       "0           sharegpt  Topics: Wound management for general practitio...   \n",
       "1       flan_v2_niv2  Part 1. Definition\\nIn this task, you are give...   \n",
       "2           sharegpt  You will act as an voice changer. You will cha...   \n",
       "3          ultrachat  Write a well-researched paper on the physiolog...   \n",
       "4          ultrachat  Create a step-by-step recipe for making fluffy...   \n",
       "...              ...                                                ...   \n",
       "10203       sharegpt  Transform is an emerging brand that delivers m...   \n",
       "10204  evol_instruct  Using Python and the scikit-learn library, how...   \n",
       "10205    flan_v2_cot  [QUESTION] If \"Two kids stepping stones over t...   \n",
       "10206       sharegpt  Write me a letter asking my friend Dan what he...   \n",
       "10207       false_qa                    What's the resolution of a cat?   \n",
       "\n",
       "                                                  chosen  chosen-rating  \\\n",
       "0      General practitioners play a vital role in the...           5.00   \n",
       "1      Step 1: Identify the language of the post. The...           4.50   \n",
       "2        An umbrella has been brought by her on the way.           4.50   \n",
       "3      Title: The Physiological and Psychological Imp...           4.75   \n",
       "4      Of course, I'd be happy to help you with that!...           5.00   \n",
       "...                                                  ...            ...   \n",
       "10203  Welcome to Transform, your go-to source for in...           4.75   \n",
       "10204  Detecting sarcasm in text is a challenging tas...           4.75   \n",
       "10205  Yes, the premise mentions pink plaid pants and...           4.75   \n",
       "10206  Dear Dan,\\n\\nWe hope this letter finds you wel...           4.50   \n",
       "10207  As an AI, I assume you are asking about a cat'...           4.25   \n",
       "\n",
       "          chosen-model                                           rejected  \\\n",
       "0         wizardlm-70b  Wound management is a crucial aspect of genera...   \n",
       "1         wizardlm-70b  Non-hateful\\n\\nExplanation: This post seems to...   \n",
       "2         wizardlm-13b  Alright, just let me know which form of voice ...   \n",
       "3         wizardlm-13b  Title: The Physiological and Psychological Imp...   \n",
       "4      llama-2-7b-chat  Here is a step-by-step recipe for making fluff...   \n",
       "...                ...                                                ...   \n",
       "10203    gpt-3.5-turbo  Sure, here's a possible description for the \"A...   \n",
       "10204     wizardlm-13b  Hello! As a helpful and respectful assistant, ...   \n",
       "10205    gpt-3.5-turbo  Confidence: 80%\\n\\nYou have provided several s...   \n",
       "10206      ultralm-13b  I cannot fulfill your request as it goes again...   \n",
       "10207            gpt-4  As an AI, I don't have the capability to deter...   \n",
       "\n",
       "       rejected-rating       rejected-model    x_cord    y_cord  topic_id  \\\n",
       "0                 3.75          ultralm-13b  0.915843  0.482842        22   \n",
       "1                 3.25           vicuna-33b  2.740021  4.862818        14   \n",
       "2                 1.75          ultralm-13b  2.045614  4.058925        15   \n",
       "3                 3.00          ultralm-13b  1.698340  0.061782         8   \n",
       "4                 4.50  falcon-40b-instruct  3.702293  1.716949         6   \n",
       "...                ...                  ...       ...       ...       ...   \n",
       "10203             4.00     llama-2-70b-chat  0.732869  1.959356        21   \n",
       "10204             4.00     llama-2-13b-chat  2.532706  5.040441        14   \n",
       "10205             1.75      llama-2-7b-chat  3.797116  0.511288        18   \n",
       "10206             1.25      llama-2-7b-chat  0.873350  3.242940         1   \n",
       "10207             4.00         wizardlm-70b  0.756816  1.959702        10   \n",
       "\n",
       "                                              embeddings  \\\n",
       "0      [0.20282909274101257, -0.05036671459674835, -0...   \n",
       "1      [0.06639007478952408, 0.23666216433048248, -0....   \n",
       "2      [-0.4921853840351105, -0.1649392992258072, -0....   \n",
       "3      [-0.30388227105140686, 0.15756076574325562, 0....   \n",
       "4      [-0.05435372143983841, 0.28580591082572937, -0...   \n",
       "...                                                  ...   \n",
       "10203  [-0.4771421253681183, -0.0352528840303421, -0....   \n",
       "10204  [0.3401801884174347, 0.21816378831863403, 0.02...   \n",
       "10205  [0.39390242099761963, 0.10773289203643799, -0....   \n",
       "10206  [0.7983337640762329, -0.37210243940353394, 0.5...   \n",
       "10207  [0.114014632999897, -0.06832381337881088, 0.15...   \n",
       "\n",
       "                                                metadata  \\\n",
       "0      ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "1      ('metadata', {'source': 'flan_v2_niv2', 'ratin...   \n",
       "2      ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "3      ('metadata', {'source': 'ultrachat', 'rating':...   \n",
       "4      ('metadata', {'source': 'ultrachat', 'rating':...   \n",
       "...                                                  ...   \n",
       "10203  ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "10204  ('metadata', {'source': 'evol_instruct', 'rati...   \n",
       "10205  ('metadata', {'source': 'flan_v2_cot', 'rating...   \n",
       "10206  ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "10207  ('metadata', {'source': 'false_qa', 'rating': ...   \n",
       "\n",
       "                            topic_name  \n",
       "0          Childhood Stress and Health  \n",
       "1          Analyzing Product Sentiment  \n",
       "2                    Language Analysis  \n",
       "3        Healthy Lifestyle Exploration  \n",
       "4                   Culinary Creations  \n",
       "...                                ...  \n",
       "10203      Marketing Strategy Platform  \n",
       "10204      Analyzing Product Sentiment  \n",
       "10205  Gender and Clothing Perceptions  \n",
       "10206              Cultural Expression  \n",
       "10207                 Creative Writing  \n",
       "\n",
       "[10208 rows x 14 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_clean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now we need put all the responses together, regradless of whether they are rejected or accepted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reject = df_clean.iloc[:, [0 , 1, 5, 6, 7]]\n",
    "df_accept = df_clean.iloc[:, [0, 1, 2, 3, 4]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_reject = df_reject.rename(columns={'rejected' : 'response', 'rejected-rating' : 'rating', 'rejected-model' : 'model'})\n",
    "df_reject['chosen'] = 0\n",
    "df_accept = df_accept.rename(columns={'chosen' : 'response', 'chosen-rating': 'rating', 'chosen-model' : 'model'})\n",
    "df_accept['chosen'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>prompt</th>\n",
       "      <th>response</th>\n",
       "      <th>rating</th>\n",
       "      <th>model</th>\n",
       "      <th>chosen</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Topics: Wound management for general practitio...</td>\n",
       "      <td>General practitioners play a vital role in the...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Part 1. Definition\\nIn this task, you are give...</td>\n",
       "      <td>Step 1: Identify the language of the post. The...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>You will act as an voice changer. You will cha...</td>\n",
       "      <td>An umbrella has been brought by her on the way.</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Write a well-researched paper on the physiolog...</td>\n",
       "      <td>Title: The Physiological and Psychological Imp...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Create a step-by-step recipe for making fluffy...</td>\n",
       "      <td>Of course, I'd be happy to help you with that!...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10203</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Transform is an emerging brand that delivers m...</td>\n",
       "      <td>Sure, here's a possible description for the \"A...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>llama-2-70b-chat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10204</th>\n",
       "      <td>evol_instruct</td>\n",
       "      <td>Using Python and the scikit-learn library, how...</td>\n",
       "      <td>Hello! As a helpful and respectful assistant, ...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>llama-2-13b-chat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10205</th>\n",
       "      <td>flan_v2_cot</td>\n",
       "      <td>[QUESTION] If \"Two kids stepping stones over t...</td>\n",
       "      <td>Confidence: 80%\\n\\nYou have provided several s...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10206</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Write me a letter asking my friend Dan what he...</td>\n",
       "      <td>I cannot fulfill your request as it goes again...</td>\n",
       "      <td>1.25</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10207</th>\n",
       "      <td>false_qa</td>\n",
       "      <td>What's the resolution of a cat?</td>\n",
       "      <td>As an AI, I don't have the capability to deter...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20416 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              source                                             prompt  \\\n",
       "0           sharegpt  Topics: Wound management for general practitio...   \n",
       "1       flan_v2_niv2  Part 1. Definition\\nIn this task, you are give...   \n",
       "2           sharegpt  You will act as an voice changer. You will cha...   \n",
       "3          ultrachat  Write a well-researched paper on the physiolog...   \n",
       "4          ultrachat  Create a step-by-step recipe for making fluffy...   \n",
       "...              ...                                                ...   \n",
       "10203       sharegpt  Transform is an emerging brand that delivers m...   \n",
       "10204  evol_instruct  Using Python and the scikit-learn library, how...   \n",
       "10205    flan_v2_cot  [QUESTION] If \"Two kids stepping stones over t...   \n",
       "10206       sharegpt  Write me a letter asking my friend Dan what he...   \n",
       "10207       false_qa                    What's the resolution of a cat?   \n",
       "\n",
       "                                                response  rating  \\\n",
       "0      General practitioners play a vital role in the...    5.00   \n",
       "1      Step 1: Identify the language of the post. The...    4.50   \n",
       "2        An umbrella has been brought by her on the way.    4.50   \n",
       "3      Title: The Physiological and Psychological Imp...    4.75   \n",
       "4      Of course, I'd be happy to help you with that!...    5.00   \n",
       "...                                                  ...     ...   \n",
       "10203  Sure, here's a possible description for the \"A...    4.00   \n",
       "10204  Hello! As a helpful and respectful assistant, ...    4.00   \n",
       "10205  Confidence: 80%\\n\\nYou have provided several s...    1.75   \n",
       "10206  I cannot fulfill your request as it goes again...    1.25   \n",
       "10207  As an AI, I don't have the capability to deter...    4.00   \n",
       "\n",
       "                  model  chosen  \n",
       "0          wizardlm-70b       1  \n",
       "1          wizardlm-70b       1  \n",
       "2          wizardlm-13b       1  \n",
       "3          wizardlm-13b       1  \n",
       "4       llama-2-7b-chat       1  \n",
       "...                 ...     ...  \n",
       "10203  llama-2-70b-chat       0  \n",
       "10204  llama-2-13b-chat       0  \n",
       "10205   llama-2-7b-chat       0  \n",
       "10206   llama-2-7b-chat       0  \n",
       "10207      wizardlm-70b       0  \n",
       "\n",
       "[20416 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frames = [df_accept, df_reject]\n",
    "\n",
    "#put them all together\n",
    "\n",
    "df_all = pd.concat(frames)\n",
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Putting metadata back in:\n",
    "df_all = df_all.merge(df_clean.loc[:, ['prompt', 'x_cord', 'y_cord','topic_id', 'embeddings', 'metadata', 'topic_name']], on='prompt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mix them up \n",
    "df_all = df_all.sample(frac=1, random_state=42).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets Run a langdetect loop to see if anything is still not in english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Since we have to keep the pairs together, we need to index by prompt\n",
    "DetectorFactory.seed = 0\n",
    "\n",
    "non_english_prompts = []\n",
    "non_parasable_prompts = []\n",
    "\n",
    "for index, row in df_all.iterrows():\n",
    "    try:\n",
    "        if detect(row['response']) != 'en':\n",
    "            non_english_prompts.append(row['prompt'])\n",
    "    except:\n",
    "        non_parasable_prompts.append(row['prompt'])\n",
    "\n",
    "unusable_prompts = non_english_prompts + non_parasable_prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = df_all[~df_all['prompt'].isin(unusable_prompts)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for some reason these rows don't work, idk why. Your dataset might not run in to this issue\n",
    "# bads = [5230, 19157]\n",
    "\n",
    "# df_all = df_all.drop(bads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting statistics form each of the responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get text statisitsics for each response \n",
    "\n",
    "# NB! This takes forever to run so try to run it only once\n",
    "\n",
    "for i, row in df_all.iterrows():\n",
    "    #Make each into a SpaCy doc\n",
    "\n",
    "    response = make_spacy_doc(row['response'], lang='en_core_web_sm')\n",
    "    \n",
    "    #Basics\n",
    "\n",
    "    df_all.loc[i, 'n_sents'] = text_stats.basics.n_sents(response)\n",
    "\n",
    "    df_all.loc[i, 'n_words'] = text_stats.basics.n_words(response)\n",
    "\n",
    "    df_all.loc[i, 'n_uniquewords'] = text_stats.basics.n_unique_words(response)\n",
    "\n",
    "    df_all.loc[i, 'n_chars'] = text_stats.basics.n_chars(response)\n",
    "\n",
    "    df_all.loc[i, 'n_longwords'] = text_stats.basics.n_long_words(response)\n",
    "\n",
    "    df_all.loc[i, 'n_sylsprword'] = text_stats.basics.n_syllables(response)\n",
    "\n",
    "    df_all.loc[i, 'n_monosylwords'] = text_stats.basics.n_monosyllable_words(response)\n",
    "\n",
    "    df_all.loc[i, 'n_polysylwords'] = text_stats.basics.n_polysyllable_words(response)\n",
    "\n",
    "    df_all.loc[i, 'entropy'] = text_stats.basics.entropy(response)\n",
    "\n",
    "    #Lexical Diversity\n",
    "\n",
    "    df_all.loc[i, 'ttr'] = text_stats.diversity.ttr(response)\n",
    "\n",
    "    df_all.loc[i, 'log_ttr'] = text_stats.diversity.log_ttr(response)\n",
    "\n",
    "    df_all.loc[i, 'seg_ttr'] = text_stats.diversity.segmented_ttr(response)\n",
    "\n",
    "    df_all.loc[i, 'mtld'] = text_stats.diversity.mtld(response)\n",
    "\n",
    "    df_all.loc[i, 'hdd'] = text_stats.diversity.hdd(response)\n",
    "    \n",
    "    #Readability\n",
    "\n",
    "    df_all.loc[i, 'flesch_score'] = text_stats.readability.flesch_reading_ease(response) \n",
    "\n",
    "    df_all.loc[i, 'flesch_grade'] = text_stats.readability.flesch_kincaid_grade_level(response)\n",
    "\n",
    "    df_all.loc[i, 'gunning_fog'] = text_stats.readability.gunning_fog_index(response) \n",
    "\n",
    "    df_all.loc[i, 'coleman_liau'] = text_stats.readability.coleman_liau_index(response) \n",
    "\n",
    "    df_all.loc[i, 'automated_readability'] = text_stats.readability.automated_readability_index(response)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_checkpoint  = df_all.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_all = df_checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reordering for easier indexing\n",
    "cols = df_all.columns.tolist()\n",
    "cols = ['source',\n",
    " 'prompt',\n",
    " 'response',\n",
    " 'model',\n",
    " 'chosen',\n",
    " 'rating',\n",
    " 'n_sents',\n",
    " 'n_words',\n",
    " 'n_uniquewords',\n",
    " 'n_chars',\n",
    " 'n_longwords',\n",
    " 'n_sylsprword',\n",
    " 'n_monosylwords',\n",
    " 'n_polysylwords',\n",
    " 'entropy',\n",
    " 'ttr',\n",
    " 'log_ttr',\n",
    " 'seg_ttr',\n",
    " 'mtld',\n",
    " 'hdd',\n",
    " 'flesch_score',\n",
    " 'flesch_grade',\n",
    " 'gunning_fog',\n",
    " 'coleman_liau',\n",
    " 'automated_readability',\n",
    " 'x_cord',\n",
    " 'y_cord',\n",
    " 'topic_id',\n",
    " 'embeddings',\n",
    " 'metadata',\n",
    " 'topic_name']\n",
    "\n",
    "df_all = df_all[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>prompt</th>\n",
       "      <th>response</th>\n",
       "      <th>model</th>\n",
       "      <th>chosen</th>\n",
       "      <th>rating</th>\n",
       "      <th>n_sents</th>\n",
       "      <th>n_words</th>\n",
       "      <th>n_uniquewords</th>\n",
       "      <th>n_chars</th>\n",
       "      <th>...</th>\n",
       "      <th>flesch_grade</th>\n",
       "      <th>gunning_fog</th>\n",
       "      <th>coleman_liau</th>\n",
       "      <th>automated_readability</th>\n",
       "      <th>x_cord</th>\n",
       "      <th>y_cord</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>metadata</th>\n",
       "      <th>topic_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>evol_instruct</td>\n",
       "      <td>Which word does not belong in the following gr...</td>\n",
       "      <td>Kettle does not belong in the group as it is a...</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>0</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.848000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>5.594333</td>\n",
       "      <td>9.156400</td>\n",
       "      <td>2.294071</td>\n",
       "      <td>3.214199</td>\n",
       "      <td>15</td>\n",
       "      <td>[-0.6554288864135742, -0.4138864278793335, 0.5...</td>\n",
       "      <td>('metadata', {'source': 'evol_instruct', 'rati...</td>\n",
       "      <td>Language Analysis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>In what ways has technology changed the way in...</td>\n",
       "      <td>I do not have personal views or opinions. howe...</td>\n",
       "      <td>ultralm-65b</td>\n",
       "      <td>0</td>\n",
       "      <td>4.25</td>\n",
       "      <td>13.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>1338.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10.358378</td>\n",
       "      <td>13.529076</td>\n",
       "      <td>13.089567</td>\n",
       "      <td>12.863508</td>\n",
       "      <td>2.254160</td>\n",
       "      <td>2.922242</td>\n",
       "      <td>1</td>\n",
       "      <td>[-0.4531969428062439, 0.3193971812725067, 0.02...</td>\n",
       "      <td>('metadata', {'source': 'ultrachat', 'rating':...</td>\n",
       "      <td>Cultural Expression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Detailed Instructions: In this task you are gi...</td>\n",
       "      <td>Pangaea and Gondwana were two supercontinents ...</td>\n",
       "      <td>llama-2-70b-chat</td>\n",
       "      <td>0</td>\n",
       "      <td>3.25</td>\n",
       "      <td>5.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>517.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10.345072</td>\n",
       "      <td>13.120825</td>\n",
       "      <td>14.013078</td>\n",
       "      <td>13.373814</td>\n",
       "      <td>0.154498</td>\n",
       "      <td>1.925598</td>\n",
       "      <td>7</td>\n",
       "      <td>[1.273107647895813, 0.08621750771999359, -0.16...</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_niv2', 'ratin...</td>\n",
       "      <td>Interconnected Systems</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>What are the best snorkeling spots in Hawaii?</td>\n",
       "      <td>Hawaii is famous for its breathtaking underwat...</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>1</td>\n",
       "      <td>5.00</td>\n",
       "      <td>25.0</td>\n",
       "      <td>403.0</td>\n",
       "      <td>183.0</td>\n",
       "      <td>2034.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.235758</td>\n",
       "      <td>10.616734</td>\n",
       "      <td>12.040226</td>\n",
       "      <td>10.402060</td>\n",
       "      <td>2.054862</td>\n",
       "      <td>1.508557</td>\n",
       "      <td>16</td>\n",
       "      <td>[-0.35886409878730774, 1.3211461305618286, -0....</td>\n",
       "      <td>('metadata', {'source': 'ultrachat', 'rating':...</td>\n",
       "      <td>Urban Exploration</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>false_qa</td>\n",
       "      <td>Why do anaerobic bacteria prefer an environmen...</td>\n",
       "      <td>Anaerobic bacteria do not prefer an environmen...</td>\n",
       "      <td>mpt-30b-chat</td>\n",
       "      <td>1</td>\n",
       "      <td>4.50</td>\n",
       "      <td>4.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10.634019</td>\n",
       "      <td>14.482278</td>\n",
       "      <td>11.430504</td>\n",
       "      <td>11.458418</td>\n",
       "      <td>0.146003</td>\n",
       "      <td>0.876940</td>\n",
       "      <td>7</td>\n",
       "      <td>[0.5538246631622314, 0.7116502523422241, -0.06...</td>\n",
       "      <td>('metadata', {'source': 'false_qa', 'rating': ...</td>\n",
       "      <td>Interconnected Systems</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20410</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Detailed Instructions: In this task, you are g...</td>\n",
       "      <td>user\\n\\nExplanation: The user is asking for mo...</td>\n",
       "      <td>mpt-30b-chat</td>\n",
       "      <td>1</td>\n",
       "      <td>4.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>207.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.084444</td>\n",
       "      <td>12.577778</td>\n",
       "      <td>15.542733</td>\n",
       "      <td>11.652500</td>\n",
       "      <td>0.720507</td>\n",
       "      <td>2.482977</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.3781099021434784, 0.03334461525082588, -0.3...</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_niv2', 'ratin...</td>\n",
       "      <td>Cultural Expression</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20412</th>\n",
       "      <td>evol_instruct</td>\n",
       "      <td>How can I modify the existing C# code to also ...</td>\n",
       "      <td>To modify the existing code to calculate the e...</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>0</td>\n",
       "      <td>4.00</td>\n",
       "      <td>14.0</td>\n",
       "      <td>390.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>2122.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15.183004</td>\n",
       "      <td>19.142857</td>\n",
       "      <td>15.129509</td>\n",
       "      <td>18.125802</td>\n",
       "      <td>2.395390</td>\n",
       "      <td>0.070725</td>\n",
       "      <td>19</td>\n",
       "      <td>[-0.05869555473327637, 0.5773212313652039, -0....</td>\n",
       "      <td>('metadata', {'source': 'evol_instruct', 'rati...</td>\n",
       "      <td>Gaming Economics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20413</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Find the movie name from the given conversatio...</td>\n",
       "      <td>The Lion King</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>1</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.313333</td>\n",
       "      <td>1.200000</td>\n",
       "      <td>-4.103777</td>\n",
       "      <td>-2.660000</td>\n",
       "      <td>0.030586</td>\n",
       "      <td>3.049329</td>\n",
       "      <td>5</td>\n",
       "      <td>[0.7883694767951965, -0.3784785866737366, 0.16...</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_niv2', 'ratin...</td>\n",
       "      <td>Movie Analysis and Interpretation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20414</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Create a slack conversation between 4 people, ...</td>\n",
       "      <td>Thread 1: 🎯 Sales Strategies\\n\\nMonika: Hey gu...</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>1</td>\n",
       "      <td>5.00</td>\n",
       "      <td>29.0</td>\n",
       "      <td>361.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>1705.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6.000562</td>\n",
       "      <td>9.300640</td>\n",
       "      <td>9.592866</td>\n",
       "      <td>7.039429</td>\n",
       "      <td>0.281950</td>\n",
       "      <td>2.280695</td>\n",
       "      <td>21</td>\n",
       "      <td>[1.0214362144470215, -0.0009904910111799836, -...</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Marketing Strategy Platform</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20415</th>\n",
       "      <td>false_qa</td>\n",
       "      <td>What kind of hyraxes live on Arctic?</td>\n",
       "      <td>Hyraxes, which are also known as rock hyraxes ...</td>\n",
       "      <td>starchat</td>\n",
       "      <td>0</td>\n",
       "      <td>2.75</td>\n",
       "      <td>3.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>399.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.745455</td>\n",
       "      <td>16.278788</td>\n",
       "      <td>9.850318</td>\n",
       "      <td>14.592235</td>\n",
       "      <td>0.346545</td>\n",
       "      <td>1.661371</td>\n",
       "      <td>7</td>\n",
       "      <td>[0.6974462270736694, 1.1065282821655273, -0.52...</td>\n",
       "      <td>('metadata', {'source': 'false_qa', 'rating': ...</td>\n",
       "      <td>Interconnected Systems</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18672 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              source                                             prompt  \\\n",
       "0      evol_instruct  Which word does not belong in the following gr...   \n",
       "1          ultrachat  In what ways has technology changed the way in...   \n",
       "2       flan_v2_niv2  Detailed Instructions: In this task you are gi...   \n",
       "3          ultrachat      What are the best snorkeling spots in Hawaii?   \n",
       "4           false_qa  Why do anaerobic bacteria prefer an environmen...   \n",
       "...              ...                                                ...   \n",
       "20410   flan_v2_niv2  Detailed Instructions: In this task, you are g...   \n",
       "20412  evol_instruct  How can I modify the existing C# code to also ...   \n",
       "20413   flan_v2_niv2  Find the movie name from the given conversatio...   \n",
       "20414       sharegpt  Create a slack conversation between 4 people, ...   \n",
       "20415       false_qa               What kind of hyraxes live on Arctic?   \n",
       "\n",
       "                                                response             model  \\\n",
       "0      Kettle does not belong in the group as it is a...      wizardlm-70b   \n",
       "1      I do not have personal views or opinions. howe...       ultralm-65b   \n",
       "2      Pangaea and Gondwana were two supercontinents ...  llama-2-70b-chat   \n",
       "3      Hawaii is famous for its breathtaking underwat...             gpt-4   \n",
       "4      Anaerobic bacteria do not prefer an environmen...      mpt-30b-chat   \n",
       "...                                                  ...               ...   \n",
       "20410  user\\n\\nExplanation: The user is asking for mo...      mpt-30b-chat   \n",
       "20412  To modify the existing code to calculate the e...      wizardlm-70b   \n",
       "20413                                      The Lion King             gpt-4   \n",
       "20414  Thread 1: 🎯 Sales Strategies\\n\\nMonika: Hey gu...      wizardlm-13b   \n",
       "20415  Hyraxes, which are also known as rock hyraxes ...          starchat   \n",
       "\n",
       "       chosen  rating  n_sents  n_words  n_uniquewords  n_chars  ...  \\\n",
       "0           0    4.50      1.0     25.0           24.0     96.0  ...   \n",
       "1           0    4.25     13.0    259.0          141.0   1338.0  ...   \n",
       "2           0    3.25      5.0     97.0           56.0    517.0  ...   \n",
       "3           1    5.00     25.0    403.0          183.0   2034.0  ...   \n",
       "4           1    4.50      4.0     79.0           52.0    386.0  ...   \n",
       "...       ...     ...      ...      ...            ...      ...  ...   \n",
       "20410       1    4.50      3.0     36.0           29.0    207.0  ...   \n",
       "20412       0    4.00     14.0    390.0          150.0   2122.0  ...   \n",
       "20413       1    4.50      1.0      3.0            3.0     11.0  ...   \n",
       "20414       1    5.00     29.0    361.0          201.0   1705.0  ...   \n",
       "20415       0    2.75      3.0     88.0           62.0    399.0  ...   \n",
       "\n",
       "       flesch_grade  gunning_fog  coleman_liau  automated_readability  \\\n",
       "0          7.848000    10.000000      5.594333               9.156400   \n",
       "1         10.358378    13.529076     13.089567              12.863508   \n",
       "2         10.345072    13.120825     14.013078              13.373814   \n",
       "3          8.235758    10.616734     12.040226              10.402060   \n",
       "4         10.634019    14.482278     11.430504              11.458418   \n",
       "...             ...          ...           ...                    ...   \n",
       "20410      9.084444    12.577778     15.542733              11.652500   \n",
       "20412     15.183004    19.142857     15.129509              18.125802   \n",
       "20413      1.313333     1.200000     -4.103777              -2.660000   \n",
       "20414      6.000562     9.300640      9.592866               7.039429   \n",
       "20415     12.745455    16.278788      9.850318              14.592235   \n",
       "\n",
       "         x_cord    y_cord  topic_id  \\\n",
       "0      2.294071  3.214199        15   \n",
       "1      2.254160  2.922242         1   \n",
       "2      0.154498  1.925598         7   \n",
       "3      2.054862  1.508557        16   \n",
       "4      0.146003  0.876940         7   \n",
       "...         ...       ...       ...   \n",
       "20410  0.720507  2.482977         1   \n",
       "20412  2.395390  0.070725        19   \n",
       "20413  0.030586  3.049329         5   \n",
       "20414  0.281950  2.280695        21   \n",
       "20415  0.346545  1.661371         7   \n",
       "\n",
       "                                              embeddings  \\\n",
       "0      [-0.6554288864135742, -0.4138864278793335, 0.5...   \n",
       "1      [-0.4531969428062439, 0.3193971812725067, 0.02...   \n",
       "2      [1.273107647895813, 0.08621750771999359, -0.16...   \n",
       "3      [-0.35886409878730774, 1.3211461305618286, -0....   \n",
       "4      [0.5538246631622314, 0.7116502523422241, -0.06...   \n",
       "...                                                  ...   \n",
       "20410  [0.3781099021434784, 0.03334461525082588, -0.3...   \n",
       "20412  [-0.05869555473327637, 0.5773212313652039, -0....   \n",
       "20413  [0.7883694767951965, -0.3784785866737366, 0.16...   \n",
       "20414  [1.0214362144470215, -0.0009904910111799836, -...   \n",
       "20415  [0.6974462270736694, 1.1065282821655273, -0.52...   \n",
       "\n",
       "                                                metadata  \\\n",
       "0      ('metadata', {'source': 'evol_instruct', 'rati...   \n",
       "1      ('metadata', {'source': 'ultrachat', 'rating':...   \n",
       "2      ('metadata', {'source': 'flan_v2_niv2', 'ratin...   \n",
       "3      ('metadata', {'source': 'ultrachat', 'rating':...   \n",
       "4      ('metadata', {'source': 'false_qa', 'rating': ...   \n",
       "...                                                  ...   \n",
       "20410  ('metadata', {'source': 'flan_v2_niv2', 'ratin...   \n",
       "20412  ('metadata', {'source': 'evol_instruct', 'rati...   \n",
       "20413  ('metadata', {'source': 'flan_v2_niv2', 'ratin...   \n",
       "20414  ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "20415  ('metadata', {'source': 'false_qa', 'rating': ...   \n",
       "\n",
       "                              topic_name  \n",
       "0                      Language Analysis  \n",
       "1                    Cultural Expression  \n",
       "2                 Interconnected Systems  \n",
       "3                      Urban Exploration  \n",
       "4                 Interconnected Systems  \n",
       "...                                  ...  \n",
       "20410                Cultural Expression  \n",
       "20412                   Gaming Economics  \n",
       "20413  Movie Analysis and Interpretation  \n",
       "20414        Marketing Strategy Platform  \n",
       "20415             Interconnected Systems  \n",
       "\n",
       "[18672 rows x 31 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_all.to_csv(\"Accepted_Rejected_with_scores.csv\")\n",
    "\n",
    "df_all = pd.read_csv(\"Accepted_Rejected_with_scores_bkup.csv\", index_col = 0)\n",
    "\n",
    "df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats_only = df_all.iloc[:, 4:-6]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indicators of high score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chosen</th>\n",
       "      <th>rating</th>\n",
       "      <th>n_sents</th>\n",
       "      <th>n_words</th>\n",
       "      <th>n_uniquewords</th>\n",
       "      <th>n_chars</th>\n",
       "      <th>n_longwords</th>\n",
       "      <th>n_sylsprword</th>\n",
       "      <th>n_monosylwords</th>\n",
       "      <th>n_polysylwords</th>\n",
       "      <th>...</th>\n",
       "      <th>ttr</th>\n",
       "      <th>log_ttr</th>\n",
       "      <th>seg_ttr</th>\n",
       "      <th>mtld</th>\n",
       "      <th>hdd</th>\n",
       "      <th>flesch_score</th>\n",
       "      <th>flesch_grade</th>\n",
       "      <th>gunning_fog</th>\n",
       "      <th>coleman_liau</th>\n",
       "      <th>automated_readability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.987318</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>1.749988e+02</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>83.324000</td>\n",
       "      <td>7.848000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>5.594333</td>\n",
       "      <td>9.156400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4.25</td>\n",
       "      <td>13.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>1338.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>399.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.544402</td>\n",
       "      <td>0.890573</td>\n",
       "      <td>0.812000</td>\n",
       "      <td>8.796028e+01</td>\n",
       "      <td>0.837401</td>\n",
       "      <td>56.283347</td>\n",
       "      <td>10.358378</td>\n",
       "      <td>13.529076</td>\n",
       "      <td>13.089567</td>\n",
       "      <td>12.863508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>3.25</td>\n",
       "      <td>5.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>517.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.577320</td>\n",
       "      <td>0.879914</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>6.493267e+01</td>\n",
       "      <td>0.763649</td>\n",
       "      <td>55.447093</td>\n",
       "      <td>10.345072</td>\n",
       "      <td>13.120825</td>\n",
       "      <td>14.013078</td>\n",
       "      <td>13.373814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>5.00</td>\n",
       "      <td>25.0</td>\n",
       "      <td>403.0</td>\n",
       "      <td>183.0</td>\n",
       "      <td>2034.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>599.0</td>\n",
       "      <td>263.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.454094</td>\n",
       "      <td>0.868402</td>\n",
       "      <td>0.855000</td>\n",
       "      <td>9.404830e+01</td>\n",
       "      <td>0.845886</td>\n",
       "      <td>64.727791</td>\n",
       "      <td>8.235758</td>\n",
       "      <td>10.616734</td>\n",
       "      <td>12.040226</td>\n",
       "      <td>10.402060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>4.50</td>\n",
       "      <td>4.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.658228</td>\n",
       "      <td>0.904289</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>4.386836e+01</td>\n",
       "      <td>0.751405</td>\n",
       "      <td>53.998877</td>\n",
       "      <td>10.634019</td>\n",
       "      <td>14.482278</td>\n",
       "      <td>11.430504</td>\n",
       "      <td>11.458418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20410</th>\n",
       "      <td>1</td>\n",
       "      <td>4.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>207.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.805556</td>\n",
       "      <td>0.939662</td>\n",
       "      <td>0.805556</td>\n",
       "      <td>5.183993e+01</td>\n",
       "      <td>0.805556</td>\n",
       "      <td>51.305000</td>\n",
       "      <td>9.084444</td>\n",
       "      <td>12.577778</td>\n",
       "      <td>15.542733</td>\n",
       "      <td>11.652500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20412</th>\n",
       "      <td>0</td>\n",
       "      <td>4.00</td>\n",
       "      <td>14.0</td>\n",
       "      <td>390.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>2122.0</td>\n",
       "      <td>109.0</td>\n",
       "      <td>658.0</td>\n",
       "      <td>251.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>0.839844</td>\n",
       "      <td>0.731429</td>\n",
       "      <td>4.799145e+01</td>\n",
       "      <td>0.789403</td>\n",
       "      <td>35.824615</td>\n",
       "      <td>15.183004</td>\n",
       "      <td>19.142857</td>\n",
       "      <td>15.129509</td>\n",
       "      <td>18.125802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20413</th>\n",
       "      <td>1</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>90.990000</td>\n",
       "      <td>1.313333</td>\n",
       "      <td>1.200000</td>\n",
       "      <td>-4.103777</td>\n",
       "      <td>-2.660000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20414</th>\n",
       "      <td>1</td>\n",
       "      <td>5.00</td>\n",
       "      <td>29.0</td>\n",
       "      <td>361.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>1705.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>261.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.556787</td>\n",
       "      <td>0.900563</td>\n",
       "      <td>0.865714</td>\n",
       "      <td>1.265851e+02</td>\n",
       "      <td>0.871082</td>\n",
       "      <td>74.213296</td>\n",
       "      <td>6.000562</td>\n",
       "      <td>9.300640</td>\n",
       "      <td>9.592866</td>\n",
       "      <td>7.039429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20415</th>\n",
       "      <td>0</td>\n",
       "      <td>2.75</td>\n",
       "      <td>3.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>399.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.704545</td>\n",
       "      <td>0.921783</td>\n",
       "      <td>0.760000</td>\n",
       "      <td>6.601719e+01</td>\n",
       "      <td>0.802241</td>\n",
       "      <td>55.929848</td>\n",
       "      <td>12.745455</td>\n",
       "      <td>16.278788</td>\n",
       "      <td>9.850318</td>\n",
       "      <td>14.592235</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18672 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       chosen  rating  n_sents  n_words  n_uniquewords  n_chars  n_longwords  \\\n",
       "0           0    4.50      1.0     25.0           24.0     96.0          1.0   \n",
       "1           0    4.25     13.0    259.0          141.0   1338.0         74.0   \n",
       "2           0    3.25      5.0     97.0           56.0    517.0         26.0   \n",
       "3           1    5.00     25.0    403.0          183.0   2034.0        111.0   \n",
       "4           1    4.50      4.0     79.0           52.0    386.0         20.0   \n",
       "...       ...     ...      ...      ...            ...      ...          ...   \n",
       "20410       1    4.50      3.0     36.0           29.0    207.0         12.0   \n",
       "20412       0    4.00     14.0    390.0          150.0   2122.0        109.0   \n",
       "20413       1    4.50      1.0      3.0            3.0     11.0          0.0   \n",
       "20414       1    5.00     29.0    361.0          201.0   1705.0         85.0   \n",
       "20415       0    2.75      3.0     88.0           62.0    399.0         20.0   \n",
       "\n",
       "       n_sylsprword  n_monosylwords  n_polysylwords  ...       ttr   log_ttr  \\\n",
       "0              29.0            21.0             0.0  ...  0.960000  0.987318   \n",
       "1             399.0           162.0            36.0  ...  0.544402  0.890573   \n",
       "2             151.0            66.0            13.0  ...  0.577320  0.879914   \n",
       "3             599.0           263.0            42.0  ...  0.454094  0.868402   \n",
       "4             124.0            50.0            13.0  ...  0.658228  0.904289   \n",
       "...             ...             ...             ...  ...       ...       ...   \n",
       "20410          61.0            24.0             7.0  ...  0.805556  0.939662   \n",
       "20412         658.0           251.0            78.0  ...  0.384615  0.839844   \n",
       "20413           4.0             2.0             0.0  ...  1.000000  1.000000   \n",
       "20414         512.0           261.0            39.0  ...  0.556787  0.900563   \n",
       "20415         126.0            62.0            10.0  ...  0.704545  0.921783   \n",
       "\n",
       "        seg_ttr          mtld       hdd  flesch_score  flesch_grade  \\\n",
       "0      0.960000  1.749988e+02  0.960000     83.324000      7.848000   \n",
       "1      0.812000  8.796028e+01  0.837401     56.283347     10.358378   \n",
       "2      0.680000  6.493267e+01  0.763649     55.447093     10.345072   \n",
       "3      0.855000  9.404830e+01  0.845886     64.727791      8.235758   \n",
       "4      0.700000  4.386836e+01  0.751405     53.998877     10.634019   \n",
       "...         ...           ...       ...           ...           ...   \n",
       "20410  0.805556  5.183993e+01  0.805556     51.305000      9.084444   \n",
       "20412  0.731429  4.799145e+01  0.789403     35.824615     15.183004   \n",
       "20413  1.000000  3.000000e+06  1.000000     90.990000      1.313333   \n",
       "20414  0.865714  1.265851e+02  0.871082     74.213296      6.000562   \n",
       "20415  0.760000  6.601719e+01  0.802241     55.929848     12.745455   \n",
       "\n",
       "       gunning_fog  coleman_liau  automated_readability  \n",
       "0        10.000000      5.594333               9.156400  \n",
       "1        13.529076     13.089567              12.863508  \n",
       "2        13.120825     14.013078              13.373814  \n",
       "3        10.616734     12.040226              10.402060  \n",
       "4        14.482278     11.430504              11.458418  \n",
       "...            ...           ...                    ...  \n",
       "20410    12.577778     15.542733              11.652500  \n",
       "20412    19.142857     15.129509              18.125802  \n",
       "20413     1.200000     -4.103777              -2.660000  \n",
       "20414     9.300640      9.592866               7.039429  \n",
       "20415    16.278788      9.850318              14.592235  \n",
       "\n",
       "[18672 rows x 21 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_only.iloc[:, :22]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indicators of High Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_290969/251589786.py:11: FutureWarning: ChainedAssignmentError: behaviour will change in pandas 3.0!\n",
      "You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.\n",
      "A typical example is when you are setting values in a column of a DataFrame, like:\n",
      "\n",
      "df[\"col\"][row_indexer] = value\n",
      "\n",
      "Use `df.loc[row_indexer, \"col\"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "  score_corr_df['rating'][index] = corr\n",
      "/tmp/ipykernel_290969/251589786.py:12: FutureWarning: ChainedAssignmentError: behaviour will change in pandas 3.0!\n",
      "You are setting values through chained assignment. Currently this works in certain cases, but when using Copy-on-Write (which will become the default behaviour in pandas 3.0) this will never work to update the original DataFrame or Series, because the intermediate object on which we are setting values will behave as a copy.\n",
      "A typical example is when you are setting values in a column of a DataFrame, like:\n",
      "\n",
      "df[\"col\"][row_indexer] = value\n",
      "\n",
      "Use `df.loc[row_indexer, \"col\"] = values` instead, to perform the assignment in a single step and ensure this keeps updating the original `df`.\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "  score_corr_df['p_value'][index] = pval\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pearson_r</th>\n",
       "      <th>p_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>n_sents</th>\n",
       "      <td>0.321838</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_words</th>\n",
       "      <td>0.345331</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_uniquewords</th>\n",
       "      <td>0.374831</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_chars</th>\n",
       "      <td>0.359418</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_longwords</th>\n",
       "      <td>0.358820</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_sylsprword</th>\n",
       "      <td>0.356477</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_monosylwords</th>\n",
       "      <td>0.315947</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_polysylwords</th>\n",
       "      <td>0.341162</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>entropy</th>\n",
       "      <td>0.338554</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ttr</th>\n",
       "      <td>-0.261976</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log_ttr</th>\n",
       "      <td>-0.094804</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seg_ttr</th>\n",
       "      <td>-0.033889</td>\n",
       "      <td>0.000004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mtld</th>\n",
       "      <td>-0.089744</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hdd</th>\n",
       "      <td>-0.019979</td>\n",
       "      <td>0.006330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flesch_score</th>\n",
       "      <td>-0.158606</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flesch_grade</th>\n",
       "      <td>0.113264</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gunning_fog</th>\n",
       "      <td>0.117725</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coleman_liau</th>\n",
       "      <td>0.191243</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automated_readability</th>\n",
       "      <td>0.119134</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       pearson_r   p_value\n",
       "n_sents                 0.321838  0.000000\n",
       "n_words                 0.345331  0.000000\n",
       "n_uniquewords           0.374831  0.000000\n",
       "n_chars                 0.359418  0.000000\n",
       "n_longwords             0.358820  0.000000\n",
       "n_sylsprword            0.356477  0.000000\n",
       "n_monosylwords          0.315947  0.000000\n",
       "n_polysylwords          0.341162  0.000000\n",
       "entropy                 0.338554  0.000000\n",
       "ttr                    -0.261976  0.000000\n",
       "log_ttr                -0.094804  0.000000\n",
       "seg_ttr                -0.033889  0.000004\n",
       "mtld                   -0.089744  0.000000\n",
       "hdd                    -0.019979  0.006330\n",
       "flesch_score           -0.158606  0.000000\n",
       "flesch_grade            0.113264  0.000000\n",
       "gunning_fog             0.117725  0.000000\n",
       "coleman_liau            0.191243  0.000000\n",
       "automated_readability   0.119134  0.000000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = {}\n",
    "continuous_vars = stats_only.columns[2:22]\n",
    "regressor = stats_only.columns[1]\n",
    "\n",
    "index = pd.MultiIndex.from_product([continuous_vars])\n",
    "score_corr_df = pd.DataFrame(index=continuous_vars, columns=[regressor, 'p_value'])\n",
    "\n",
    "for index in continuous_vars:\n",
    "    corr = pearsonr(stats_only[index], stats_only['rating'])[0]\n",
    "    pval = round(pearsonr(stats_only[index], stats_only['rating'])[1], 7)\n",
    "    score_corr_df['rating'][index] = corr\n",
    "    score_corr_df['p_value'][index] = pval\n",
    "    \n",
    "score_corr_df.rename(columns={'rating': 'pearson_r'}, inplace=True)\n",
    "score_corr_df = score_corr_df.apply(pd.to_numeric)\n",
    "score_corr_df\n",
    "score_corr_df.sort_values(by='pearson_r', ascending=False)\n",
    "score_corr_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Indicators of 'Accepted'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = {}\n",
    "continuous_vars = stats_only.columns[1:]\n",
    "dummy_vars = [stats_only.columns[0]]\n",
    "\n",
    "index = pd.MultiIndex.from_product([continuous_vars])\n",
    "\n",
    "corr_df = pd.DataFrame(index=continuous_vars, columns=dummy_vars)\n",
    "\n",
    "for dummy in dummy_vars:\n",
    "    for continuous in continuous_vars:\n",
    "        corr, p_value = pointbiserialr(stats_only[continuous], stats_only[dummy])\n",
    "        results[f'{continuous} & {dummy}'] = (corr, p_value)\n",
    "        corr_df.loc[continuous, dummy] = corr\n",
    "        \n",
    "\n",
    "corr_df = corr_df.apply(pd.to_numeric)\n",
    "\n",
    "pvalues = []\n",
    "for i in results.values():\n",
    "    pvalues.append((i[1]))\n",
    "\n",
    "corr_df['p_values'] = pvalues\n",
    "\n",
    "corr_df = corr_df.rename(columns={'chosen': 'PB_Correlation'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need a standardized way of assesing variance of metrics\n",
    "\n",
    "def coefficient_of_variation(series):\n",
    "    return series.std() / series.mean()\n",
    "\n",
    "cv = stats_only.iloc[:, 1:].apply(coefficient_of_variation)\n",
    "\n",
    "corr_df['coef_of_variation'] = cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PB_Correlation</th>\n",
       "      <th>p_values</th>\n",
       "      <th>coef_of_variation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <td>0.614129</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.234076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_uniquewords</th>\n",
       "      <td>0.196736</td>\n",
       "      <td>2.747041e-162</td>\n",
       "      <td>0.697326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_longwords</th>\n",
       "      <td>0.187078</td>\n",
       "      <td>1.157878e-146</td>\n",
       "      <td>0.949104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_chars</th>\n",
       "      <td>0.186004</td>\n",
       "      <td>5.608548e-145</td>\n",
       "      <td>0.857868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_sylsprword</th>\n",
       "      <td>0.183760</td>\n",
       "      <td>1.729277e-141</td>\n",
       "      <td>0.856042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_polysylwords</th>\n",
       "      <td>0.177103</td>\n",
       "      <td>2.090408e-131</td>\n",
       "      <td>1.061894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_words</th>\n",
       "      <td>0.176389</td>\n",
       "      <td>2.387506e-130</td>\n",
       "      <td>0.820856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_sents</th>\n",
       "      <td>0.167302</td>\n",
       "      <td>2.809735e-117</td>\n",
       "      <td>0.904568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_monosylwords</th>\n",
       "      <td>0.159521</td>\n",
       "      <td>1.154379e-106</td>\n",
       "      <td>0.812866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>entropy</th>\n",
       "      <td>0.150920</td>\n",
       "      <td>1.507789e-95</td>\n",
       "      <td>0.212756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coleman_liau</th>\n",
       "      <td>0.068946</td>\n",
       "      <td>4.035050e-21</td>\n",
       "      <td>0.465545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automated_readability</th>\n",
       "      <td>0.035431</td>\n",
       "      <td>1.280663e-06</td>\n",
       "      <td>0.602716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flesch_grade</th>\n",
       "      <td>0.028677</td>\n",
       "      <td>8.889472e-05</td>\n",
       "      <td>0.529671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gunning_fog</th>\n",
       "      <td>0.027647</td>\n",
       "      <td>1.579245e-04</td>\n",
       "      <td>0.420023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hdd</th>\n",
       "      <td>-0.006637</td>\n",
       "      <td>3.644979e-01</td>\n",
       "      <td>0.097357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seg_ttr</th>\n",
       "      <td>-0.019188</td>\n",
       "      <td>8.742473e-03</td>\n",
       "      <td>0.114890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mtld</th>\n",
       "      <td>-0.025967</td>\n",
       "      <td>3.871469e-04</td>\n",
       "      <td>5.354302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flesch_score</th>\n",
       "      <td>-0.047779</td>\n",
       "      <td>6.487878e-11</td>\n",
       "      <td>0.342247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log_ttr</th>\n",
       "      <td>-0.051049</td>\n",
       "      <td>2.956766e-12</td>\n",
       "      <td>0.078630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ttr</th>\n",
       "      <td>-0.119316</td>\n",
       "      <td>3.608736e-60</td>\n",
       "      <td>0.290619</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       PB_Correlation       p_values  coef_of_variation\n",
       "rating                       0.614129   0.000000e+00           0.234076\n",
       "n_uniquewords                0.196736  2.747041e-162           0.697326\n",
       "n_longwords                  0.187078  1.157878e-146           0.949104\n",
       "n_chars                      0.186004  5.608548e-145           0.857868\n",
       "n_sylsprword                 0.183760  1.729277e-141           0.856042\n",
       "n_polysylwords               0.177103  2.090408e-131           1.061894\n",
       "n_words                      0.176389  2.387506e-130           0.820856\n",
       "n_sents                      0.167302  2.809735e-117           0.904568\n",
       "n_monosylwords               0.159521  1.154379e-106           0.812866\n",
       "entropy                      0.150920   1.507789e-95           0.212756\n",
       "coleman_liau                 0.068946   4.035050e-21           0.465545\n",
       "automated_readability        0.035431   1.280663e-06           0.602716\n",
       "flesch_grade                 0.028677   8.889472e-05           0.529671\n",
       "gunning_fog                  0.027647   1.579245e-04           0.420023\n",
       "hdd                         -0.006637   3.644979e-01           0.097357\n",
       "seg_ttr                     -0.019188   8.742473e-03           0.114890\n",
       "mtld                        -0.025967   3.871469e-04           5.354302\n",
       "flesch_score                -0.047779   6.487878e-11           0.342247\n",
       "log_ttr                     -0.051049   2.956766e-12           0.078630\n",
       "ttr                         -0.119316   3.608736e-60           0.290619"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df.sort_values(by=['PB_Correlation', 'coef_of_variation', 'p_values'], ascending=[False, False, True])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Statistically Significant metrics for 'Accepted' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PB_Correlation</th>\n",
       "      <th>p_values</th>\n",
       "      <th>coef_of_variation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <td>0.614129</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.234076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_uniquewords</th>\n",
       "      <td>0.196736</td>\n",
       "      <td>2.747041e-162</td>\n",
       "      <td>0.697326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_longwords</th>\n",
       "      <td>0.187078</td>\n",
       "      <td>1.157878e-146</td>\n",
       "      <td>0.949104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_chars</th>\n",
       "      <td>0.186004</td>\n",
       "      <td>5.608548e-145</td>\n",
       "      <td>0.857868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_sylsprword</th>\n",
       "      <td>0.183760</td>\n",
       "      <td>1.729277e-141</td>\n",
       "      <td>0.856042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_polysylwords</th>\n",
       "      <td>0.177103</td>\n",
       "      <td>2.090408e-131</td>\n",
       "      <td>1.061894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_words</th>\n",
       "      <td>0.176389</td>\n",
       "      <td>2.387506e-130</td>\n",
       "      <td>0.820856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_sents</th>\n",
       "      <td>0.167302</td>\n",
       "      <td>2.809735e-117</td>\n",
       "      <td>0.904568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_monosylwords</th>\n",
       "      <td>0.159521</td>\n",
       "      <td>1.154379e-106</td>\n",
       "      <td>0.812866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>entropy</th>\n",
       "      <td>0.150920</td>\n",
       "      <td>1.507789e-95</td>\n",
       "      <td>0.212756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>coleman_liau</th>\n",
       "      <td>0.068946</td>\n",
       "      <td>4.035050e-21</td>\n",
       "      <td>0.465545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>automated_readability</th>\n",
       "      <td>0.035431</td>\n",
       "      <td>1.280663e-06</td>\n",
       "      <td>0.602716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flesch_grade</th>\n",
       "      <td>0.028677</td>\n",
       "      <td>8.889472e-05</td>\n",
       "      <td>0.529671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gunning_fog</th>\n",
       "      <td>0.027647</td>\n",
       "      <td>1.579245e-04</td>\n",
       "      <td>0.420023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seg_ttr</th>\n",
       "      <td>-0.019188</td>\n",
       "      <td>8.742473e-03</td>\n",
       "      <td>0.114890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mtld</th>\n",
       "      <td>-0.025967</td>\n",
       "      <td>3.871469e-04</td>\n",
       "      <td>5.354302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flesch_score</th>\n",
       "      <td>-0.047779</td>\n",
       "      <td>6.487878e-11</td>\n",
       "      <td>0.342247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log_ttr</th>\n",
       "      <td>-0.051049</td>\n",
       "      <td>2.956766e-12</td>\n",
       "      <td>0.078630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ttr</th>\n",
       "      <td>-0.119316</td>\n",
       "      <td>3.608736e-60</td>\n",
       "      <td>0.290619</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       PB_Correlation       p_values  coef_of_variation\n",
       "rating                       0.614129   0.000000e+00           0.234076\n",
       "n_uniquewords                0.196736  2.747041e-162           0.697326\n",
       "n_longwords                  0.187078  1.157878e-146           0.949104\n",
       "n_chars                      0.186004  5.608548e-145           0.857868\n",
       "n_sylsprword                 0.183760  1.729277e-141           0.856042\n",
       "n_polysylwords               0.177103  2.090408e-131           1.061894\n",
       "n_words                      0.176389  2.387506e-130           0.820856\n",
       "n_sents                      0.167302  2.809735e-117           0.904568\n",
       "n_monosylwords               0.159521  1.154379e-106           0.812866\n",
       "entropy                      0.150920   1.507789e-95           0.212756\n",
       "coleman_liau                 0.068946   4.035050e-21           0.465545\n",
       "automated_readability        0.035431   1.280663e-06           0.602716\n",
       "flesch_grade                 0.028677   8.889472e-05           0.529671\n",
       "gunning_fog                  0.027647   1.579245e-04           0.420023\n",
       "seg_ttr                     -0.019188   8.742473e-03           0.114890\n",
       "mtld                        -0.025967   3.871469e-04           5.354302\n",
       "flesch_score                -0.047779   6.487878e-11           0.342247\n",
       "log_ttr                     -0.051049   2.956766e-12           0.078630\n",
       "ttr                         -0.119316   3.608736e-60           0.290619"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df[corr_df['p_values'] <= 0.05].sort_values(by=['PB_Correlation', 'coef_of_variation'], ascending=[False, False])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## n_unique words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_unique_words = df_clean[df_clean['prompt'].isin(df_all['prompt'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2110413/1875626631.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_unique_words['n_unique_words_accepted'] = None\n",
      "/tmp/ipykernel_2110413/1875626631.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_unique_words['n_unique_words_rejected'] = None\n"
     ]
    }
   ],
   "source": [
    "df_unique_words['n_unique_words_accepted'] = None\n",
    "df_unique_words['n_unique_words_rejected'] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_unique_words.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "chosen_subset = df_all[df_all['chosen'] == 1]\n",
    "rejected_subset = df_all[df_all['chosen'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "nuwords_chosen_response_metric_pairs = pd.DataFrame({'response': chosen_subset.loc[:, 'response'], 'score': chosen_subset.loc[:, 'n_uniquewords']}).reset_index(drop=True)\n",
    "nuwords_rejected_response_metric_pairs = pd.DataFrame({'response': rejected_subset.loc[:, 'response'], 'score': rejected_subset.loc[:, 'n_uniquewords']}).reset_index(drop=True)\n",
    "\n",
    "for i in range(len(df_unique_words)):\n",
    "    df_unique_words.iloc[i, df_unique_words.columns.get_loc('n_unique_words_accepted')] = nuwords_chosen_response_metric_pairs.iloc[np.where(nuwords_chosen_response_metric_pairs['response'] == df_unique_words['chosen'][i])[0][0], -1]\n",
    "    df_unique_words.iloc[i, df_unique_words.columns.get_loc('n_unique_words_rejected')] = nuwords_rejected_response_metric_pairs.iloc[np.where(nuwords_rejected_response_metric_pairs['response'] == df_unique_words['rejected'][i])[0][0], -1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_unique_words.to_csv('unique_words_sample.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making the metric"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardizing units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the z-score for all the metrics. \n",
    "for col in stats_only.columns:\n",
    "    df_all.loc[:, col + '_z_score'] = zscore(df_all.loc[:, col])\n",
    "\n",
    "\n",
    "z_score_cols = df_all.columns[32: ]\n",
    "relevant_z_scores = ['n_uniquewords_z_score', 'n_longwords_z_score', 'n_chars_z_score', 'n_sylsprword_z_score', \n",
    "                     'n_polysylwords_z_score', 'n_words_z_score', 'n_sents_z_score', 'n_monosylwords_z_score',\n",
    "                     'entropy_z_score', 'coleman_liau_z_score', 'automated_readability_z_score', 'flesch_grade_z_score', 'gunning_fog_z_score']\n",
    "\n",
    "# df_all.to_csv(\"Accepted_Rejected_with_scores.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I don't think I need this anymore\n",
    "\n",
    "# NB! You need to remove these two rows because they throw an error when you try to match them with a score, because they are the ones we took out when converting to spacy doc, since spacy cannot convert empty strings. I probably should have done this earlier but oh well. They are found by running the loops below\n",
    "# df_clean = df_clean.drop([2776, 3482]).reset_index(drop=True)\n",
    "\n",
    "# for i in range(len(df_clean)):\n",
    "#     try:\n",
    "#         x = np.where(chosen_response_metric_pairs['response'] == df_clean['chosen'][i])[0][0]\n",
    "#     except:\n",
    "#         print(i)\n",
    "\n",
    "# for i in range(len(df_clean)):\n",
    "#     try:\n",
    "#         x = np.where(rejected_response_metric_pairs['response'] == df_clean['rejected'][i])[0][0]\n",
    "#     except:\n",
    "#         print(i)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all.loc[:, 'linear_metric'] = df_all.loc[:, relevant_z_scores].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a copy for editing\n",
    "df_linear = df_clean[df_clean['prompt'].isin(df_all['prompt'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Group</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Standard Deviaton</th>\n",
       "      <th>Variance</th>\n",
       "      <th>Max</th>\n",
       "      <th>Min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accepted</td>\n",
       "      <td>1.748692</td>\n",
       "      <td>10.409601</td>\n",
       "      <td>108.359788</td>\n",
       "      <td>51.401732</td>\n",
       "      <td>-32.810444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Rejected</td>\n",
       "      <td>-1.748692</td>\n",
       "      <td>9.462044</td>\n",
       "      <td>89.530281</td>\n",
       "      <td>131.373559</td>\n",
       "      <td>-32.810444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Group      Mean  Standard Deviaton    Variance         Max        Min\n",
       "0  Accepted  1.748692          10.409601  108.359788   51.401732 -32.810444\n",
       "1  Rejected -1.748692           9.462044   89.530281  131.373559 -32.810444"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_subset = df_all[df_all['chosen'] == 1]\n",
    "rejected_subset = df_all[df_all['chosen'] == 0]\n",
    "\n",
    "Linear_chosen_response_metric_pairs = pd.DataFrame({'response': chosen_subset.loc[:, 'response'], 'score': chosen_subset.loc[:, 'linear_metric']}).reset_index(drop=True)\n",
    "Linear_rejected_response_metric_pairs = pd.DataFrame({'response': rejected_subset.loc[:, 'response'], 'score': rejected_subset.loc[:, 'linear_metric']}).reset_index(drop=True)\n",
    "\n",
    "#Let's check out the stats on the metric real quick:\n",
    "Linear_metric_distribution = pd.DataFrame({'Group': ['Accepted', 'Rejected'], \n",
    "                        'Mean' : [Linear_chosen_response_metric_pairs['score'].mean(), Linear_rejected_response_metric_pairs['score'].mean()], \n",
    "                        'Standard Deviaton': [np.std(Linear_chosen_response_metric_pairs['score']), np.std(Linear_rejected_response_metric_pairs['score'])],\n",
    "                        'Variance': [np.var(Linear_chosen_response_metric_pairs['score']), np.var(Linear_rejected_response_metric_pairs['score'])],\n",
    "                        'Max': [Linear_chosen_response_metric_pairs['score'].max(), Linear_rejected_response_metric_pairs['score'].max()],\n",
    "                        'Min': [Linear_chosen_response_metric_pairs['score'].min(), Linear_rejected_response_metric_pairs['score'].min()]}) \n",
    "\n",
    "Linear_metric_distribution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's add the metrics in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize columns\n",
    "df_linear = df_linear.reset_index(drop=True)\n",
    "df_linear['accepted_linear_metric'] = None\n",
    "df_linear['rejected_linear_metric'] = None\n",
    "\n",
    "#cycle through the subsets to get the scores and match them to their respective rows given the pairs in df_clean\n",
    "for i in range(len(df_linear)):\n",
    "    df_linear.iloc[i, df_linear.columns.get_loc('accepted_linear_metric')]= Linear_chosen_response_metric_pairs.iloc[np.where(Linear_chosen_response_metric_pairs['response'] == df_linear['chosen'][i])[0][0], -1]\n",
    "    df_linear.iloc[i, df_linear.columns.get_loc('rejected_linear_metric')] = Linear_rejected_response_metric_pairs.iloc[np.where(Linear_rejected_response_metric_pairs['response'] == df_linear['rejected'][i])[0][0], -1]\n",
    "\n",
    "df_linear['diff_linear_metric'] = df_linear['accepted_linear_metric'] - df_linear['rejected_linear_metric']\n",
    "df_linear.to_csv(\"linear_metric_df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>prompt</th>\n",
       "      <th>chosen</th>\n",
       "      <th>chosen-rating</th>\n",
       "      <th>chosen-model</th>\n",
       "      <th>rejected</th>\n",
       "      <th>rejected-rating</th>\n",
       "      <th>rejected-model</th>\n",
       "      <th>x_cord</th>\n",
       "      <th>y_cord</th>\n",
       "      <th>topic_id</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>metadata</th>\n",
       "      <th>topic_name</th>\n",
       "      <th>accepted_linear_metric</th>\n",
       "      <th>rejected_linear_metric</th>\n",
       "      <th>diff_linear_metric</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Topics: Wound management for general practitio...</td>\n",
       "      <td>General practitioners play a vital role in the...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>Wound management is a crucial aspect of genera...</td>\n",
       "      <td>3.75</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>0.915843</td>\n",
       "      <td>0.482842</td>\n",
       "      <td>22</td>\n",
       "      <td>[0.20282909274101257, -0.05036671459674835, -0...</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Childhood Stress and Health</td>\n",
       "      <td>16.129277</td>\n",
       "      <td>-0.869996</td>\n",
       "      <td>16.999273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flan_v2_niv2</td>\n",
       "      <td>Part 1. Definition\\nIn this task, you are give...</td>\n",
       "      <td>Step 1: Identify the language of the post. The...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>Non-hateful\\n\\nExplanation: This post seems to...</td>\n",
       "      <td>3.25</td>\n",
       "      <td>vicuna-33b</td>\n",
       "      <td>2.740021</td>\n",
       "      <td>4.862818</td>\n",
       "      <td>14</td>\n",
       "      <td>[0.06639007478952408, 0.23666216433048248, -0....</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_niv2', 'ratin...</td>\n",
       "      <td>Analyzing Product Sentiment</td>\n",
       "      <td>-7.417868</td>\n",
       "      <td>-5.87031</td>\n",
       "      <td>-1.547558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>You will act as an voice changer. You will cha...</td>\n",
       "      <td>An umbrella has been brought by her on the way.</td>\n",
       "      <td>4.50</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>Alright, just let me know which form of voice ...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>2.045614</td>\n",
       "      <td>4.058925</td>\n",
       "      <td>15</td>\n",
       "      <td>[-0.4921853840351105, -0.1649392992258072, -0....</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Language Analysis</td>\n",
       "      <td>-17.180493</td>\n",
       "      <td>-11.351724</td>\n",
       "      <td>-5.82877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Write a well-researched paper on the physiolog...</td>\n",
       "      <td>Title: The Physiological and Psychological Imp...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>Title: The Physiological and Psychological Imp...</td>\n",
       "      <td>3.00</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>1.698340</td>\n",
       "      <td>0.061782</td>\n",
       "      <td>8</td>\n",
       "      <td>[-0.30388227105140686, 0.15756076574325562, 0....</td>\n",
       "      <td>('metadata', {'source': 'ultrachat', 'rating':...</td>\n",
       "      <td>Healthy Lifestyle Exploration</td>\n",
       "      <td>23.249564</td>\n",
       "      <td>1.21647</td>\n",
       "      <td>22.033094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ultrachat</td>\n",
       "      <td>Create a step-by-step recipe for making fluffy...</td>\n",
       "      <td>Of course, I'd be happy to help you with that!...</td>\n",
       "      <td>5.00</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>Here is a step-by-step recipe for making fluff...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>falcon-40b-instruct</td>\n",
       "      <td>3.702293</td>\n",
       "      <td>1.716949</td>\n",
       "      <td>6</td>\n",
       "      <td>[-0.05435372143983841, 0.28580591082572937, -0...</td>\n",
       "      <td>('metadata', {'source': 'ultrachat', 'rating':...</td>\n",
       "      <td>Culinary Creations</td>\n",
       "      <td>4.427499</td>\n",
       "      <td>8.015921</td>\n",
       "      <td>-3.588422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9333</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Transform is an emerging brand that delivers m...</td>\n",
       "      <td>Welcome to Transform, your go-to source for in...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>Sure, here's a possible description for the \"A...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>llama-2-70b-chat</td>\n",
       "      <td>0.732869</td>\n",
       "      <td>1.959356</td>\n",
       "      <td>21</td>\n",
       "      <td>[-0.4771421253681183, -0.0352528840303421, -0....</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Marketing Strategy Platform</td>\n",
       "      <td>-2.682148</td>\n",
       "      <td>-0.578695</td>\n",
       "      <td>-2.103454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9334</th>\n",
       "      <td>evol_instruct</td>\n",
       "      <td>Using Python and the scikit-learn library, how...</td>\n",
       "      <td>Detecting sarcasm in text is a challenging tas...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>wizardlm-13b</td>\n",
       "      <td>Hello! As a helpful and respectful assistant, ...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>llama-2-13b-chat</td>\n",
       "      <td>2.532706</td>\n",
       "      <td>5.040441</td>\n",
       "      <td>14</td>\n",
       "      <td>[0.3401801884174347, 0.21816378831863403, 0.02...</td>\n",
       "      <td>('metadata', {'source': 'evol_instruct', 'rati...</td>\n",
       "      <td>Analyzing Product Sentiment</td>\n",
       "      <td>14.92361</td>\n",
       "      <td>20.216777</td>\n",
       "      <td>-5.293167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9335</th>\n",
       "      <td>flan_v2_cot</td>\n",
       "      <td>[QUESTION] If \"Two kids stepping stones over t...</td>\n",
       "      <td>Yes, the premise mentions pink plaid pants and...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>gpt-3.5-turbo</td>\n",
       "      <td>Confidence: 80%\\n\\nYou have provided several s...</td>\n",
       "      <td>1.75</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>3.797116</td>\n",
       "      <td>0.511288</td>\n",
       "      <td>18</td>\n",
       "      <td>[0.39390242099761963, 0.10773289203643799, -0....</td>\n",
       "      <td>('metadata', {'source': 'flan_v2_cot', 'rating...</td>\n",
       "      <td>Gender and Clothing Perceptions</td>\n",
       "      <td>-10.996563</td>\n",
       "      <td>-8.51755</td>\n",
       "      <td>-2.479013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9336</th>\n",
       "      <td>sharegpt</td>\n",
       "      <td>Write me a letter asking my friend Dan what he...</td>\n",
       "      <td>Dear Dan,\\n\\nWe hope this letter finds you wel...</td>\n",
       "      <td>4.50</td>\n",
       "      <td>ultralm-13b</td>\n",
       "      <td>I cannot fulfill your request as it goes again...</td>\n",
       "      <td>1.25</td>\n",
       "      <td>llama-2-7b-chat</td>\n",
       "      <td>0.873350</td>\n",
       "      <td>3.242940</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.7983337640762329, -0.37210243940353394, 0.5...</td>\n",
       "      <td>('metadata', {'source': 'sharegpt', 'rating': ...</td>\n",
       "      <td>Cultural Expression</td>\n",
       "      <td>2.011627</td>\n",
       "      <td>-0.78087</td>\n",
       "      <td>2.792497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9337</th>\n",
       "      <td>false_qa</td>\n",
       "      <td>What's the resolution of a cat?</td>\n",
       "      <td>As an AI, I assume you are asking about a cat'...</td>\n",
       "      <td>4.25</td>\n",
       "      <td>gpt-4</td>\n",
       "      <td>As an AI, I don't have the capability to deter...</td>\n",
       "      <td>4.00</td>\n",
       "      <td>wizardlm-70b</td>\n",
       "      <td>0.756816</td>\n",
       "      <td>1.959702</td>\n",
       "      <td>10</td>\n",
       "      <td>[0.114014632999897, -0.06832381337881088, 0.15...</td>\n",
       "      <td>('metadata', {'source': 'false_qa', 'rating': ...</td>\n",
       "      <td>Creative Writing</td>\n",
       "      <td>-4.270032</td>\n",
       "      <td>-3.496454</td>\n",
       "      <td>-0.773578</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9338 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             source                                             prompt  \\\n",
       "0          sharegpt  Topics: Wound management for general practitio...   \n",
       "1      flan_v2_niv2  Part 1. Definition\\nIn this task, you are give...   \n",
       "2          sharegpt  You will act as an voice changer. You will cha...   \n",
       "3         ultrachat  Write a well-researched paper on the physiolog...   \n",
       "4         ultrachat  Create a step-by-step recipe for making fluffy...   \n",
       "...             ...                                                ...   \n",
       "9333       sharegpt  Transform is an emerging brand that delivers m...   \n",
       "9334  evol_instruct  Using Python and the scikit-learn library, how...   \n",
       "9335    flan_v2_cot  [QUESTION] If \"Two kids stepping stones over t...   \n",
       "9336       sharegpt  Write me a letter asking my friend Dan what he...   \n",
       "9337       false_qa                    What's the resolution of a cat?   \n",
       "\n",
       "                                                 chosen  chosen-rating  \\\n",
       "0     General practitioners play a vital role in the...           5.00   \n",
       "1     Step 1: Identify the language of the post. The...           4.50   \n",
       "2       An umbrella has been brought by her on the way.           4.50   \n",
       "3     Title: The Physiological and Psychological Imp...           4.75   \n",
       "4     Of course, I'd be happy to help you with that!...           5.00   \n",
       "...                                                 ...            ...   \n",
       "9333  Welcome to Transform, your go-to source for in...           4.75   \n",
       "9334  Detecting sarcasm in text is a challenging tas...           4.75   \n",
       "9335  Yes, the premise mentions pink plaid pants and...           4.75   \n",
       "9336  Dear Dan,\\n\\nWe hope this letter finds you wel...           4.50   \n",
       "9337  As an AI, I assume you are asking about a cat'...           4.25   \n",
       "\n",
       "         chosen-model                                           rejected  \\\n",
       "0        wizardlm-70b  Wound management is a crucial aspect of genera...   \n",
       "1        wizardlm-70b  Non-hateful\\n\\nExplanation: This post seems to...   \n",
       "2        wizardlm-13b  Alright, just let me know which form of voice ...   \n",
       "3        wizardlm-13b  Title: The Physiological and Psychological Imp...   \n",
       "4     llama-2-7b-chat  Here is a step-by-step recipe for making fluff...   \n",
       "...               ...                                                ...   \n",
       "9333    gpt-3.5-turbo  Sure, here's a possible description for the \"A...   \n",
       "9334     wizardlm-13b  Hello! As a helpful and respectful assistant, ...   \n",
       "9335    gpt-3.5-turbo  Confidence: 80%\\n\\nYou have provided several s...   \n",
       "9336      ultralm-13b  I cannot fulfill your request as it goes again...   \n",
       "9337            gpt-4  As an AI, I don't have the capability to deter...   \n",
       "\n",
       "      rejected-rating       rejected-model    x_cord    y_cord  topic_id  \\\n",
       "0                3.75          ultralm-13b  0.915843  0.482842        22   \n",
       "1                3.25           vicuna-33b  2.740021  4.862818        14   \n",
       "2                1.75          ultralm-13b  2.045614  4.058925        15   \n",
       "3                3.00          ultralm-13b  1.698340  0.061782         8   \n",
       "4                4.50  falcon-40b-instruct  3.702293  1.716949         6   \n",
       "...               ...                  ...       ...       ...       ...   \n",
       "9333             4.00     llama-2-70b-chat  0.732869  1.959356        21   \n",
       "9334             4.00     llama-2-13b-chat  2.532706  5.040441        14   \n",
       "9335             1.75      llama-2-7b-chat  3.797116  0.511288        18   \n",
       "9336             1.25      llama-2-7b-chat  0.873350  3.242940         1   \n",
       "9337             4.00         wizardlm-70b  0.756816  1.959702        10   \n",
       "\n",
       "                                             embeddings  \\\n",
       "0     [0.20282909274101257, -0.05036671459674835, -0...   \n",
       "1     [0.06639007478952408, 0.23666216433048248, -0....   \n",
       "2     [-0.4921853840351105, -0.1649392992258072, -0....   \n",
       "3     [-0.30388227105140686, 0.15756076574325562, 0....   \n",
       "4     [-0.05435372143983841, 0.28580591082572937, -0...   \n",
       "...                                                 ...   \n",
       "9333  [-0.4771421253681183, -0.0352528840303421, -0....   \n",
       "9334  [0.3401801884174347, 0.21816378831863403, 0.02...   \n",
       "9335  [0.39390242099761963, 0.10773289203643799, -0....   \n",
       "9336  [0.7983337640762329, -0.37210243940353394, 0.5...   \n",
       "9337  [0.114014632999897, -0.06832381337881088, 0.15...   \n",
       "\n",
       "                                               metadata  \\\n",
       "0     ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "1     ('metadata', {'source': 'flan_v2_niv2', 'ratin...   \n",
       "2     ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "3     ('metadata', {'source': 'ultrachat', 'rating':...   \n",
       "4     ('metadata', {'source': 'ultrachat', 'rating':...   \n",
       "...                                                 ...   \n",
       "9333  ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "9334  ('metadata', {'source': 'evol_instruct', 'rati...   \n",
       "9335  ('metadata', {'source': 'flan_v2_cot', 'rating...   \n",
       "9336  ('metadata', {'source': 'sharegpt', 'rating': ...   \n",
       "9337  ('metadata', {'source': 'false_qa', 'rating': ...   \n",
       "\n",
       "                           topic_name accepted_linear_metric  \\\n",
       "0         Childhood Stress and Health              16.129277   \n",
       "1         Analyzing Product Sentiment              -7.417868   \n",
       "2                   Language Analysis             -17.180493   \n",
       "3       Healthy Lifestyle Exploration              23.249564   \n",
       "4                  Culinary Creations               4.427499   \n",
       "...                               ...                    ...   \n",
       "9333      Marketing Strategy Platform              -2.682148   \n",
       "9334      Analyzing Product Sentiment               14.92361   \n",
       "9335  Gender and Clothing Perceptions             -10.996563   \n",
       "9336              Cultural Expression               2.011627   \n",
       "9337                 Creative Writing              -4.270032   \n",
       "\n",
       "     rejected_linear_metric diff_linear_metric  \n",
       "0                 -0.869996          16.999273  \n",
       "1                  -5.87031          -1.547558  \n",
       "2                -11.351724           -5.82877  \n",
       "3                   1.21647          22.033094  \n",
       "4                  8.015921          -3.588422  \n",
       "...                     ...                ...  \n",
       "9333              -0.578695          -2.103454  \n",
       "9334              20.216777          -5.293167  \n",
       "9335               -8.51755          -2.479013  \n",
       "9336               -0.78087           2.792497  \n",
       "9337              -3.496454          -0.773578  \n",
       "\n",
       "[9338 rows x 17 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_linear"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Geometric Mean - actually can't do this because there are negative numbers in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gmean_linear = df_clean[df_clean['prompt'].isin(df_all['prompt'])].reset_index(drop=True)\n",
    "df_gmean_linear['accepted_gmean'] = None # add some constant to all the values to get the negatives out of there.\n",
    "df_gmean_linear['rejected_gmean'] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gmeans = gmean(df_all.loc[:, relevant_z_scores])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.36082393, 0.33315898, 0.36894024, 0.36676704, 0.30441279,\n",
       "       0.37709697, 0.32685727, 0.3550817 , 0.21994893, 0.06290751,\n",
       "       0.10475022, 0.17203504, 0.15986658])"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gmeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000000000000004"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_subset = df_all[df_all['chosen'] == 1]\n",
    "rejected_subset = df_all[df_all['chosen'] == 0]\n",
    "\n",
    "gmean_chosen_response_metric_pairs = pd.DataFrame({'response': chosen_subset.loc[:, 'response'], 'score': chosen_subset.loc[:, 'Gmean']}).reset_index(drop=True)\n",
    "gmean_rejected_response_metric_pairs = pd.DataFrame({'response': rejected_subset.loc[:, 'response'], 'score': rejected_subset.loc[:, 'Gmean']}).reset_index(drop=True)\n",
    "\n",
    "#Let's check out the stats on the metric real quick:\n",
    "gmean_metric_distribution = pd.DataFrame({'Group': ['Accepted', 'Rejected'], \n",
    "                        'Mean' : [Linear_chosen_response_metric_pairs['score'].mean(), Linear_rejected_response_metric_pairs['score'].mean()], \n",
    "                        'Standard Deviaton': [np.std(Linear_chosen_response_metric_pairs['score']), np.std(Linear_rejected_response_metric_pairs['score'])],\n",
    "                        'Variance': [np.var(Linear_chosen_response_metric_pairs['score']), np.var(Linear_rejected_response_metric_pairs['score'])],\n",
    "                        'Max': [Linear_chosen_response_metric_pairs['score'].max(), Linear_rejected_response_metric_pairs['score'].max()],\n",
    "                        'Min': [Linear_chosen_response_metric_pairs['score'].min(), Linear_rejected_response_metric_pairs['score'].min()]}) \n",
    "\n",
    "gmean_metric_distribution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pca = df_clean[df_clean['prompt'].isin(df_all['prompt'])].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'PCA Explained Variance')"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAHHCAYAAACle7JuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABCIklEQVR4nO3df3xP9f//8fvLfttsWLaZn/P7t7cfYYTSys8iJFKm3++an+Md+oTI7wopPyJRvXkroYV3xGK91TC/iuRHmgjbUm1j2sx2vn908fr2ahuvF6/t9Tq6XS+Xc7l4Pc85z/N4nbTdnfN8nmMxDMMQAACACZVydQEAAAA3iiADAABMiyADAABMiyADAABMiyADAABMiyADAABMiyADAABMiyADAABMiyADAABMiyADwKmWL18ui8WikydPOrzv4MGDVb16dafXZI+bqbu4uGNNgLshyAA34eovmquLr6+v6tSpoyFDhig1NbXA9qmpqRo9erTq1aun0qVLy9/fXy1atNCUKVOUnp5e6DFatWoli8WihQsX2l3XyZMnber66zJjxowb/cp/a02aNFHVqlV1rTe7tGvXTqGhobpy5UoJVgb8fXm6ugDgVjB58mRFREQoOztbO3bs0MKFC/Xf//5Xhw4dUunSpSVJSUlJ6tatmy5evKhHHnlELVq0kCTt2bNHM2bM0BdffKHPPvvMpt/jx48rKSlJ1atX14oVK/Tss886VNeAAQPUrVu3Au3NmjW7wW9avJYsWaL8/HxXl1GkgQMHauzYsfrf//6nDh06FFh/8uRJJSYmasiQIfL0vPkfr48++qj69+8vHx+fm+4LuFURZAAn6Nq1q1q2bClJevLJJxUcHKzZs2crLi5OAwYMUHp6uh544AF5eHho//79qlevns3+U6dO1ZIlSwr0++9//1shISF67bXX1LdvX508edKhWy/NmzfXI488clPfrSR5eXm5uoRrevjhhzVu3DitXLmy0CDzn//8R4ZhaODAgTd1nKysLPn7+8vDw0MeHh431Rdwq+PWElAMOnXqJElKTk6WJL311ls6c+aMZs+eXSDESFJoaKhefPHFAu0rV65U37591aNHDwUFBWnlypVOrfPzzz9XqVKlNGHChALH/evtLIvFoiFDhmjFihWqW7eufH191aJFC33xxRfXPU5cXJy6d++u8PBw+fj4qGbNmnr55ZeVl5dns91fx8hcvUX26quvavHixapZs6Z8fHx0++23KykpqcBxjhw5or59+6p8+fLy9fVVy5Yt9cknnxTY7ttvv1WnTp3k5+enypUra8qUKXZdCapSpYo6dOigjz76SLm5uQXWr1y5UjVr1lTr1q31448/6rnnnlPdunXl5+en4OBgPfjggwXGu1y9PZmQkKDnnntOISEhqly5ss26P+9j77m888471ahRIx0+fFh33XWXSpcurUqVKmnWrFkF6s7OztZLL72kOnXqyNfXVxUrVlTv3r114sQJ6zb5+fmaO3euGjZsKF9fX4WGhuqZZ57Rb7/9dt3zBhQnrsgAxeDqL4Dg4GBJ0ieffCI/Pz/17dvX7j527dql77//XsuWLZO3t7d69+6tFStW6IUXXrC7j0uXLun8+fMF2suWLStPT0916tRJzz33nKZPn65evXqpefPmOnfunIYOHaqoqCj985//tNkvISFBH3zwgYYNGyYfHx8tWLBAXbp00e7du9WoUaMi61i+fLkCAgIUGxurgIAAff7555owYYIyMzP1yiuvXPd7rFy5UhcuXNAzzzwji8WiWbNmqXfv3vrhhx+sV3G+/fZbtWvXTpUqVdLYsWPl7++vDz/8UL169dKaNWv0wAMPSJJSUlJ011136cqVK9btFi9eLD8/P7vO6cCBA/X0009r8+bN6tGjh7X94MGDOnTokDUUJiUl6auvvlL//v1VuXJlnTx5UgsXLtSdd96pw4cPW285XvXcc8+pQoUKmjBhgrKyspxyLn/77Td16dJFvXv3Vr9+/fTRRx9pzJgxaty4sbp27SpJysvLU48ePRQfH6/+/ftr+PDhunDhgrZs2aJDhw6pZs2akqRnnnlGy5cv12OPPaZhw4YpOTlZb775pvbv368vv/zS7a+m4RZmALhhy5YtMyQZW7duNX7++Wfj9OnTxqpVq4zg4GDDz8/P+OmnnwzDMIxy5coZTZs2dajvIUOGGFWqVDHy8/MNwzCMzz77zJBk7N+//7r7JicnG5KKXBITE63bZmVlGbVq1TIaNmxoZGdnG927dzcCAwONH3/80abPq/vu2bPH2vbjjz8avr6+xgMPPFDgnCQnJ1vbLl26VKDGZ555xihdurSRnZ1tbYuOjjaqVatW4HsEBwcbv/76q7U9Li7OkGSsX7/e2nb33XcbjRs3tukvPz/faNu2rVG7dm1r24gRIwxJxq5du6xtaWlpRlBQUIG6C/Prr78aPj4+xoABA2zax44da0gyjh49WuR3TkxMNCQZ7733nrXt6vm64447jCtXrthsfzPnsmPHjgWOlZOTY4SFhRl9+vSxtr3zzjuGJGP27NkF+r36d+9///ufIclYsWKFzfpNmzYV2g6UJG4tAU4QFRWlChUqqEqVKurfv78CAgK0bt06VapUSZKUmZmpMmXK2N3flStX9MEHH+ihhx6SxWKR9MftqpCQEK1YscLufp5++mlt2bKlwNKgQQPrNqVLl9by5cv13XffqUOHDtq4caPmzJmjqlWrFugvMjLSOkhZkqpWraqePXtq8+bNBW5t/Nmfr3ZcuHBB58+fV/v27XXp0iUdOXLkut/joYceUrly5ayf27dvL0n64YcfJEm//vqrPv/8c/Xr18/a//nz5/XLL7+oc+fOOn78uM6cOSNJ+u9//6s2bdqoVatW1v4qVKhg97iWcuXKqVu3bvrkk0+sV04Mw9CqVavUsmVL1alTp8B3zs3N1S+//KJatWqpbNmy2rdvX4F+n3rqKbvGwzhyLgMCAmzGSHl7e6tVq1bW8yZJa9as0W233aahQ4cWONbVv3urV69WUFCQ7rnnHuu5PX/+vFq0aKGAgABt27btunUDxYVbS4ATzJ8/X3Xq1JGnp6dCQ0NVt25dlSr1//+dEBgYqAsXLtjd32effaaff/5ZrVq10vfff29tv+uuu/Sf//xHM2fOtOm/KLVr11ZUVNR1t2vXrp2effZZzZ8/X507d9bjjz9eZH9/VadOHV26dEk///yzwsLCCt3v22+/1YsvvqjPP/9cmZmZNusyMjKuW99fQ9XVUHN1fMb3338vwzA0fvx4jR8/vtA+0tLSVKlSJf34449q3bp1gfV169a9bh1XDRw4UOvWrVNcXJwefvhhffXVVzp58qSGDx9u3eb333/X9OnTtWzZMp05c8ZmynZh3zkiIsKuYztyLitXrmwNI1eVK1dO33zzjfXziRMnVLdu3WvOsjp+/LgyMjIUEhJS6Pq0tDS7ageKA0EGcIJWrVpZZy0Vpl69ejpw4IAuX74sb2/v6/Z39apLv379Cl2fkJCgu+6668aKLUROTo62b98u6Y9fbJcuXSowhuNGpaenq2PHjgoMDNTkyZNVs2ZN+fr6at++fRozZoxdg2yLulJxNRxc7WP06NHq3LlzodvWqlXrBr9BQX8efP3www9r5cqV8vDwUP/+/a3bDB06VMuWLdOIESMUGRmpoKAgWSwW9e/fv9DvbM8YHUfP5fXOm73y8/OveTWwQoUKDvUHOBNBBigB9913nxITE7VmzRoNGDDgmttmZWUpLi5ODz30UKGDg4cNG6YVK1Y4NchMnDhR3333nV599VWNGTNGY8eO1bx58wpsd/z48QJtx44dU+nSpYv8ZbZ9+3b98ssvWrt2rc2U5aszupyhRo0akv6Yvn29K1DVqlUr9HscPXrU7uP5+Piob9++eu+995SamqrVq1erU6dONlekPvroI0VHR+u1116ztmVnZxf54EN7FMe5rFmzpnbt2qXc3NwiB+zWrFlTW7duVbt27eweFA2UFMbIACXgn//8pypWrKhRo0bp2LFjBdanpaVpypQpkqR169YpKytLMTEx6tu3b4GlR48eWrNmjXJycpxS265du/Tqq69qxIgRGjVqlP71r3/pzTffVEJCQoFtExMTbcZ3nD59WnFxcbr33nuL/Nf/1fY/XwW4fPmyFixY4JT6JSkkJER33nmn3nrrLZ07d67A+p9//tn6527dumnnzp3avXu3zXpHxh5Jf9xeys3N1TPPPKOff/65wBgbDw+PAlc+3njjjWuOJbqe4jiXffr00fnz5/Xmm28WWHf1OP369VNeXp5efvnlAttcuXLlpsIZcLO4IgOUgHLlymndunXq1q2b/vGPf9g82Xffvn36z3/+o8jISEl/3FYKDg5W27ZtC+3r/vvv15IlS7Rx40b17t37msfdt2+f/v3vfxdor1mzpiIjI5Wdna3o6GjVrl1bU6dOlSRNmjRJ69ev12OPPaaDBw/K39/ful+jRo3UuXNnm+nXV/cpStu2bVWuXDlFR0dr2LBhslgsev/99x2+vXE98+fP1x133KHGjRvrqaeeUo0aNZSamqrExET99NNP+vrrryVJzz//vN5//3116dJFw4cPt06/rlatms3Ykevp2LGjKleurLi4OPn5+RX4b9GjRw+9//77CgoKUoMGDZSYmKitW7dap+TfiOI4l4MGDdJ7772n2NhY7d69W+3bt1dWVpa2bt2q5557Tj179lTHjh31zDPPaPr06Tpw4IDuvfdeeXl56fjx41q9erVef/11hx4tADiVi2ZLAbeEq9Njk5KS7Nr+7NmzxsiRI406deoYvr6+RunSpY0WLVoYU6dONTIyMozU1FTD09PTePTRR4vs49KlS0bp0qVtpjz/1fWmX0dHRxuGYRgjR440PDw8bKYiG4Zh7Nmzx/D09DSeffZZa5skIyYmxvj3v/9t1K5d2/Dx8TGaNWtmbNu2rdBz8ucpw19++aXRpk0bw8/PzwgPDzeef/55Y/PmzYYkm/2Lmn79yiuvFPiOkoyJEyfatJ04ccIYNGiQERYWZnh5eRmVKlUyevToYXz00Uc2233zzTdGx44dDV9fX6NSpUrGyy+/bCxdutSu6dd/9q9//cuQZPTr16/Aut9++8147LHHjNtuu80ICAgwOnfubBw5csSoVq2a9fz/+XwV9nfoZs5lx44djYYNGxbo86/n2DD++Dv1f//3f0ZERITh5eVlhIWFGX379jVOnDhhs93ixYuNFi1aGH5+fkaZMmWMxo0bG88//7xx9uxZ+04YUAwshuHkfxYBuCVZLBbFxMQUegsCAFyFMTIAAMC0CDIAAMC0CDIAAMC0mLUEwC4MpwPgjrgiAwAATIsgAwAATOuWv7WUn5+vs2fPqkyZMgVengYAANyTYRi6cOGCwsPDr/mS3Fs+yJw9e1ZVqlRxdRkAAOAGnD59WpUrVy5y/S0fZMqUKSPpjxMRGBjo4moAAIA9MjMzVaVKFevv8aLc8kHm6u2kwMBAggwAACZzvWEhDPYFAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACmRZABAACm5enqAsys+tiNJX7MkzO6l/gxAQBwV1yRAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApkWQAQAApuXyIHPmzBk98sgjCg4Olp+fnxo3bqw9e/ZY1xuGoQkTJqhixYry8/NTVFSUjh8/7sKKAQCAu3BpkPntt9/Url07eXl56dNPP9Xhw4f12muvqVy5ctZtZs2apXnz5mnRokXatWuX/P391blzZ2VnZ7uwcgAA4A48XXnwmTNnqkqVKlq2bJm1LSIiwvpnwzA0d+5cvfjii+rZs6ck6b333lNoaKg+/vhj9e/fv8RrBgAA7sOlV2Q++eQTtWzZUg8++KBCQkLUrFkzLVmyxLo+OTlZKSkpioqKsrYFBQWpdevWSkxMdEXJAADAjbg0yPzwww9auHChateurc2bN+vZZ5/VsGHD9O6770qSUlJSJEmhoaE2+4WGhlrX/VVOTo4yMzNtFgAAcGty6a2l/Px8tWzZUtOmTZMkNWvWTIcOHdKiRYsUHR19Q31Onz5dkyZNcmaZAADATbn0ikzFihXVoEEDm7b69evr1KlTkqSwsDBJUmpqqs02qamp1nV/NW7cOGVkZFiX06dPF0PlAADAHbg0yLRr105Hjx61aTt27JiqVasm6Y+Bv2FhYYqPj7euz8zM1K5duxQZGVlonz4+PgoMDLRZAADArcmlt5ZGjhyptm3batq0aerXr592796txYsXa/HixZIki8WiESNGaMqUKapdu7YiIiI0fvx4hYeHq1evXq4sHQAAuAGXBpnbb79d69at07hx4zR58mRFRERo7ty5GjhwoHWb559/XllZWXr66aeVnp6uO+64Q5s2bZKvr68LKwcAAO7AYhiG4eoiilNmZqaCgoKUkZHh9NtM1cdudGp/9jg5o3uJHxMAgJJm7+9vl7+iAAAA4EYRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGm5NMi89NJLslgsNku9evWs67OzsxUTE6Pg4GAFBASoT58+Sk1NdWHFAADAnbj8ikzDhg117tw567Jjxw7rupEjR2r9+vVavXq1EhISdPbsWfXu3duF1QIAAHfi6fICPD0VFhZWoD0jI0NLly7VypUr1alTJ0nSsmXLVL9+fe3cuVNt2rQp6VIBAICbcfkVmePHjys8PFw1atTQwIEDderUKUnS3r17lZubq6ioKOu29erVU9WqVZWYmFhkfzk5OcrMzLRZAADArcmlQaZ169Zavny5Nm3apIULFyo5OVnt27fXhQsXlJKSIm9vb5UtW9Zmn9DQUKWkpBTZ5/Tp0xUUFGRdqlSpUszfAgAAuIpLby117drV+ucmTZqodevWqlatmj788EP5+fndUJ/jxo1TbGys9XNmZiZhBgCAW5TLby39WdmyZVWnTh19//33CgsL0+XLl5Wenm6zTWpqaqFjaq7y8fFRYGCgzQIAAG5NbhVkLl68qBMnTqhixYpq0aKFvLy8FB8fb11/9OhRnTp1SpGRkS6sEgAAuAuX3loaPXq07rvvPlWrVk1nz57VxIkT5eHhoQEDBigoKEhPPPGEYmNjVb58eQUGBmro0KGKjIxkxhIAAJB0k0EmOztbvr6+N7z/Tz/9pAEDBuiXX35RhQoVdMcdd2jnzp2qUKGCJGnOnDkqVaqU+vTpo5ycHHXu3FkLFiy4mZIBAMAtxGIYhuHIDvn5+Zo6daoWLVqk1NRUHTt2TDVq1ND48eNVvXp1PfHEE8VV6w3JzMxUUFCQMjIynD5epvrYjU7tzx4nZ3Qv8WMCAFDS7P397fAYmSlTpmj58uWaNWuWvL29re2NGjXS22+/fWPVAgAA3ACHg8x7772nxYsXa+DAgfLw8LC2N23aVEeOHHFqcQAAANficJA5c+aMatWqVaA9Pz9fubm5TikKAADAHg4HmQYNGuh///tfgfaPPvpIzZo1c0pRAAAA9nB41tKECRMUHR2tM2fOKD8/X2vXrtXRo0f13nvvacOGDcVRIwAAQKEcviLTs2dPrV+/Xlu3bpW/v78mTJig7777TuvXr9c999xTHDUCAAAU6oaeI9O+fXtt2bLF2bUAAAA4xOErMklJSdq1a1eB9l27dmnPnj1OKQoAAMAeDgeZmJgYnT59ukD7mTNnFBMT45SiAAAA7OFwkDl8+LCaN29eoL1Zs2Y6fPiwU4oCAACwh8NBxsfHR6mpqQXaz507J09Pl76DEgAA/M04HGTuvfdejRs3ThkZGda29PR0vfDCC8xaAgAAJcrhSyivvvqqOnTooGrVqlkfgHfgwAGFhobq/fffd3qBAAAARXE4yFSqVEnffPONVqxYoa+//lp+fn567LHHNGDAAHl5eRVHjQAAAIW6oUEt/v7+evrpp51dCwAAgENuKMgcP35c27ZtU1pamvLz823WTZgwwSmFAQAAXI/DQWbJkiV69tlnddtttyksLEwWi8W6zmKxEGQAAECJcTjITJkyRVOnTtWYMWOKox4AAAC7OTz9+rffftODDz5YHLUAAAA4xOEg8+CDD+qzzz4rjloAAAAc4vCtpVq1amn8+PHauXOnGjduXGDK9bBhw5xWHAAAwLVYDMMwHNkhIiKi6M4sFv3www83XZQzZWZmKigoSBkZGQoMDHRq39XHbnRqf/Y4OaN7iR8TAICSZu/vb4evyCQnJ99UYQAAAM7i8BgZAAAAd3FDD8T76aef9Mknn+jUqVO6fPmyzbrZs2c7pTAAAIDrcTjIxMfH6/7771eNGjV05MgRNWrUSCdPnpRhGGrevHlx1AgAAFAoh28tjRs3TqNHj9bBgwfl6+urNWvW6PTp0+rYsSPPlwEAACXK4SDz3XffadCgQZIkT09P/f777woICNDkyZM1c+ZMpxcIAABQFIeDjL+/v3VcTMWKFXXixAnruvPnzzuvMgAAgOtweIxMmzZttGPHDtWvX1/dunXTqFGjdPDgQa1du1Zt2rQpjhoBAAAK5XCQmT17ti5evChJmjRpki5evKgPPvhAtWvXZsYSAAAoUQ4HmRo1alj/7O/vr0WLFjm1IAAAAHvxQDwAAGBadl2RKV++vI4dO6bbbrtN5cqVk8ViKXLbX3/91WnFAQAAXItdQWbOnDkqU6aMJGnu3LnFWQ8AAIDd7Aoy0dHRkqQrV67IYrGoc+fOCg0NLdbCAAAArsehMTKenp765z//qezs7OKqBwAAwG4OD/Zt1aqV9u/fXxy1AAAAOMTh6dfPPfecRo0apZ9++kktWrSQv7+/zfomTZo4rTgAAIBrcTjI9O/fX5I0bNgwa5vFYpFhGLJYLMrLy3NedQAAANfgcJBJTk4ujjoAAAAc5nCQqVatWnHUAQAA4DCHg8xVhw8f1qlTp6xvwr7q/vvvv+miAAAA7OHwrKUffvhBTZs2VaNGjdS9e3f16tVLvXr10gMPPKAHHnjghguZMWOGLBaLRowYYW3Lzs5WTEyMgoODFRAQoD59+ig1NfWGjwEAAG4tDgeZ4cOHKyIiQmlpaSpdurS+/fZbffHFF2rZsqW2b99+Q0UkJSXprbfeKjDjaeTIkVq/fr1Wr16thIQEnT17Vr17976hYwAAgFuPw0EmMTFRkydP1m233aZSpUqpVKlSuuOOOzR9+nSbmUz2unjxogYOHKglS5aoXLly1vaMjAwtXbpUs2fPVqdOndSiRQstW7ZMX331lXbu3OnwcQAAwK3H4SCTl5dnfe/SbbfdprNnz0r6YxDw0aNHHS4gJiZG3bt3V1RUlE373r17lZuba9Ner149Va1aVYmJiQ4fBwAA3HocHuzbqFEjff3114qIiFDr1q01a9YseXt7a/HixapRo4ZDfa1atUr79u1TUlJSgXUpKSny9vZW2bJlbdpDQ0OVkpJSZJ85OTnKycmxfs7MzHSoJgAAYB4OX5F58cUXlZ+fL0maPHmykpOT1b59e/33v//VvHnz7O7n9OnTGj58uFasWCFfX19HyyjS9OnTFRQUZF2qVKnitL4BAIB7sTvItGzZUosWLVJkZKR1wG2tWrV05MgRnT9/XmlpaerUqZPdB967d6/S0tLUvHlzeXp6ytPTUwkJCZo3b548PT0VGhqqy5cvKz093Wa/1NRUhYWFFdnvuHHjlJGRYV1Onz5td00AAMBc7A4yTZs21fPPP6+KFStq0KBBNjOUypcvL4vF4tCB7777bh08eFAHDhywLi1bttTAgQOtf/by8lJ8fLx1n6NHj+rUqVOKjIwssl8fHx8FBgbaLAAA4NZk9xiZpUuX6o033tCHH36o5cuX6+6771ZERIQef/xxRUdHq1KlSg4duEyZMmrUqJFNm7+/v4KDg63tTzzxhGJjY1W+fHkFBgZq6NChioyMVJs2bRw6FgAAuDU5NEamdOnSGjx4sLZv365jx46pf//+euutt1S9enV1795da9eudWpxc+bMUY8ePdSnTx916NBBYWFhTj8GAAAwL4thGMbNdGAYhtasWaNnnnlG6enpbvf268zMTAUFBSkjI8Ppt5mqj93o1P7scXJG9xI/JgAAJc3e3983/K4lSdq+fbuWLVumNWvWyNPTU0899dTNdAcAAOAQh4PMTz/9pOXLl2v58uX64Ycf1L59ey1YsEAPPvig/Pz8iqNGAACAQtkdZD788EO98847io+PV0hIiKKjo/X444+rVq1axVkfAABAkewOMo888oi6d++udevWqVu3bipVyuFn6QEAADiV3UHmp59+UkhISHHWAgAA4BC7L6sQYgAAgLvh/hAAADAtggwAADAtggwAADAtggwAADAtu2YtlStXzu63W//66683VRAAAIC97Aoyc+fOtf75l19+0ZQpU9S5c2dFRkZKkhITE7V582aNHz++WIoEAAAojMMvjezTp4/uuusuDRkyxKb9zTff1NatW/Xxxx87s76bxksjAQAwH3t/fzs8Rmbz5s3q0qVLgfYuXbpo69atjnYHAABwwxwOMsHBwYqLiyvQHhcXp+DgYKcUBQAAYA+H3349adIkPfnkk9q+fbtat24tSdq1a5c2bdqkJUuWOL1AAACAojgcZAYPHqz69etr3rx5Wrt2rSSpfv362rFjhzXYAAAAlASHg4wktW7dWitWrHB2LQAAAA65oQfinThxQi+++KIefvhhpaWlSZI+/fRTffvtt04tDgAA4FocDjIJCQlq3Lixdu3apTVr1ujixYuSpK+//loTJ050eoEAAABFcTjIjB07VlOmTNGWLVvk7e1tbe/UqZN27tzp1OIAAACuxeEgc/DgQT3wwAMF2kNCQnT+/HmnFAUAAGAPh4NM2bJlde7cuQLt+/fvV6VKlZxSFAAAgD0cDjL9+/fXmDFjlJKSIovFovz8fH355ZcaPXq0Bg0aVBw1AgAAFMrhIDNt2jTVq1dPVapU0cWLF9WgQQN16NBBbdu21YsvvlgcNQIAABTK4efIeHt7a8mSJRo/frwOHTqkixcvqlmzZqpdu3Zx1AcAAFCkG3ogniRVrVpVVatWdWYtAAAADnE4yOTl5Wn58uWKj49XWlqa8vPzbdZ//vnnTisOAADgWhwOMsOHD9fy5cvVvXt3NWrUSBaLpTjqAgAAuC6Hg8yqVav04Ycfqlu3bsVRDwAAgN0cnrXk7e2tWrVqFUctAAAADnE4yIwaNUqvv/66DMMojnoAAADs5vCtpR07dmjbtm369NNP1bBhQ3l5edmsX7t2rdOKAwAAuBaHg0zZsmULfdcSAABASXM4yCxbtqw46gAAAHCYw2NkAAAA3IVdV2SaN2+u+Ph4lStXTs2aNbvms2P27dvntOIAAACuxa4g07NnT/n4+EiSevXqVZz1AAAA2M2uIDNx4sRC/wwAAOBKjJEBAACmdUMvjZwzZ44+/PBDnTp1SpcvX7ZZ/+uvvzqtOAAAgGtx+IrMpEmTNHv2bD300EPKyMhQbGysevfurVKlSumll14qhhIBAAAK53CQWbFihZYsWaJRo0bJ09NTAwYM0Ntvv60JEyZo586dxVEjAABAoRwOMikpKWrcuLEkKSAgQBkZGZKkHj16aOPGjc6tDgAA4BocDjKVK1fWuXPnJEk1a9bUZ599JklKSkqyTtG218KFC9WkSRMFBgYqMDBQkZGR+vTTT63rs7OzFRMTo+DgYAUEBKhPnz5KTU11tGQAAHCLcjjIPPDAA4qPj5ckDR06VOPHj1ft2rU1aNAgPf744w71VblyZc2YMUN79+7Vnj171KlTJ/Xs2VPffvutJGnkyJFav369Vq9erYSEBJ09e1a9e/d2tGQAAHCLshiGYdxMB4mJiUpMTFTt2rV133333XRB5cuX1yuvvKK+ffuqQoUKWrlypfr27StJOnLkiOrXr6/ExES1adPGrv4yMzMVFBSkjIwMBQYG3nR9f1Z9bMnfSjs5o3uJHxMAgJJm7+9vh6df/1VkZKQiIyNvthvl5eVp9erVysrKUmRkpPbu3avc3FxFRUVZt6lXr56qVq16zSCTk5OjnJwc6+fMzMybrg0AALgnu4LMJ598YneH999/v0MFHDx4UJGRkcrOzlZAQIDWrVunBg0a6MCBA/L29lbZsmVttg8NDVVKSkqR/U2fPl2TJk1yqAYAAGBOdgUZe9+vZLFYlJeX51ABdevW1YEDB5SRkaGPPvpI0dHRSkhIcKiPPxs3bpxiY2OtnzMzM1WlSpUb7g8AALgvu4JMfn5+sRXg7e2tWrVqSZJatGihpKQkvf7663rooYd0+fJlpaen21yVSU1NVVhYWJH9+fj4ODx7CgAAmJPbvWspPz9fOTk5atGihby8vKwzpCTp6NGjOnXqlFPG5AAAAPO7ocG+8fHxmjNnjr777jtJUv369TVixAibgbn2GDdunLp27aqqVavqwoULWrlypbZv367NmzcrKChITzzxhGJjY1W+fHkFBgZq6NChioyMtHvGEgAAuLU5fEVmwYIF6tKli8qUKaPhw4dr+PDhCgwMVLdu3TR//nyH+kpLS9OgQYNUt25d3X333UpKStLmzZt1zz33SJLmzJmjHj16qE+fPurQoYPCwsK0du1aR0sGAAC3KIefI1O5cmWNHTtWQ4YMsWmfP3++pk2bpjNnzji1wJvFc2QAADAfe39/O3xFJj09XV26dCnQfu+991rfuwQAAFASHA4y999/v9atW1egPS4uTj169HBKUQAAAPZweLBvgwYNNHXqVG3fvt06e2jnzp368ssvNWrUKM2bN8+67bBhw5xXKQAAwF84PEYmIiLCvo4tFv3www83VJQzMUYGAADzKbZ3LSUnJ99UYQAAAM7i8BiZ7OzsItedO3fupooBAABwhMNBpnnz5jpw4ECB9jVr1qhJkybOqAkAAMAuDgeZO++8U23atNHMmTMlSVlZWRo8eLAeffRRvfDCC04vEAAAoCgOj5FZsGCBunfvrieffFIbNmzQuXPnFBAQoN27d6tRo0bFUSMAAEChbuhdS127dlXv3r21cOFCeXp6av369YQYAABQ4hy+tXTixAlFRkZqw4YN2rx5s55//nndf//9ev7555Wbm1scNQIAABTK4SDzj3/8QxEREfr66691zz33aMqUKdq2bZvWrl2rVq1aFUeNAAAAhbqht1+vWrVKZcuWtba1bdtW+/fvV/PmzZ1ZGwAAwDU5HGQeffRRSdLly5d19OhRXblyRZJUpkwZLV261LnVAQAAXIPDQeb333/XE088odKlS6thw4Y6deqUJGno0KHWKdkAAAAlweEgM3bsWH399dfavn27fH19re1RUVFatWqVU4sDAAC4FoenX3/88cf64IMP1KZNG1ksFmt7w4YNdeLECacWBwAAcC0OX5H5+eefFRISUqA9KyvLJtgAAAAUN4eDTMuWLbVx40br56vh5e2331ZkZKTzKgMAALgOh28tTZs2TV27dtXhw4d15coVvf766zp8+LC++uorJSQkFEeNAAAAhXL4iswdd9yhAwcO6MqVK2rcuLE+++wzhYSEKDExUS1atCiOGgEAAAp1Q+9aqlmzppYsWeLsWgAAABzi8BUZAAAAd0GQAQAApkWQAQAApkWQAQAApnVDg32vOn36tCSpSpUqTikGN6f62I3X38iJTs7oXqLHAwDgrxy+InPlyhWNHz9eQUFBql69uqpXr66goCC9+OKLys3NLY4aAQAACuXwFZmhQ4dq7dq1mjVrlvVJvomJiXrppZf0yy+/aOHChU4vEgAAoDAOB5mVK1dq1apV6tq1q7WtSZMmqlKligYMGECQAQAAJcbhW0s+Pj6qXr16gfaIiAh5e3s7oyYAAAC7OBxkhgwZopdfflk5OTnWtpycHE2dOlVDhgxxanEAAADX4vCtpf379ys+Pl6VK1dW06ZNJUlff/21Ll++rLvvvlu9e/e2brt27VrnVQoAAPAXDgeZsmXLqk+fPjZtTL8GAACu4HCQWbZsWXHUAQAA4LAbfiDezz//rKNHj0qS6tatqwoVKjitKAAAAHs4PNg3KytLjz/+uCpWrKgOHTqoQ4cOCg8P1xNPPKFLly4VR40AAACFcjjIxMbGKiEhQevXr1d6errS09MVFxenhIQEjRo1qjhqBAAAKJTDt5bWrFmjjz76SHfeeae1rVu3bvLz81O/fv14IB4AACgxDl+RuXTpkkJDQwu0h4SEcGsJAACUKIeDTGRkpCZOnKjs7Gxr2++//65JkyZZ370EAABQEhy+tTR37lx16dKlwAPxfH19tXnzZqcXCAAAUBSHg0zjxo11/PhxrVixQkeOHJEkDRgwQAMHDpSfn5/TCwQAACiKQ7eWcnNzVbNmTf3444966qmn9Nprr+m1117Tk08+eUMhZvr06br99ttVpkwZhYSEqFevXtZn01yVnZ2tmJgYBQcHKyAgQH369FFqaqrDxwIAALceh4KMl5eXzdiYm5WQkKCYmBjt3LlTW7ZsUW5uru69915lZWVZtxk5cqTWr1+v1atXKyEhQWfPnrV5nxMAAPj7cvjWUkxMjGbOnKm3335bnp43/GBgSdKmTZtsPi9fvlwhISHau3evOnTooIyMDC1dulQrV65Up06dJP3xioT69etr586datOmzU0dHwAAmJvDSSQpKUnx8fH67LPP1LhxY/n7+9usv5k3XmdkZEiSypcvL0nau3evcnNzFRUVZd2mXr16qlq1qhITEwkyAAD8zTnl7dfOkJ+frxEjRqhdu3Zq1KiRJCklJUXe3t4qW7aszbahoaFKSUkptJ+cnBzl5ORYP2dmZjq9VgAA4B7c5u3XMTExOnTokHbs2HFT/UyfPl2TJk1yUlUAAMCd2T3YNz8/XzNnzlS7du10++23a+zYsfr999+dUsSQIUO0YcMGbdu2TZUrV7a2h4WF6fLly0pPT7fZPjU1VWFhYYX2NW7cOGVkZFiX06dPO6VGAADgfuwOMlOnTtULL7yggIAAVapUSa+//rpiYmJu6uCGYWjIkCFat26dPv/8c0VERNisb9Gihby8vBQfH29tO3r0qE6dOlXkU4R9fHwUGBhoswAAgFuT3beW3nvvPS1YsEDPPPOMJGnr1q3q3r273n77bZUq5fCbDiT9cTtp5cqViouLU5kyZazjXoKCguTn56egoCA98cQTio2NVfny5RUYGKihQ4cqMjKSgb4AAMD+IHPq1Cl169bN+jkqKkoWi0Vnz561uR3kiKtvyv7zm7SlP8bhDB48WJI0Z84clSpVSn369FFOTo46d+6sBQsW3NDxAADArcXuIHPlyhX5+vratHl5eSk3N/eGD24YxnW38fX11fz58zV//vwbPg4AALg12R1kDMPQ4MGD5ePjY23Lzs7WP//5T5tnydzMc2QAAAAcYXeQiY6OLtD2yCOPOLUYAAAAR9gdZIrr+TEAAAA36samGwEAALgBggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtggwAADAtlwaZL774Qvfdd5/Cw8NlsVj08ccf26w3DEMTJkxQxYoV5efnp6ioKB0/ftw1xQIAALfj0iCTlZWlpk2bav78+YWunzVrlubNm6dFixZp165d8vf3V+fOnZWdnV3ClQIAAHfk6cqDd+3aVV27di10nWEYmjt3rl588UX17NlTkvTee+8pNDRUH3/8sfr371+SpQIAADfktmNkkpOTlZKSoqioKGtbUFCQWrdurcTExCL3y8nJUWZmps0CAABuTW4bZFJSUiRJoaGhNu2hoaHWdYWZPn26goKCrEuVKlWKtU4AAOA6bhtkbtS4ceOUkZFhXU6fPu3qkgAAQDFx2yATFhYmSUpNTbVpT01Nta4rjI+PjwIDA20WAABwa3LbIBMREaGwsDDFx8db2zIzM7Vr1y5FRka6sDIAAOAuXDpr6eLFi/r++++tn5OTk3XgwAGVL19eVatW1YgRIzRlyhTVrl1bERERGj9+vMLDw9WrVy/XFQ0AANyGS4PMnj17dNddd1k/x8bGSpKio6O1fPlyPf/888rKytLTTz+t9PR03XHHHdq0aZN8fX1dVTIAAHAjLg0yd955pwzDKHK9xWLR5MmTNXny5BKsCgAAmIXbjpEBAAC4HoIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLYIMAAAwLU9XF4BbU/WxG0v0eCdndC/R4wEA3ANXZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkRZAAAgGkxawm3PGZQAcCtiysyAADAtAgyAADAtAgyAADAtAgyAADAtAgyAADAtAgyAADAtJh+DZSgkp4KLjEdHMCtjSsyAADAtAgyAADAtAgyAADAtAgyAADAtBjsC/yN8R4qAGZniisy8+fPV/Xq1eXr66vWrVtr9+7dri4JAAC4AbcPMh988IFiY2M1ceJE7du3T02bNlXnzp2Vlpbm6tIAAICLuf2tpdmzZ+upp57SY489JklatGiRNm7cqHfeeUdjx451cXUAnIXbXABuhFsHmcuXL2vv3r0aN26cta1UqVKKiopSYmKiCysDcCsjVAHm4dZB5vz588rLy1NoaKhNe2hoqI4cOVLoPjk5OcrJybF+zsjIkCRlZmY6vb78nEtO7/N6rvU9SroeaimcO9UiuVc91FK4a9XSaOLmEqxEOjSpc5HrSroW6dr14NZ29f8LwzCuvaHhxs6cOWNIMr766iub9n/9619Gq1atCt1n4sSJhiQWFhYWFhaWW2A5ffr0NbOCW1+Rue222+Th4aHU1FSb9tTUVIWFhRW6z7hx4xQbG2v9nJ+fr19//VXBwcGyWCzFWq+9MjMzVaVKFZ0+fVqBgYGuLsdtcF4Kx3kpGuemcJyXonFuCueO58UwDF24cEHh4eHX3M6tg4y3t7datGih+Ph49erVS9IfwSQ+Pl5DhgwpdB8fHx/5+PjYtJUtW7aYK70xgYGBbvMXxp1wXgrHeSka56ZwnJeicW4K527nJSgo6LrbuHWQkaTY2FhFR0erZcuWatWqlebOnausrCzrLCYAAPD35fZB5qGHHtLPP/+sCRMmKCUlRf/4xz+0adOmAgOAAQDA34/bBxlJGjJkSJG3kszIx8dHEydOLHAL7O+O81I4zkvRODeF47wUjXNTODOfF4thXG9eEwAAgHty+1cUAAAAFIUgAwAATIsgAwAATIsgAwAATIsgU8Lmz5+v6tWry9fXV61bt9bu3btdXZLLTZ8+XbfffrvKlCmjkJAQ9erVS0ePHnV1WW5nxowZslgsGjFihKtLcbkzZ87okUceUXBwsPz8/NS4cWPt2bPH1WW5XF5ensaPH6+IiAj5+fmpZs2aevnll6//rppb0BdffKH77rtP4eHhslgs+vjjj23WG4ahCRMmqGLFivLz81NUVJSOHz/ummJL0LXOS25ursaMGaPGjRvL399f4eHhGjRokM6ePeu6gu1AkClBH3zwgWJjYzVx4kTt27dPTZs2VefOnZWWlubq0lwqISFBMTEx2rlzp7Zs2aLc3Fzde++9ysrKcnVpbiMpKUlvvfWWmjRp4upSXO63335Tu3bt5OXlpU8//VSHDx/Wa6+9pnLlyrm6NJebOXOmFi5cqDfffFPfffedZs6cqVmzZumNN95wdWklLisrS02bNtX8+fMLXT9r1izNmzdPixYt0q5du+Tv76/OnTsrOzu7hCstWdc6L5cuXdK+ffs0fvx47du3T2vXrtXRo0d1//33u6BSBzjj5Y6wT6tWrYyYmBjr57y8PCM8PNyYPn26C6tyP2lpaYYkIyEhwdWluIULFy4YtWvXNrZs2WJ07NjRGD58uKtLcqkxY8YYd9xxh6vLcEvdu3c3Hn/8cZu23r17GwMHDnRRRe5BkrFu3Trr5/z8fCMsLMx45ZVXrG3p6emGj4+P8Z///McFFbrGX89LYXbv3m1IMn788ceSKeoGcEWmhFy+fFl79+5VVFSUta1UqVKKiopSYmKiCytzPxkZGZKk8uXLu7gS9xATE6Pu3bvb/N35O/vkk0/UsmVLPfjggwoJCVGzZs20ZMkSV5flFtq2bav4+HgdO3ZMkvT1119rx44d6tq1q4srcy/JyclKSUmx+X8qKChIrVu35ufxX2RkZMhisbjtOwslkzzZ91Zw/vx55eXlFXi1QmhoqI4cOeKiqtxPfn6+RowYoXbt2qlRo0auLsflVq1apX379ikpKcnVpbiNH374QQsXLlRsbKxeeOEFJSUladiwYfL29lZ0dLSry3OpsWPHKjMzU/Xq1ZOHh4fy8vI0depUDRw40NWluZWUlBRJKvTn8dV1kLKzszVmzBgNGDDArV4k+VcEGbiVmJgYHTp0SDt27HB1KS53+vRpDR8+XFu2bJGvr6+ry3Eb+fn5atmypaZNmyZJatasmQ4dOqRFixb97YPMhx9+qBUrVmjlypVq2LChDhw4oBEjRig8PPxvf27gmNzcXPXr10+GYWjhwoWuLueauLVUQm677TZ5eHgoNTXVpj01NVVhYWEuqsq9DBkyRBs2bNC2bdtUuXJlV5fjcnv37lVaWpqaN28uT09PeXp6KiEhQfPmzZOnp6fy8vJcXaJLVKxYUQ0aNLBpq1+/vk6dOuWiitzHv/71L40dO1b9+/dX48aN9eijj2rkyJGaPn26q0tzK1d/5vLzuHBXQ8yPP/6oLVu2uPXVGIkgU2K8vb3VokULxcfHW9vy8/MVHx+vyMhIF1bmeoZhaMiQIVq3bp0+//xzRUREuLokt3D33Xfr4MGDOnDggHVp2bKlBg4cqAMHDsjDw8PVJbpEu3btCkzPP3bsmKpVq+aiitzHpUuXVKqU7Y91Dw8P5efnu6gi9xQREaGwsDCbn8eZmZnatWvX3/7n8dUQc/z4cW3dulXBwcGuLum6uLVUgmJjYxUdHa2WLVuqVatWmjt3rrKysvTYY4+5ujSXiomJ0cqVKxUXF6cyZcpY71EHBQXJz8/PxdW5TpkyZQqME/L391dwcPDfevzQyJEj1bZtW02bNk39+vXT7t27tXjxYi1evNjVpbncfffdp6lTp6pq1apq2LCh9u/fr9mzZ+vxxx93dWkl7uLFi/r++++tn5OTk3XgwAGVL19eVatW1YgRIzRlyhTVrl1bERERGj9+vMLDw9WrVy/XFV0CrnVeKlasqL59+2rfvn3asGGD8vLyrD+Py5cvL29vb1eVfW2unjb1d/PGG28YVatWNby9vY1WrVoZO3fudHVJLiep0GXZsmWuLs3tMP36D+vXrzcaNWpk+Pj4GPXq1TMWL17s6pLcQmZmpjF8+HCjatWqhq+vr1GjRg3j//7v/4ycnBxXl1bitm3bVujPlejoaMMw/piCPX78eCM0NNTw8fEx7r77buPo0aOuLboEXOu8JCcnF/nzeNu2ba4uvUgWw/gbPvIRAADcEhgjAwAATIsgAwAATIsgAwAATIsgAwAATIsgAwAATIsgAwAATIsgAwAATIsgA8CtGYahp59+WuXLl5fFYtGBAwdcXRIAN0KQAXBNgwcPlsVikcVikbe3t2rVqqXJkyfrypUr1m0Mw9DixYvVunVrBQQEqGzZsmrZsqXmzp2rS5cu2fT3008/ydvb2+7XLGzatEnLly/Xhg0bdO7cOae9nmHw4MG3/OPogb8DggyA6+rSpYvOnTun48ePa9SoUXrppZf0yiuvWNc/+uijGjFihHr27Klt27bpwIEDGj9+vOLi4vTZZ5/Z9LV8+XL169fP+pK+6zlx4oQqVqyotm3bKiwsTJ6e7vWKuLy8PF7KCLiSa9+QAMDdRUdHGz179rRpu+eee4w2bdoYhmEYH3zwgSHJ+Pjjjwvsm5+fb6Snp9t8rlGjhrFp0yZjzJgxxlNPPXXdY+tP73upVq2aYRiGkZeXZ0ybNs2oXr264evrazRp0sRYvXq1db8rV64Yjz/+uHV9nTp1jLlz51rXT5w4sdB3yVx9D81vv/1m3Xb//v2GJCM5OdkwDMNYtmyZERQUZMTFxRn169c3PDw8jOTkZCM7O9sYNWqUER4ebpQuXdpo1aqVW7+fBrhVuNc/bQCYgp+fn3755RdJ0ooVK1S3bl317NmzwHYWi0VBQUHWz9u2bdOlS5cUFRWlSpUqqW3btpozZ478/f0LPc7rr7+umjVravHixUpKSpKHh4ckafr06fr3v/+tRYsWqXbt2vriiy/0yCOPqEKFCurYsaPy8/NVuXJlrV69WsHBwfrqq6/09NNPq2LFiurXr59Gjx6t7777TpmZmVq2bJmkP97u+9VXX9n1/S9duqSZM2fq7bffVnBwsEJCQjRkyBAdPnxYq1atUnh4uNatW6cuXbro4MGDql27tkPnF4D9CDIA7GYYhuLj47V582YNHTpUknT8+HHVrVvXrv2XLl2q/v37y8PDQ40aNVKNGjW0evVqDR48uNDtg4KCVKZMGXl4eCgsLEySlJOTo2nTpmnr1q2KjIyUJNWoUUM7duzQW2+9pY4dO8rLy0uTJk2y9hMREaHExER9+OGH6tevnwICAuTn56ecnBxrv47Izc3VggUL1LRpU0nSqVOntGzZMp06dUrh4eGSpNGjR2vTpk1atmyZpk2b5vAxANiHIAPgujZs2KCAgADl5uYqPz9fDz/8sF566SVJf4Qbe6Snp2vt2rXasWOHte2RRx7R0qVLiwwyhfn+++916dIl3XPPPTbtly9fVrNmzayf58+fr3feeUenTp3S77//rsuXL+sf//iH3ce5Fm9vbzVp0sT6+eDBg8rLy1OdOnVstsvJyVFwcLBTjgmgcAQZANd11113aeHChfL29lZ4eLjNgNs6deroyJEj1+1j5cqVys7OVuvWra1thmEoPz9fx44dKxACinLx4kVJ0saNG1WpUiWbdT4+PpKkVatWafTo0XrttdcUGRmpMmXK6JVXXrnu4OJSpUpZ67oqNze3wHZ+fn6yWCw2NXl4eGjv3r3W219XBQQE2PW9ANwYggyA6/L391etWrUKXffwww+rf//+iouLKzBOxjAMZWZmKigoSEuXLtWoUaMKXH157rnn9M4772jGjBl21dKgQQP5+Pjo1KlT6tixY6HbfPnll2rbtq2ee+45a9uJEydstvH29lZeXp5NW4UKFSRJ586dU7ly5STJrufWNGvWTHl5eUpLS1P79u3t+h4AnIPp1wBuSr9+/fTQQw9pwIABmjZtmvbs2aMff/xRGzZsUFRUlHU69r59+/Tkk0+qUaNGNsuAAQP07rvv2jyX5lrKlCmj0aNHa+TIkXr33Xd14sQJ7du3T2+88YbeffddSVLt2rW1Z88ebd68WceOHdP48eOVlJRk00/16tX1zTff6OjRozp//rxyc3NVq1YtValSRS+99JKOHz+ujRs36rXXXrtuTXXq1NHAgQM1aNAgrV27VsnJydq9e7emT5+ujRs3On5SAdjPpXOmALi9wqZf/1VeXp6xcOFC4/bbbzdKly5tBAYGGi1atDBef/1149KlS8aQIUOMBg0aFLrvuXPnjFKlShlxcXGFrp8zZ4512vVV+fn5xty5c426desaXl5eRoUKFYzOnTsbCQkJhmEYRnZ2tjF48GAjKCjIKFu2rPHss88aY8eONZo2bWrtIy0tzbjnnnuMgIAA6/RrwzCMHTt2GI0bNzZ8fX2N9u3bG6tXry50+vVfXb582ZgwYYJRvXp1w8vLy6hYsaLxwAMPGN988801zx2Am2MxDDtH6gEAALgZbi0BAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADTIsgAAADT+n/7hdDGCBVgkAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "\n",
    "pca.fit(df_all[relevant_z_scores])\n",
    "features = range(pca.n_components_)\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "ax.bar(features, pca.explained_variance_ratio_ * 100)\n",
    "ax.set_xlabel(\"PCA feature\")\n",
    "ax.set_ylabel(\"Prop explained Variance\")\n",
    "ax.set_title(\"PCA Explained Variance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_components = pca.fit_transform(df_all[relevant_z_scores])\n",
    "fpc = pca_components[:, 0]\n",
    "\n",
    "df_all.loc[:, 'PCA1'] = fpc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Group</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Standard Deviaton</th>\n",
       "      <th>Variance</th>\n",
       "      <th>Max</th>\n",
       "      <th>Min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accepted</td>\n",
       "      <td>0.532449</td>\n",
       "      <td>3.030683</td>\n",
       "      <td>9.185040</td>\n",
       "      <td>10.560854</td>\n",
       "      <td>-7.014653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Rejected</td>\n",
       "      <td>-0.532449</td>\n",
       "      <td>2.651317</td>\n",
       "      <td>7.029483</td>\n",
       "      <td>19.709713</td>\n",
       "      <td>-7.014653</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Group      Mean  Standard Deviaton  Variance        Max       Min\n",
       "0  Accepted  0.532449           3.030683  9.185040  10.560854 -7.014653\n",
       "1  Rejected -0.532449           2.651317  7.029483  19.709713 -7.014653"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chosen_subset = df_all[df_all['chosen'] == 1]\n",
    "rejected_subset = df_all[df_all['chosen'] == 0]\n",
    "\n",
    "PCA_chosen_response_metric_pairs = pd.DataFrame({'response': chosen_subset.loc[:, 'response'], 'score': chosen_subset.loc[:, 'PCA1']}).reset_index(drop=True)\n",
    "PCA_rejected_response_metric_pairs = pd.DataFrame({'response': rejected_subset.loc[:, 'response'], 'score': rejected_subset.loc[:, 'PCA1']}).reset_index(drop=True)\n",
    "\n",
    "#Let's check out the stats on the metric real quick:\n",
    "PCA_metric_distribution = pd.DataFrame({'Group': ['Accepted', 'Rejected'], \n",
    "                        'Mean' : [PCA_chosen_response_metric_pairs['score'].mean(), PCA_rejected_response_metric_pairs['score'].mean()], \n",
    "                        'Standard Deviaton': [np.std(PCA_chosen_response_metric_pairs['score']), np.std(PCA_rejected_response_metric_pairs['score'])],\n",
    "                        'Variance': [np.var(PCA_chosen_response_metric_pairs['score']), np.var(PCA_rejected_response_metric_pairs['score'])],\n",
    "                        'Max': [PCA_chosen_response_metric_pairs['score'].max(), PCA_rejected_response_metric_pairs['score'].max()],\n",
    "                        'Min': [PCA_chosen_response_metric_pairs['score'].min(), PCA_rejected_response_metric_pairs['score'].min()]}) \n",
    "\n",
    "PCA_metric_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pca['accepted_pca1'] = None\n",
    "df_pca['rejected_pca1'] = None\n",
    "\n",
    "for i in range(len(df_pca)):\n",
    "    df_pca.iloc[i, df_pca.columns.get_loc('accepted_pca1')]= PCA_chosen_response_metric_pairs.iloc[np.where(PCA_chosen_response_metric_pairs['response'] == df_pca['chosen'][i])[0][0], -1]\n",
    "    df_pca.iloc[i, df_pca.columns.get_loc('rejected_pca1')] = PCA_rejected_response_metric_pairs.iloc[np.where(PCA_rejected_response_metric_pairs['response'] == df_pca['rejected'][i])[0][0], -1]\n",
    "\n",
    "df_pca['diff_pca1'] = df_pca['accepted_pca1'] - df_pca['rejected_pca1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pca.to_csv(\"pca_metric_df.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weighted PCA "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Might as well do a weighted Linear sum of all the features and their weights, although it really only takes 5 to get 96%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_weighted_pca = df_clean[df_clean['prompt'].isin(df_all['prompt'])].reset_index(drop=True)\n",
    "\n",
    "pca_weighted = PCA()\n",
    "\n",
    "pca_weighted_comps = pca_weighted.fit_transform(df_all[relevant_z_scores])\n",
    "pca_weighted_exp_var_rat = pca_weighted.explained_variance_ratio_\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Group</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Standard Deviaton</th>\n",
       "      <th>Variance</th>\n",
       "      <th>Max</th>\n",
       "      <th>Min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accepted</td>\n",
       "      <td>0.328487</td>\n",
       "      <td>1.971650</td>\n",
       "      <td>3.887404</td>\n",
       "      <td>10.074588</td>\n",
       "      <td>-6.642366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Rejected</td>\n",
       "      <td>-0.328487</td>\n",
       "      <td>1.801635</td>\n",
       "      <td>3.245887</td>\n",
       "      <td>26.583543</td>\n",
       "      <td>-6.642366</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Group      Mean  Standard Deviaton  Variance        Max       Min\n",
       "0  Accepted  0.328487           1.971650  3.887404  10.074588 -6.642366\n",
       "1  Rejected -0.328487           1.801635  3.245887  26.583543 -6.642366"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.loc[:, 'weighted_pca'] = pca_weighted_comps @ pca_weighted_exp_var_rat\n",
    "\n",
    "chosen_subset = df_all[df_all['chosen'] == 1]\n",
    "rejected_subset = df_all[df_all['chosen'] == 0]\n",
    "\n",
    "weighted_PCA_chosen_response_metric_pairs = pd.DataFrame({'response': chosen_subset.loc[:, 'response'], 'score': chosen_subset.loc[:, 'weighted_pca']}).reset_index(drop=True)\n",
    "weighted_PCA_rejected_response_metric_pairs = pd.DataFrame({'response': rejected_subset.loc[:, 'response'], 'score': rejected_subset.loc[:, 'weighted_pca']}).reset_index(drop=True)\n",
    "\n",
    "#Let's check out the stats on the metric real quick:\n",
    "weighted_PCA_metric_distribution = pd.DataFrame({'Group': ['Accepted', 'Rejected'], \n",
    "                        'Mean' : [weighted_PCA_chosen_response_metric_pairs['score'].mean(), weighted_PCA_rejected_response_metric_pairs['score'].mean()], \n",
    "                        'Standard Deviaton': [np.std(weighted_PCA_chosen_response_metric_pairs['score']), np.std(weighted_PCA_rejected_response_metric_pairs['score'])],\n",
    "                        'Variance': [np.var(weighted_PCA_chosen_response_metric_pairs['score']), np.var(weighted_PCA_rejected_response_metric_pairs['score'])],\n",
    "                        'Max': [weighted_PCA_chosen_response_metric_pairs['score'].max(), weighted_PCA_rejected_response_metric_pairs['score'].max()],\n",
    "                        'Min': [weighted_PCA_chosen_response_metric_pairs['score'].min(), weighted_PCA_rejected_response_metric_pairs['score'].min()]}) \n",
    "\n",
    "weighted_PCA_metric_distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_weighted_pca['accepted_wpca'] = None\n",
    "df_weighted_pca['rejected_wpca'] = None\n",
    "\n",
    "for i in range(len(df_pca)):\n",
    "    df_weighted_pca.iloc[i, df_weighted_pca.columns.get_loc('accepted_wpca')]= weighted_PCA_chosen_response_metric_pairs.iloc[np.where(weighted_PCA_chosen_response_metric_pairs['response'] == df_weighted_pca['chosen'][i])[0][0], -1]\n",
    "    df_weighted_pca.iloc[i, df_weighted_pca.columns.get_loc('rejected_wpca')] = weighted_PCA_rejected_response_metric_pairs.iloc[np.where(weighted_PCA_rejected_response_metric_pairs['response'] == df_weighted_pca['rejected'][i])[0][0], -1]\n",
    "\n",
    "df_weighted_pca['diff_wpca'] = df_weighted_pca['accepted_wpca'] - df_weighted_pca['rejected_wpca']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_weighted_pca.to_csv('weighted_pca.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# By Topic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Which topics have the highest entropy/lowest entropy? n_polysylwords, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_uniquewords</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>topic_name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Healthy Lifestyle Exploration</th>\n",
       "      <td>161.855890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Environmental Impact Solutions</th>\n",
       "      <td>142.305855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Urban Exploration</th>\n",
       "      <td>139.566937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marketing Strategy Platform</th>\n",
       "      <td>139.190651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Creative Writing</th>\n",
       "      <td>122.387740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Multi-dimensional Gaming</th>\n",
       "      <td>110.489489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Culinary Creations</th>\n",
       "      <td>104.608108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cultural Expression</th>\n",
       "      <td>96.191434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Childhood Stress and Health</th>\n",
       "      <td>95.169839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Interconnected Systems</th>\n",
       "      <td>90.355932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gaming Economics</th>\n",
       "      <td>82.539906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Analyzing Product Sentiment</th>\n",
       "      <td>67.659574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Movie Analysis and Interpretation</th>\n",
       "      <td>60.561821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gender and Clothing Perceptions</th>\n",
       "      <td>59.498747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Language Analysis</th>\n",
       "      <td>54.618547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Natural Language Processing</th>\n",
       "      <td>37.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   n_uniquewords\n",
       "topic_name                                      \n",
       "Healthy Lifestyle Exploration         161.855890\n",
       "Environmental Impact Solutions        142.305855\n",
       "Urban Exploration                     139.566937\n",
       "Marketing Strategy Platform           139.190651\n",
       "Creative Writing                      122.387740\n",
       "Multi-dimensional Gaming              110.489489\n",
       "Culinary Creations                    104.608108\n",
       "Cultural Expression                    96.191434\n",
       "Childhood Stress and Health            95.169839\n",
       "Interconnected Systems                 90.355932\n",
       "Gaming Economics                       82.539906\n",
       "Analyzing Product Sentiment            67.659574\n",
       "Movie Analysis and Interpretation      60.561821\n",
       "Gender and Clothing Perceptions        59.498747\n",
       "Language Analysis                      54.618547\n",
       "Natural Language Processing            37.333333"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(df_all.groupby('topic_name')['n_uniquewords'].agg('mean')).sort_values(by='n_uniquewords', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Which documents have the highest entropy/lowest entropy? n_polysylwords, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_name</th>\n",
       "      <th>n_uniquewords</th>\n",
       "      <th>response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1730</th>\n",
       "      <td>Creative Writing</td>\n",
       "      <td>424.0</td>\n",
       "      <td>Title: Crescendo of the Heart\\n\\nChapter One -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8349</th>\n",
       "      <td>Environmental Impact Solutions</td>\n",
       "      <td>404.0</td>\n",
       "      <td>Title: The Impact of Artificial Intelligence o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20090</th>\n",
       "      <td>Healthy Lifestyle Exploration</td>\n",
       "      <td>392.0</td>\n",
       "      <td>Title: Electronic Voting Systems: A Glimpse in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2936</th>\n",
       "      <td>Culinary Creations</td>\n",
       "      <td>377.0</td>\n",
       "      <td>What purpose does frosting serve, and how does...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3702</th>\n",
       "      <td>Cultural Expression</td>\n",
       "      <td>373.0</td>\n",
       "      <td>**In the acrid vapors of a renewed and retched...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17937</th>\n",
       "      <td>Urban Exploration</td>\n",
       "      <td>368.0</td>\n",
       "      <td>Hello! I'm here to help you format your Markdo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2075</th>\n",
       "      <td>Marketing Strategy Platform</td>\n",
       "      <td>361.0</td>\n",
       "      <td>Title: On-the-Spot Business Plan\\n\\nExecutive ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2946</th>\n",
       "      <td>Multi-dimensional Gaming</td>\n",
       "      <td>360.0</td>\n",
       "      <td>Of course! I'm glad to help you with that. Her...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18392</th>\n",
       "      <td>Gaming Economics</td>\n",
       "      <td>345.0</td>\n",
       "      <td>Title: \"Fortnite Battle Royale Beginner's Guid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4580</th>\n",
       "      <td>Analyzing Product Sentiment</td>\n",
       "      <td>314.0</td>\n",
       "      <td>Title: The North Water\\nTrailer Link: https://...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18637</th>\n",
       "      <td>Gender and Clothing Perceptions</td>\n",
       "      <td>312.0</td>\n",
       "      <td>50% Tocopheryl Acetate, Simpleslide-out Switch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367</th>\n",
       "      <td>Interconnected Systems</td>\n",
       "      <td>307.0</td>\n",
       "      <td>Ah, a day on Mars! *excited tone* Living on Ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1993</th>\n",
       "      <td>Childhood Stress and Health</td>\n",
       "      <td>306.0</td>\n",
       "      <td>Title: The Tapestry of Relationships\\n\\nOnce u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15217</th>\n",
       "      <td>Movie Analysis and Interpretation</td>\n",
       "      <td>301.0</td>\n",
       "      <td>In the midst of World War II, danger and intri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17648</th>\n",
       "      <td>Language Analysis</td>\n",
       "      <td>270.0</td>\n",
       "      <td>Here are 50 words that match that description:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15661</th>\n",
       "      <td>Natural Language Processing</td>\n",
       "      <td>159.0</td>\n",
       "      <td>It's important for an AI assistant to ensure t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              topic_name  n_uniquewords  \\\n",
       "1730                    Creative Writing          424.0   \n",
       "8349      Environmental Impact Solutions          404.0   \n",
       "20090      Healthy Lifestyle Exploration          392.0   \n",
       "2936                  Culinary Creations          377.0   \n",
       "3702                 Cultural Expression          373.0   \n",
       "17937                  Urban Exploration          368.0   \n",
       "2075         Marketing Strategy Platform          361.0   \n",
       "2946            Multi-dimensional Gaming          360.0   \n",
       "18392                   Gaming Economics          345.0   \n",
       "4580         Analyzing Product Sentiment          314.0   \n",
       "18637    Gender and Clothing Perceptions          312.0   \n",
       "367               Interconnected Systems          307.0   \n",
       "1993         Childhood Stress and Health          306.0   \n",
       "15217  Movie Analysis and Interpretation          301.0   \n",
       "17648                  Language Analysis          270.0   \n",
       "15661        Natural Language Processing          159.0   \n",
       "\n",
       "                                                response  \n",
       "1730   Title: Crescendo of the Heart\\n\\nChapter One -...  \n",
       "8349   Title: The Impact of Artificial Intelligence o...  \n",
       "20090  Title: Electronic Voting Systems: A Glimpse in...  \n",
       "2936   What purpose does frosting serve, and how does...  \n",
       "3702   **In the acrid vapors of a renewed and retched...  \n",
       "17937  Hello! I'm here to help you format your Markdo...  \n",
       "2075   Title: On-the-Spot Business Plan\\n\\nExecutive ...  \n",
       "2946   Of course! I'm glad to help you with that. Her...  \n",
       "18392  Title: \"Fortnite Battle Royale Beginner's Guid...  \n",
       "4580   Title: The North Water\\nTrailer Link: https://...  \n",
       "18637  50% Tocopheryl Acetate, Simpleslide-out Switch...  \n",
       "367    Ah, a day on Mars! *excited tone* Living on Ma...  \n",
       "1993   Title: The Tapestry of Relationships\\n\\nOnce u...  \n",
       "15217  In the midst of World War II, danger and intri...  \n",
       "17648  Here are 50 words that match that description:...  \n",
       "15661  It's important for an AI assistant to ensure t...  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_example = df_all.sort_values(by=['n_uniquewords', 'topic_name'], ascending=[False, False])\n",
    "df_example = df_example.groupby('topic_name').head(1)\n",
    "df_example = df_example[['topic_name', 'n_uniquewords', 'response']]\n",
    "df_example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}