per_class_mapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I reload the data manually\n",
    "\n",
    "files = ['Monday-WorkingHours.pcap_ISCX.csv', 'Tuesday-WorkingHours.pcap_ISCX.csv', 'Wednesday-workingHours.pcap_ISCX.csv', 'Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv', 'Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv', 'Friday-WorkingHours-Morning.pcap_ISCX.csv', 'Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv', 'Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv']\n",
    "li = []\n",
    "for filename in files:\n",
    "    df = pd.read_csv(filename)\n",
    "    li.append(df)\n",
    "\n",
    "frame = pd.concat(li, axis=0, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>...</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "      <th>Label</th>\n",
       "      <th>Binary_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49188</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49188</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49188</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>49188</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49486</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>BENIGN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Destination Port   Flow Duration   Total Fwd Packets  \\\n",
       "0              49188               4                   2   \n",
       "1              49188               1                   2   \n",
       "2              49188               1                   2   \n",
       "3              49188               1                   2   \n",
       "4              49486               3                   2   \n",
       "\n",
       "    Total Backward Packets  Total Length of Fwd Packets  \\\n",
       "0                        0                           12   \n",
       "1                        0                           12   \n",
       "2                        0                           12   \n",
       "3                        0                           12   \n",
       "4                        0                           12   \n",
       "\n",
       "    Total Length of Bwd Packets   Fwd Packet Length Max  \\\n",
       "0                             0                       6   \n",
       "1                             0                       6   \n",
       "2                             0                       6   \n",
       "3                             0                       6   \n",
       "4                             0                       6   \n",
       "\n",
       "    Fwd Packet Length Min   Fwd Packet Length Mean   Fwd Packet Length Std  \\\n",
       "0                       6                      6.0                     0.0   \n",
       "1                       6                      6.0                     0.0   \n",
       "2                       6                      6.0                     0.0   \n",
       "3                       6                      6.0                     0.0   \n",
       "4                       6                      6.0                     0.0   \n",
       "\n",
       "   ...  Active Mean   Active Std   Active Max   Active Min  Idle Mean  \\\n",
       "0  ...          0.0          0.0            0            0        0.0   \n",
       "1  ...          0.0          0.0            0            0        0.0   \n",
       "2  ...          0.0          0.0            0            0        0.0   \n",
       "3  ...          0.0          0.0            0            0        0.0   \n",
       "4  ...          0.0          0.0            0            0        0.0   \n",
       "\n",
       "    Idle Std   Idle Max   Idle Min   Label  Binary_Label  \n",
       "0        0.0          0          0  BENIGN             0  \n",
       "1        0.0          0          0  BENIGN             0  \n",
       "2        0.0          0          0  BENIGN             0  \n",
       "3        0.0          0          0  BENIGN             0  \n",
       "4        0.0          0          0  BENIGN             0  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove leading spaces\n",
    "frame.columns = frame.columns.str.lstrip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BENIGN                        2273097\n",
       "DoS Hulk                       231073\n",
       "PortScan                       158930\n",
       "DDoS                           128027\n",
       "DoS GoldenEye                   10293\n",
       "FTP-Patator                      7938\n",
       "SSH-Patator                      5897\n",
       "DoS slowloris                    5796\n",
       "DoS Slowhttptest                 5499\n",
       "Bot                              1966\n",
       "Web Attack � Brute Force         1507\n",
       "Web Attack � XSS                  652\n",
       "Infiltration                       36\n",
       "Web Attack � Sql Injection         21\n",
       "Heartbleed                         11\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.value_counts(frame['Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# relevancy as defined by  Thomas\n",
    "relevant_features_rf = ['Average Packet Size',\n",
    "  'Avg Fwd Segment Size',\n",
    "  'Flow Bytes/s',\n",
    "  'Flow IAT Max',\n",
    "  'Flow IAT Mean',\n",
    "  'Fwd Header Length.1',\n",
    "  'Fwd IAT Mean',\n",
    "  'Fwd IAT Min',\n",
    "  'Fwd IAT Total',\n",
    "  'Fwd Packet Length Mean',\n",
    "  'Fwd Packet Length Std',\n",
    "  'Init_Win_bytes_backward',\n",
    "  'Init_Win_bytes_forward',\n",
    "  'PSH Flag Count',\n",
    "  'Packet Length Mean',\n",
    "  'Subflow Bwd Bytes',\n",
    "  'Subflow Fwd Bytes',\n",
    "  'Total Fwd Packets',\n",
    "  'Total Length of Bwd Packets',\n",
    "  'Total Length of Fwd Packets',\n",
    "  'act_data_pkt_fwd']\n",
    "\n",
    "# relevancy defined by easiness of guessing and rf importance\n",
    "relevant_features = ['Destination Port',\n",
    "'Flow Duration',\n",
    "'Total Backward Packets',\n",
    "'Total Fwd Packets',\n",
    "'Packet Length Mean',\n",
    "'Flow Bytes/s',\n",
    "'Flow IAT Min',\n",
    "'Flow IAT Max',\n",
    "'PSH Flag Count',\n",
    "'SYN Flag Count',\n",
    "'RST Flag Count',\n",
    "'ACK Flag Count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 557646 entries, 541265 to 2823364\n",
      "Data columns (total 14 columns):\n",
      " #   Column                  Non-Null Count   Dtype  \n",
      "---  ------                  --------------   -----  \n",
      " 0   Packet Length Mean      557646 non-null  float64\n",
      " 1   Flow IAT Max            557646 non-null  int64  \n",
      " 2   Flow IAT Min            557646 non-null  int64  \n",
      " 3   SYN Flag Count          557646 non-null  int64  \n",
      " 4   Total Fwd Packets       557646 non-null  int64  \n",
      " 5   Destination Port        557646 non-null  int64  \n",
      " 6   Flow Duration           557646 non-null  int64  \n",
      " 7   Total Backward Packets  557646 non-null  int64  \n",
      " 8   Flow Bytes/s            557646 non-null  float64\n",
      " 9   PSH Flag Count          557646 non-null  int64  \n",
      " 10  ACK Flag Count          557646 non-null  int64  \n",
      " 11  RST Flag Count          557646 non-null  int64  \n",
      " 12  Label                   557646 non-null  object \n",
      " 13  Binary_Label            557646 non-null  int64  \n",
      "dtypes: float64(2), int64(11), object(1)\n",
      "memory usage: 63.8+ MB\n"
     ]
    }
   ],
   "source": [
    "# get only the relevant data\n",
    "sub_frame = frame[list(relevant_features)+['Label', 'Binary_Label']]\n",
    "sub_frame = sub_frame[sub_frame.Binary_Label != 0]\n",
    "sub_frame.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Packet Length Mean</th>\n",
       "      <th>Flow IAT Max</th>\n",
       "      <th>Flow IAT Min</th>\n",
       "      <th>SYN Flag Count</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Flow Bytes/s</th>\n",
       "      <th>PSH Flag Count</th>\n",
       "      <th>ACK Flag Count</th>\n",
       "      <th>RST Flag Count</th>\n",
       "      <th>Label</th>\n",
       "      <th>Binary_Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>541265</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5214882</td>\n",
       "      <td>124</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>80</td>\n",
       "      <td>5216127</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>FTP-Patator</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541266</th>\n",
       "      <td>0.0</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>FTP-Patator</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541267</th>\n",
       "      <td>0.0</td>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>FTP-Patator</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541268</th>\n",
       "      <td>0.0</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>FTP-Patator</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541269</th>\n",
       "      <td>0.0</td>\n",
       "      <td>68</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>FTP-Patator</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Packet Length Mean  Flow IAT Max  Flow IAT Min  SYN Flag Count  \\\n",
       "541265                 0.0       5214882           124               0   \n",
       "541266                 0.0            20            20               0   \n",
       "541267                 0.0            38            38               0   \n",
       "541268                 0.0            80            80               0   \n",
       "541269                 0.0            68            68               0   \n",
       "\n",
       "        Total Fwd Packets  Destination Port  Flow Duration  \\\n",
       "541265                  3                80        5216127   \n",
       "541266                  1                21             20   \n",
       "541267                  1                21             38   \n",
       "541268                  1                21             80   \n",
       "541269                  1                21             68   \n",
       "\n",
       "        Total Backward Packets  Flow Bytes/s  PSH Flag Count  ACK Flag Count  \\\n",
       "541265                       1           0.0               1               0   \n",
       "541266                       1           0.0               0               1   \n",
       "541267                       1           0.0               0               1   \n",
       "541268                       1           0.0               0               1   \n",
       "541269                       1           0.0               0               1   \n",
       "\n",
       "        RST Flag Count        Label  Binary_Label  \n",
       "541265               0  FTP-Patator             1  \n",
       "541266               0  FTP-Patator             1  \n",
       "541267               0  FTP-Patator             1  \n",
       "541268               0  FTP-Patator             1  \n",
       "541269               0  FTP-Patator             1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Packet Length Mean</th>\n",
       "      <th>Flow IAT Max</th>\n",
       "      <th>Flow IAT Min</th>\n",
       "      <th>SYN Flag Count</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Flow Bytes/s</th>\n",
       "      <th>PSH Flag Count</th>\n",
       "      <th>ACK Flag Count</th>\n",
       "      <th>RST Flag Count</th>\n",
       "      <th>Binary_Label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Bot</th>\n",
       "      <td>51.260684</td>\n",
       "      <td>1.726650e+05</td>\n",
       "      <td>1.992487e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.193795</td>\n",
       "      <td>17783.990844</td>\n",
       "      <td>3.509427e+05</td>\n",
       "      <td>3.343845</td>\n",
       "      <td>3.055371e+05</td>\n",
       "      <td>0.624619</td>\n",
       "      <td>0.375381</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DDoS</th>\n",
       "      <td>736.885283</td>\n",
       "      <td>1.583782e+07</td>\n",
       "      <td>2.442396e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.472439</td>\n",
       "      <td>81.227382</td>\n",
       "      <td>1.695559e+07</td>\n",
       "      <td>3.255821</td>\n",
       "      <td>6.111693e+04</td>\n",
       "      <td>0.454474</td>\n",
       "      <td>0.546822</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DoS GoldenEye</th>\n",
       "      <td>501.158612</td>\n",
       "      <td>1.962861e+07</td>\n",
       "      <td>1.131841e+07</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.904401</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>2.312722e+07</td>\n",
       "      <td>3.714078</td>\n",
       "      <td>6.882391e+02</td>\n",
       "      <td>0.715632</td>\n",
       "      <td>0.286700</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DoS Hulk</th>\n",
       "      <td>591.761324</td>\n",
       "      <td>5.669352e+07</td>\n",
       "      <td>7.513308e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.279656</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>5.708173e+07</td>\n",
       "      <td>4.205688</td>\n",
       "      <td>2.928097e+04</td>\n",
       "      <td>0.063132</td>\n",
       "      <td>0.682750</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DoS Slowhttptest</th>\n",
       "      <td>132.171694</td>\n",
       "      <td>3.906159e+07</td>\n",
       "      <td>1.024613e+06</td>\n",
       "      <td>0.163302</td>\n",
       "      <td>5.741771</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>5.771989e+07</td>\n",
       "      <td>0.963812</td>\n",
       "      <td>2.164242e+07</td>\n",
       "      <td>0.766685</td>\n",
       "      <td>0.233315</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DoS slowloris</th>\n",
       "      <td>50.957426</td>\n",
       "      <td>4.029260e+07</td>\n",
       "      <td>2.869085e+05</td>\n",
       "      <td>0.316943</td>\n",
       "      <td>6.337129</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>5.655437e+07</td>\n",
       "      <td>1.655797</td>\n",
       "      <td>4.392122e+04</td>\n",
       "      <td>0.668219</td>\n",
       "      <td>0.331781</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FTP-Patator</th>\n",
       "      <td>9.798894</td>\n",
       "      <td>1.631614e+06</td>\n",
       "      <td>7.985538e+01</td>\n",
       "      <td>0.497229</td>\n",
       "      <td>5.496977</td>\n",
       "      <td>21.007433</td>\n",
       "      <td>4.513245e+06</td>\n",
       "      <td>7.808390</td>\n",
       "      <td>7.991625e+05</td>\n",
       "      <td>0.500630</td>\n",
       "      <td>0.499370</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Heartbleed</th>\n",
       "      <td>1626.602318</td>\n",
       "      <td>1.452711e+06</td>\n",
       "      <td>-3.636364e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2583.727273</td>\n",
       "      <td>444.000000</td>\n",
       "      <td>1.106797e+08</td>\n",
       "      <td>1897.181818</td>\n",
       "      <td>6.590280e+04</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Infiltration</th>\n",
       "      <td>161.101292</td>\n",
       "      <td>3.766240e+07</td>\n",
       "      <td>8.209444e+03</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>830.222222</td>\n",
       "      <td>444.000000</td>\n",
       "      <td>7.840772e+07</td>\n",
       "      <td>829.611111</td>\n",
       "      <td>2.018822e+04</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PortScan</th>\n",
       "      <td>3.246025</td>\n",
       "      <td>8.029323e+04</td>\n",
       "      <td>8.784072e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.017580</td>\n",
       "      <td>8624.460177</td>\n",
       "      <td>8.282023e+04</td>\n",
       "      <td>1.004958</td>\n",
       "      <td>2.201851e+05</td>\n",
       "      <td>0.999572</td>\n",
       "      <td>0.000384</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SSH-Patator</th>\n",
       "      <td>44.328642</td>\n",
       "      <td>1.236447e+06</td>\n",
       "      <td>9.201747e+01</td>\n",
       "      <td>0.002374</td>\n",
       "      <td>11.124979</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>6.168967e+06</td>\n",
       "      <td>16.532305</td>\n",
       "      <td>4.398181e+02</td>\n",
       "      <td>0.505172</td>\n",
       "      <td>0.494828</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Web Attack � Brute Force</th>\n",
       "      <td>24.509333</td>\n",
       "      <td>4.802932e+06</td>\n",
       "      <td>1.098859e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.371599</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>6.506182e+06</td>\n",
       "      <td>6.056403</td>\n",
       "      <td>1.792807e+02</td>\n",
       "      <td>0.905109</td>\n",
       "      <td>0.094891</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Web Attack � Sql Injection</th>\n",
       "      <td>166.382924</td>\n",
       "      <td>2.858505e+06</td>\n",
       "      <td>4.085714e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.047619</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>2.870398e+06</td>\n",
       "      <td>2.666667</td>\n",
       "      <td>3.187348e+02</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Web Attack � XSS</th>\n",
       "      <td>23.479736</td>\n",
       "      <td>5.162877e+06</td>\n",
       "      <td>1.257209e+02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.903374</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>6.715357e+06</td>\n",
       "      <td>3.573620</td>\n",
       "      <td>1.033466e+02</td>\n",
       "      <td>0.964724</td>\n",
       "      <td>0.035276</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Packet Length Mean  Flow IAT Max  Flow IAT Min  \\\n",
       "Label                                                                        \n",
       "Bot                                  51.260684  1.726650e+05  1.992487e+02   \n",
       "DDoS                                736.885283  1.583782e+07  2.442396e+03   \n",
       "DoS GoldenEye                       501.158612  1.962861e+07  1.131841e+07   \n",
       "DoS Hulk                            591.761324  5.669352e+07  7.513308e+02   \n",
       "DoS Slowhttptest                    132.171694  3.906159e+07  1.024613e+06   \n",
       "DoS slowloris                        50.957426  4.029260e+07  2.869085e+05   \n",
       "FTP-Patator                           9.798894  1.631614e+06  7.985538e+01   \n",
       "Heartbleed                         1626.602318  1.452711e+06 -3.636364e-01   \n",
       "Infiltration                        161.101292  3.766240e+07  8.209444e+03   \n",
       "PortScan                              3.246025  8.029323e+04  8.784072e+01   \n",
       "SSH-Patator                          44.328642  1.236447e+06  9.201747e+01   \n",
       "Web Attack � Brute Force             24.509333  4.802932e+06  1.098859e+02   \n",
       "Web Attack � Sql Injection          166.382924  2.858505e+06  4.085714e+01   \n",
       "Web Attack � XSS                     23.479736  5.162877e+06  1.257209e+02   \n",
       "\n",
       "                            SYN Flag Count  Total Fwd Packets  \\\n",
       "Label                                                           \n",
       "Bot                               0.000000           3.193795   \n",
       "DDoS                              0.000000           4.472439   \n",
       "DoS GoldenEye                     0.000000           5.904401   \n",
       "DoS Hulk                          0.000000           5.279656   \n",
       "DoS Slowhttptest                  0.163302           5.741771   \n",
       "DoS slowloris                     0.316943           6.337129   \n",
       "FTP-Patator                       0.497229           5.496977   \n",
       "Heartbleed                        0.000000        2583.727273   \n",
       "Infiltration                      0.555556         830.222222   \n",
       "PortScan                          0.000000           1.017580   \n",
       "SSH-Patator                       0.002374          11.124979   \n",
       "Web Attack � Brute Force          0.000000          12.371599   \n",
       "Web Attack � Sql Injection        0.000000           3.047619   \n",
       "Web Attack � XSS                  0.000000           7.903374   \n",
       "\n",
       "                            Destination Port  Flow Duration  \\\n",
       "Label                                                         \n",
       "Bot                             17783.990844   3.509427e+05   \n",
       "DDoS                               81.227382   1.695559e+07   \n",
       "DoS GoldenEye                      80.000000   2.312722e+07   \n",
       "DoS Hulk                           80.000000   5.708173e+07   \n",
       "DoS Slowhttptest                   80.000000   5.771989e+07   \n",
       "DoS slowloris                      80.000000   5.655437e+07   \n",
       "FTP-Patator                        21.007433   4.513245e+06   \n",
       "Heartbleed                        444.000000   1.106797e+08   \n",
       "Infiltration                      444.000000   7.840772e+07   \n",
       "PortScan                         8624.460177   8.282023e+04   \n",
       "SSH-Patator                        22.000000   6.168967e+06   \n",
       "Web Attack � Brute Force           80.000000   6.506182e+06   \n",
       "Web Attack � Sql Injection         80.000000   2.870398e+06   \n",
       "Web Attack � XSS                   80.000000   6.715357e+06   \n",
       "\n",
       "                            Total Backward Packets  Flow Bytes/s  \\\n",
       "Label                                                              \n",
       "Bot                                       3.343845  3.055371e+05   \n",
       "DDoS                                      3.255821  6.111693e+04   \n",
       "DoS GoldenEye                             3.714078  6.882391e+02   \n",
       "DoS Hulk                                  4.205688  2.928097e+04   \n",
       "DoS Slowhttptest                          0.963812  2.164242e+07   \n",
       "DoS slowloris                             1.655797  4.392122e+04   \n",
       "FTP-Patator                               7.808390  7.991625e+05   \n",
       "Heartbleed                             1897.181818  6.590280e+04   \n",
       "Infiltration                            829.611111  2.018822e+04   \n",
       "PortScan                                  1.004958  2.201851e+05   \n",
       "SSH-Patator                              16.532305  4.398181e+02   \n",
       "Web Attack � Brute Force                  6.056403  1.792807e+02   \n",
       "Web Attack � Sql Injection                2.666667  3.187348e+02   \n",
       "Web Attack � XSS                          3.573620  1.033466e+02   \n",
       "\n",
       "                            PSH Flag Count  ACK Flag Count  RST Flag Count  \\\n",
       "Label                                                                        \n",
       "Bot                               0.624619        0.375381             0.0   \n",
       "DDoS                              0.454474        0.546822             0.0   \n",
       "DoS GoldenEye                     0.715632        0.286700             0.0   \n",
       "DoS Hulk                          0.063132        0.682750             0.0   \n",
       "DoS Slowhttptest                  0.766685        0.233315             0.0   \n",
       "DoS slowloris                     0.668219        0.331781             0.0   \n",
       "FTP-Patator                       0.500630        0.499370             0.0   \n",
       "Heartbleed                        0.090909        0.909091             0.0   \n",
       "Infiltration                      0.166667        0.833333             0.0   \n",
       "PortScan                          0.999572        0.000384             0.0   \n",
       "SSH-Patator                       0.505172        0.494828             0.0   \n",
       "Web Attack � Brute Force          0.905109        0.094891             0.0   \n",
       "Web Attack � Sql Injection        0.571429        0.428571             0.0   \n",
       "Web Attack � XSS                  0.964724        0.035276             0.0   \n",
       "\n",
       "                            Binary_Label  \n",
       "Label                                     \n",
       "Bot                                  1.0  \n",
       "DDoS                                 1.0  \n",
       "DoS GoldenEye                        1.0  \n",
       "DoS Hulk                             1.0  \n",
       "DoS Slowhttptest                     1.0  \n",
       "DoS slowloris                        1.0  \n",
       "FTP-Patator                          1.0  \n",
       "Heartbleed                           1.0  \n",
       "Infiltration                         1.0  \n",
       "PortScan                             1.0  \n",
       "SSH-Patator                          1.0  \n",
       "Web Attack � Brute Force             1.0  \n",
       "Web Attack � Sql Injection           1.0  \n",
       "Web Attack � XSS                     1.0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# means per attack\n",
    "means = sub_frame.groupby('Label').mean()\n",
    "means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define two ways to get intervals: linear (with three intervals) or percentiles (50%-median and 80%)\n",
    "\n",
    "# maybe a for loop instead of t1, t2? Enables choosing a larger step\n",
    "def get_linear_vector(_class, feature, steps=3):\n",
    "    _max = frame[feature].max()\n",
    "    _min = frame[feature].min()\n",
    "    interval = (_max-_min)/steps\n",
    "    t1, t2 = _min+interval, _max-interval\n",
    "    rep = [int(means[feature][_class]<t1), int(means[feature][_class]>t1 and means[feature][_class]<t2), int(means[feature][_class]>t2)]\n",
    "    return np.argmax(rep), rep\n",
    "\n",
    "def get_percentile_vector(_class, feature):\n",
    "    t1, t2 = frame[feature].quantile(0.5), frame[feature].quantile(0.8)\n",
    "    rep = [int(means[feature][_class]<t1), int(means[feature][_class]>t1 and means[feature][_class]<t2), int(means[feature][_class]>t2)]\n",
    "    return np.argmax(rep), rep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "representations = {}\n",
    "levels = {}\n",
    "\n",
    "for attack in means.index:\n",
    "    # initialize with the port number since we don't have levels here but a specific value\n",
    "    representations[attack] = [int(means['Destination Port'][attack])] \n",
    "    levels[attack] = [int(means['Destination Port'][attack])]\n",
    "    for feature in relevant_features[1:]: # ignore the destination port\n",
    "        level, representation = get_percentile_vector(attack, feature)\n",
    "        representations[attack].append(representation) \n",
    "        levels[attack].append(level) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Bot': [17783, 1, 1, 1, 0, 1, 2, 1, 1, 0, 0, 1],\n",
       " 'DDoS': [81, 2, 1, 1, 2, 1, 2, 2, 1, 0, 0, 1],\n",
       " 'DoS GoldenEye': [80, 2, 1, 1, 2, 0, 2, 2, 1, 0, 0, 1],\n",
       " 'DoS Hulk': [80, 2, 1, 1, 2, 1, 2, 2, 1, 0, 0, 1],\n",
       " 'DoS Slowhttptest': [80, 2, 0, 1, 1, 2, 2, 2, 1, 2, 0, 1],\n",
       " 'DoS slowloris': [80, 2, 0, 2, 0, 1, 2, 2, 1, 2, 0, 1],\n",
       " 'FTP-Patator': [21, 1, 2, 1, 0, 2, 1, 1, 1, 2, 0, 1],\n",
       " 'Heartbleed': [444, 2, 2, 2, 2, 1, 0, 1, 1, 0, 0, 1],\n",
       " 'Infiltration': [444, 2, 2, 2, 1, 1, 2, 2, 1, 2, 0, 1],\n",
       " 'PortScan': [8624, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1],\n",
       " 'SSH-Patator': [22, 2, 2, 2, 0, 0, 1, 1, 1, 2, 0, 1],\n",
       " 'Web Attack � Brute Force': [80, 2, 2, 2, 0, 0, 2, 1, 1, 0, 0, 1],\n",
       " 'Web Attack � Sql Injection': [80, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1],\n",
       " 'Web Attack � XSS': [80, 2, 1, 2, 0, 0, 2, 1, 1, 0, 0, 1]}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Bot': [17783,\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'DDoS': [81,\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'DoS GoldenEye': [80,\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'DoS Hulk': [80,\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'DoS Slowhttptest': [80,\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'DoS slowloris': [80,\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'FTP-Patator': [21,\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'Heartbleed': [444,\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'Infiltration': [444,\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'PortScan': [8624,\n",
       "  [0, 1, 0],\n",
       "  [1, 0, 0],\n",
       "  [1, 0, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'SSH-Patator': [22,\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'Web Attack � Brute Force': [80,\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'Web Attack � Sql Injection': [80,\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]],\n",
       " 'Web Attack � XSS': [80,\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 1],\n",
       "  [1, 0, 0],\n",
       "  [1, 0, 0],\n",
       "  [0, 0, 1],\n",
       "  [0, 1, 0],\n",
       "  [0, 1, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 0, 0],\n",
       "  [0, 1, 0]]}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "representations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 duplicate representations detected\n"
     ]
    }
   ],
   "source": [
    "# check if we don't have duplicate representations\n",
    "counter = 0\n",
    "for item1 in representations.values():\n",
    "    for item2 in representations.values():\n",
    "        if item1 == item2:\n",
    "            counter+=1\n",
    "print(int((counter-len(representations.values()))/2), 'duplicate representations detected')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOV0lEQVR4nO3dX4hedX7H8fc3BhMZqwMqEoztWAiB0EBaqkJhy1AoG8kOwtKLBG/chs22IF40F5u7pexNbrxYd4XtUKxLKYaweGFqqDftEKFeGMGirqSkIYsjiNU6ockq66zfXszj9mGY598858yZ+fp+wcAzv3PO73yfL8ePJ78580xkJpKkWnZ1XYAkqXmGuyQVZLhLUkGGuyQVZLhLUkGGuyQVtLvrAgDuvffenJubm/i4K1euAHDw4MGGK9qEXi2MqOXWrVvMzMxs6TkrG9rPQf1ZP95GH5ucs3+uvtdtXf9TX6Oj3nvb1+2VK/DZZ2uv77hjsvO0VFuj/933efPNNz/OzPs22rYtwn1ubo7Lly9PfNz8/DwAS0tLzRa0Gb1aGFHL0tLSb+veqnNWNrSfg/qzfryNPjY5Z/9cfa/buv6nvkZHvfe2r9v5eXjrrbXXR45Mdp6Wamv0v/s+EfHLQdtclpGkggx3SSqo03CPiIWIWLxx40aXZUhSOZ2Ge2ZeyMxTd999d5dlSFI5LstIUkGGuyQVZLhLUkHb4jn3tz+4wdyZVyY+7sNrnwBs6timnevVcnxELacPr/JkQ/WOe87KhvVzUH/Wj7fRxybn7J+r/3Vb1/+wnl4/e6zRc6k93rlLUkGGuyQVZLhLUkGGuyQVZLhLUkGGuyQVZLhLUkGdPuceEQvAwu7ZfV2WIUnlbIsPDtu1t/m/UCJJX2cuy0hSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBXU+N9QjYhdwA+Bu4DLmfmzps8hSRpurDv3iHg+Ij6KiHfWjR+NiCsRcTUizvSGHwf2A18Ay82WK0kax7jLMi8AR/sHIuI24DngMeAQcCIiDgEHgX/PzL8B/rq5UiVJ4xprWSYzL0XE3LrhR4CrmXkNICLOsXbX/j7w694+vxk0Z0ScAk4BzN5zH6cPr05UOMCzMwnA05s4tmn7e7WMeh/33zF6n6bPWdmwfg7qz/rxNvrY5Jz9c/W/buv6H9bTpaWlkccfWVkB4K0B+47aPq0jKyvcubpW/82VlYnO01ZtN2/eHKt3TZpmzf0B1oL8K8vAo8CPgB9HxDeAS4MOzsxFYBFgz74D+czbk5fy4a0AYDPHNu3hMWs5fXi1sXrHPWdlw/o5qD/rx9voY5Nz9s/V/7qt639YT68/MT96gtlZAObnB+w7avu0Zmdh9+7ey9nJztNSbUtLS+293wEaT4XM/BVwsul5JUnjm+ZRyA+AB/u+398bkyR1bJpwfwM4EBEPRcTtwHHg5UkmiIiFiFj88vNbU5QhSVpv3EchXwReBw5GxHJEnMzMVeAp4FXgPeB8Zr47yckz80Jmntq1d2bSuiVJQ4z7tMyJAeMXgYuNViRJmpofPyBJBXX6DF1ELAALu2f3dVmGJJXT6Z27a+6S1A6XZSSpIMNdkgpyzV2SCnLNXZIKcllGkgoy3CWpIMNdkgoy3CWpIJ+WkaSCfFpGkgpyWUaSCjLcJakgw12SCjLcJakgn5aRpIJ8WkaSCnJZRpIKMtwlqSDDXZIKMtwlqSDDXZIK8lFISSrIRyElqSCXZSSpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgryN1QlqSB/Q1WSCnJZRpIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqSDDXZIK8oPDJKkgPzhMkgpyWUaSCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJamgxsM9IuYj4rWI+GlEzDc9vyRptLHCPSKej4iPIuKddeNHI+JKRFyNiDO94QRuAnuB5WbLlSSNY9w79xeAo/0DEXEb8BzwGHAIOBERh4DXMvMx4PvA3zZXqiRpXLvH2SkzL0XE3LrhR4CrmXkNICLOAY9n5i962z8F9gyaMyJOAacAZu+5j9OHVyerHHh2JgF4ehPHNm1/r5ZR7+P+O0bv0/Q5KxvWz0H9WT/eRh+bnLN/rv7XbV3/w3q6tLQ08vgjKysAvDVg31Hbp3VkZYU7V9fqv7myMtF52qrt5s2bY/WuSWOF+wAPAO/3fb8MPBoR3wa+CcwCPxl0cGYuAosAe/YdyGfenryUD28FAJs5tmkPj1nL6cOrjdU77jkrG9bPQf1ZP95GH5ucs3+u/tdtXf/Denr9ifnRE8zOAjA/P2DfUdunNTsLu3f3Xs5Odp6WaltaWmrv/Q7QeCpk5kvAS03PK0ka3zTh/gHwYN/3+3tjkiRg7swrwNq/hp7svb5+9tiWnHuaRyHfAA5ExEMRcTtwHHh5kgkiYiEiFr/8/NYUZUiS1hv3UcgXgdeBgxGxHBEnM3MVeAp4FXgPOJ+Z705y8sy8kJmndu2dmbRuSdIQ4z4tc2LA+EXgYqMVSZKm5scPSFJBnT5DFxELwMLu2X1dliFJ5XR65+6auyS1w2UZSSrIcJekglxzl6SCXHOXpIJclpGkggx3SSrIcJekgvyBqiQV5A9UJakgl2UkqSDDXZIK+vr+8U1JatBXf3Vpu/DOXZIK8mkZSSrIp2UkqSCXZSSpIH+gKulraaMfgJ679gkAx8+8wvWzx7a6pEZ55y5JBRnuklSQ4S5JBRnuklSQz7lLUkE+5y5JBbksI0kFGe6SVJDhLkkFGe6SVJDhLkkFGe6SVJDhLkkF+UtMklRQp+GemReAC3v2Hfhul3VI0iS2299L3YjLMpJUkOEuSQX5l5gkaQM7YellGO/cJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgw12SCjLcJakgPxVSkgrq9M49My9k5qlde2e6LEOSynFZRpIKMtwlqSDDXZIKMtwlqSDDXZIKMtwlqSD/EpOksY3z14nOXfsEgOMD9h21fRrXzx5rfM6dyjt3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSqolXCPiJmIuBwR32pjfknScGN9cFhEPA98C/goM/+gb/wo8CPgNuDvM/Nsb9P3gfMN1ypJQ82deYVz1z7h0OdfAPCLa5+08gFlO8G4d+4vAEf7ByLiNuA54DHgEHAiIg5FxJ8DvwA+arBOSdIExrpzz8xLETG3bvgR4GpmXgOIiHPA48CdwAxrgf9ZRFzMzC/XzxkRp4BTALP33Mfpw6sTF//sTALw9CaObdr+Xi2j3sf9d4zep+lzVjasn4P6s368jT42OWf/XP2v27r+p71GR733tq/b/TPJnl3//3qS87RVW39Pl5aWGp17kGk+z/0B4P2+75eBRzPzKYCIeBL4eKNgB8jMRWARYM++A/nM25OX8uGtAGAzxzbt4TFrOX14tbF6xz1nZcP6Oag/68fb6GOTc/bP1f+6ret/2mt01Htv+7p9+FZwVy91lm/FROdpq7b+nl5/Yr7RuQdpLRUy84W25pYkDTfN0zIfAA/2fb+/NyZJ6tg04f4GcCAiHoqI24HjwMuTTBARCxGx+OXnt6YoQ5K03ljhHhEvAq8DByNiOSJOZuYq8BTwKvAecD4z353k5Jl5ITNP7do7M2ndkqQhxn1a5sSA8YvAxUYrkiRNzY8fkKSCOn2GLiIWgIXds/u6LEOSyun0zt01d0lqh8syklSQ4S5JBbnmLkkFdRrumXkBuLBn34HvdlmHJG2VuXUfQXz97LFWzuOyjCQVZLhLUkGGuyQV5A9UJakgf4lJkgpyWUaSCjLcJakgw12SCjLcJamgyMyuayAi/hv45SYOvRu40dC+w7ZvtG2zY/cCHw+tdHqT9GWaY7voaRf93Oi8bRw7zn6D9plk3J6Ot30n9PT3MvO+Dbdk5o79Ahab2nfY9o22bXYMuLyd+rLTetpFP7eqp+PsN2ifScbt6Xjbd2JP+792+rLMhQb3HbZ9o23TjLVtmnNu95520c9pzzvusePsN2ifScbt6Xjbd2JPf2tbLMt8nUTE5cz8467rqMJ+Ns+eNq+Lnu70O/edaLHrAoqxn82zp83b8p565y5JBXnnLkkFGe6SVJDhLkkFdfqRv193ETEP/BB4FziXmUudFlRAROxirad3sfZs8c86LmnHi4hvAE+wlheHMvNPOi5pR4uI3wWeBf4H+M/MPNvGebxzb1hEPB8RH0XEO+vGj0bElYi4GhFnesMJ3AT2AstbXetOMWFPHwf2A19gTweapKeZ+Vpm/hXwz4D/s9zAhNfoYeDnmfmXwB+2VtRW/9ZU9S/gT4E/At7pG7sN+C/g94Hbgf8ADgG7etvvB/6p69q369eEPT0DfK+3z8+7rn27fk3S077t54Hf6br27fg14TV6D/BvwL8C32mrJu/cG5aZl1j751a/R4CrmXktM38NnAMez8wve9s/BfZsYZk7yiQ9Ze1u/dPePr/Zuip3lgl7+tVSwo3M/N+trXRnmLCf3wF+kJl/BhxrqybDfWs8ALzf9/0y8EBEfDsi/g74R+AnnVS2c23YU+Al4JsR8WPgUheF7WCDegpwEviHLa9oZxvUz38Bno6InwLX2zq5P1DtUGa+xFoYqSGZ+SvWgkgNyswfdF1DFZn5DvAXbZ/HO/et8QHwYN/3+3tj2jx72jx72qxO+2m4b403gAMR8VBE3A4cB17uuKadzp42z542q9N+Gu4Ni4gXgdeBgxGxHBEnM3MVeAp4FXgPOJ+Z73ZZ505iT5tnT5u1HfvpB4dJUkHeuUtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQYa7JBVkuEtSQf8HMIHffLWbZm4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# play with one distribution to guess the best percentiles. Goal is to have ...meeh... separation\n",
    "feature = 'Flow IAT Max'\n",
    "plt.figure()\n",
    "frame[feature].hist(log=True)\n",
    "for xc in means[feature]:\n",
    "    plt.axvline(x=xc, c='r')\n",
    "plt.axvline(x=frame[feature].quantile(0.5), c='black')\n",
    "plt.axvline(x=frame[feature].quantile(0.8), c='black')\n",
    "plt.xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAUWElEQVR4nO3dT4idV5rf8e9PHlwjKm0LNMIIlYk8pFqMQMEDlh0YArWYJBKO2pMmzEjxwh2EhBs0XkSL0cAsJoQQZ+FF1OPgVGijLDpWjJnMSFiDsyo0EC/sHkxkWwiE8NglsJV20gZXJLpr6mRRdeNKlW7d9711//no+wFxfc8973ue855Xj1+Ozj03pRQkSXXZMe4AJEmDZ3KXpAqZ3CWpQiZ3SaqQyV2SKmRyl6QK/co4G09yDDj2ne9859R3v/vd9ie4cWP19cCBgcY1Umt9WJqZYXp6eszBrNO5th3DvsbdxvKDD1Zfn3xyXdUba1WHG9PS0lLzMWlyL964AXfvws6dW9YbVf/60qufTa9DrzpdtBqTJr7lOeSnP/3pz0ope+73WSZhnftTTz1V3n///fYHzs2tvi4sDDKc0Vrrw8If/zFznf5Mgo2xDPsadxvLXbtWX3/+83VV59aqDjemhYWF5mPS5F6cm1v9n9WTT25Zb1T960uvfja9Dr3qdNFqTJr4lueQJD8tpTx1v8+clpGkCo01uSc5lmT+q6++GmcYklSdsSb3UsrlUsrpRx99dJxhSFJ1nJaRpAqZ3CWpQs65S1KFnHOXpAqN9UtMHdduf8X+c2+3Pu7irS8BON7HsZOi04drt7/iBxPUj05cHcO+xt3G8n/c+yUAf3dd+edrdfu5Z9o4e2i58Zg0uRcv3vqSg/d+yce3vtyy3jD798nLzw78nJpMzrlLUoVM7pJUoYnYW+ZXdu0dZxiSVJ2J+AfVHb86QRtmSVIFnJaRpAqZ3CWpQiZ3SaqQyV2SKjTw1TJJdgD/CngEeL+U8p8G3YYkaWuNntyTvJ7kTpIPN5QfSXIjyc0k59aKnwNmgF8Ci4MNV5LURNNpmQvAkfUFSR4CXgWOAgeBE0kOAgeA/15K+RfADwcXqiSpqUbTMqWUq0n2byh+GrhZSrkFkOQiq0/tnwG/WKvzN93OmeQ0cBpg1+49nD203CpwgJnp1d9/7efYSdHpw6c7J6sfnbg6hh1bt7Gc2rG5/fNrdV8ackyPtRiTJvfizHRhasfq61b1htm/7f4u65Nrv2X7QZfz9Pq8aZ1uvv7664H+tux2Ypl025lz38dqIu9YBJ4B/h3woyR/H7ja7eBSyjwwDzC1d7a8cq19KIeXAkA/x06KTh++uDtZ/ejE1THs2LqN5Qsrm9v/fETjfvbQcuM2mtyLh5fCIyuwuJQt6w2zf588P7e9E6z9YHnXH6nu9XnTOl0M/AeytxHLpBv43VNK+T/AySZ13X5AkoZjO0shbwOPr3s/s1YmSRqz7ST394DZJE8keRg4DlxqcwL3lpGk4Wi6FPIN4F3gQJLFJCdLKcvAGeAd4DrwZinlozaNd35mb+XeUtu4JUlbaLpa5kSX8ivAlX4bL6VcBi5P7Z091e85JEmbuZ+7JFXI/dwlqUJuHCZJFXJaRpIq5LSMJFXIaRlJqpDTMpJUIadlJKlCTstIUoWclpGkCjktI0kVclpGkipkcpekCpncJalCk/OjnZKqtf/c2//vvy/e+hKA4+vKmrpwxH+fa8rVMpJUIVfLSFKFnHOXpAqZ3CWpQiZ3SaqQyV2SKjTw5J5kLslfJnktydygzy9J6q1Rck/yepI7ST7cUH4kyY0kN5OcWysuwNfArwKLgw1XktRE0yf3C8CR9QVJHgJeBY4CB4ETSQ4Cf1lKOQr8AfAvBxeqJKmpRl9iKqVcTbJ/Q/HTwM1Syi2AJBeB50opH699/r+BqW7nTHIaOA2wa/cezh5abhc5MDNdAPo6dlJ0+vDpzsnqRyeujmHH1m0sp3Zsbv/8Wt2XhhzTYy3GpMm9ODNdmNqx+rpVvWH270c/+fNtHf9PvvgZAP+1y3m6fX720Df/vZ2/t19//TULCwutj+vmyZ//HIAPBnjOSbGdb6juAz5b934ReCbJ94F/BOwC/qTbwaWUeWAeYGrvbHnlWvtQDi8FgH6OnRSdPnxxd7L60YmrY9ixdRvLF1Y2t//5iMb97KHlxm00uRcPL4VHVmBxKVvWG1X/+tGrn02vQ6863Vw4Ms3c3Fzr47ratQtgsOecEAO/e0opfwr8aZO6bj8gScOxndUyt4HH172fWSuTJI3Zdp7c3wNmkzzBalI/DvyzNicopVwGLk/tnT21jTgkPSCu3f6KH7TYTfKTl58dYjSTrelSyDeAd4EDSRaTnCylLANngHeA68CbpZSP2jSe5FiS+ZV7S23jliRtoelqmRNdyq8AV/pt3Cd3SRoO93OXpAq5n7skVciNwySpQk7LSFKFnJaRpAo5LSNJFXJaRpIq5LSMJFXIaRlJqpDTMpJUIadlJKlCTstIUoVM7pJUIZO7JFXI5C5JFXK1jCRVaKzJ3R/rkDRM+3v8JN/FW18C8PdGEcyIOS0jSRUyuUtShUzuklQhk7skVWgoyT3JdJL3k/zjYZxfkrS1Rsk9yetJ7iT5cEP5kSQ3ktxMcm7dR38AvDnIQCVJzTV9cr8AHFlfkOQh4FXgKHAQOJHkYJJ/AHwM3BlgnJKkFhqtcy+lXE2yf0Px08DNUsotgCQXgeeAvwVMs5rw7ya5UkpZ2XjOJKeB0wC7du/h7KHl1sHPTBeAvo6dFJ0+fLpzsvrRiatj2LF1G8upHZvbP79W96Uhx/RYizFpci/OTBemdqy+blVvVP3rR69+Nr0Ovep002ZMmujEsrCwMLBzTortfIlpH/DZuveLwDOllDMASX4A/Ox+iR2glDIPzANM7Z0tr1xrH8rhpQDQz7GTotOHL+5OVj86cXUMO7ZuY/nCyub2Px/RuJ89tNy4jSb34uGl8MgKLC5ly3qj6l8/evWz6XXoVaebNmPSRCeWubm5gZ1zUgzt7imlXOhVx+0HJGk4trNa5jbw+Lr3M2tlkqQx205yfw+YTfJEkoeB48ClNifwl5gkaTiaLoV8A3gXOJBkMcnJUsoycAZ4B7gOvFlK+ahN40mOJZlfubfUNm5J0haarpY50aX8CnCl38bdFVKShsP93CWpQmPdW8Y5d0kaDjcOk6QKOS0jSRVyWkaSKuS0jCRVyGkZSaqQ0zKSVCGnZSSpQk7LSFKFnJaRpAo5LSNJFTK5S1KFTO6SVCGTuyRVyNUyklQhV8tIUoWclpGkCpncJalCJndJqpDJXZIqNPDknuQ3kryW5K0kPxz0+SVJvTVK7kleT3InyYcbyo8kuZHkZpJzAKWU66WUF4HfBX5r8CFLknpp+uR+ATiyviDJQ8CrwFHgIHAiycG1z74HvA1cGVikkqTGGn2JqZRyNcn+DcVPAzdLKbcAklwEngM+LqVcAi4leRv4z/c7Z5LTwGmAXbv3cPbQcuvgZ6YLQF/HTopOHz7dOVn96MTVMezYuo3l1I7N7Z9fq/vSkGN6rMWYNLkXZ6YLUztWX7eqN6r+9aNXP5teh151umkzJk10YllYWBjYOSfFdr6hug/4bN37ReCZJHPA94EptnhyL6XMA/MAU3tnyyvX2odyeCkA9HPspOj04Yu7k9WPTlwdw46t21i+sLK5/c9HNO5nDy03bqPJvXh4KTyyAotL2bLeqPrXj179bHodetXpps2YNNGJZW5ubmDnnBQDv3tKKQvAQpO6bj8gScOxndUyt4HH172fWSuTJI3ZdpL7e8BskieSPAwcBy61OYF7y0jScDRdCvkG8C5wIMlikpOllGXgDPAOcB14s5TyUZvGkxxLMr9yb6lt3JKkLTRdLXOiS/kVtrHcsZRyGbg8tXf2VL/nkKTt2n/u7S0//+TlZ0cUyeC4n7skVcj93CWpQm4cJkkVclpGkirktIwkVchpGUmqkNMyklQhp2UkqUKTt+2cJE2Yrb7kNKlfcHJaRpIq5LSMJFXI1TKSVCGTuyRVyOQuSRUyuUtShVwtI0kVcrWMJFXIaRlJqpDJXZIqZHKXpAqZ3CWpQkNZLZPkd4BngUeAH5dS/tsw2pEk3V/jJ/ckrye5k+TDDeVHktxIcjPJOYBSyp+VUk4BLwK/N9iQJUm9tJmWuQAcWV+Q5CHgVeAocBA4keTguip/tPa5JGmEGk/LlFKuJtm/ofhp4GYp5RZAkovAc0muAy8Df1FK+av7nS/JaeA0wK7dezh7aLl18DPTBaCvYydFpw+f7pysfnTi6hh2bN3GcmrH5vbPr9V9acgxPdZiTJrcizPThakdq69b1RtV//rRq59Nr0OvOt20GZMmBpFDFhYWBhTNYG13zn0f8Nm694vAM8DvA78NPJrk75RSXtt4YCllHpgHmNo7W1651j6Uw0sBoJ9jJ0WnD1/cnax+dOLqGHZs3cbyhZXN7X8+onE/e2i5cRtN7sXDS+GRFVhcypb1RtW/fvTqZ9Pr0KtON23GpIlB5JBPnp8bUDSDNZS7p5RyHjjfq57bD0jScGx3KeRt4PF172fWyiRJY7Td5P4eMJvkiSQPA8eBS00Pdm8ZSRqONksh3wDeBQ4kWUxyspSyDJwB3gGuA2+WUj5qcc5jSeZX7i21jVuStIU2q2VOdCm/Alzpp/FSymXg8tTe2VP9HC9Juj/3c5ekCrmfuyRVyI3DJKlCTstIUoWclpGkCjktI0kVclpGkirktIwkVchpGUmqkNMyklQhp2UkqUJOy0hShUzuklQhk7skVcjkLkkVcrWMJFXI1TKSVKGxPrlL0rfd/nNv37f8k5efHXEk/z/n3CWpQiZ3SaqQyV2SKjTw5J7k15P8OMlbgz63JKmZRsk9yetJ7iT5cEP5kSQ3ktxMcg6glHKrlHJyGMFKkppp+uR+ATiyviDJQ8CrwFHgIHAiycGBRidJ6kujpZCllKtJ9m8ofhq4WUq5BZDkIvAc8HGTcyY5DZwG2LV7D2cPLTcM+Rsz0wWgr2MnRacPn+6crH504uoYdmzdxnJqx+b2z6/VfWnIMT3WYkya3Isz04WpHauvW9UbVf/60aufTa9DrzrdtBmTJoaZQxYWFgZ+zja2s859H/DZuveLwDNJdgP/GvjNJH9YSvk39zu4lDIPzANM7Z0tr1xrH8rhpQDQz7GTotOHL+5OVj86cXUMO7ZuY/nCyub2Px/RuJ89tNy4jSb34uGl8MgKLC5ly3qj6l8/evWz6XXoVaebNmPSxDBzyCfPzw38nG0MvEellC+BF5vUdfsBSRqO7ayWuQ08vu79zFqZJGnMtpPc3wNmkzyR5GHgOHCpzQncW0aShqPpUsg3gHeBA0kWk5wspSwDZ4B3gOvAm6WUj9o0nuRYkvmVe0tt45YkbaHpapkTXcqvAFf6bbyUchm4PLV39lS/55AkbeZ+7pJUIfdzl6QK+eQuSUNwv33eR7nHu0/uklQht/yVpAo5LSNJFXJaRpIq5LSMJFXI5C5JFXLOXZIq5Jy7JFXIaRlJqpDJXZIqZHKXpAqZ3CWpQq6WkaQKuVpGkirktIwkVcjkLkkVMrlLUoVM7pJUoYGvlkkyDfx74BfAQinlJ4NuQ5K0tUZP7kleT3InyYcbyo8kuZHkZpJza8XfB94qpZwCvjfgeCVJDTSdlrkAHFlfkOQh4FXgKHAQOJHkIDADfLZW7W8GE6YkqY1G0zKllKtJ9m8ofhq4WUq5BZDkIvAcsMhqgv+ALf7nkeQ0cBpg1+49nD203DZ2ZqYLQF/HTopOHz7dOVn96MTVMezYuo3l1I7N7Z9fq/vSkGN6rMWYNLkXZ6YLUztWX7eqN6r+9aNXP5teh151umkzJk2MOof86Cd/vqns0L5Hh9LWdubc9/HNEzqsJvVngPPAnyR5Frjc7eBSyjwwDzC1d7a8cq19KIeXAkA/x06KTh++uDtZ/ejE1THs2LqN5Qsrm9v/fETjfvbQcuM2mtyLh5fCIyuwuJQt642qf/3o1c+m16FXnW7ajEkTk5BDPnl+bijnHXiPSilLwD9vUtftByRpOLazFPI28Pi69zNrZZKkMdtOcn8PmE3yRJKHgePApTYncG8ZSRqOpksh3wDeBQ4kWUxyspSyDJwB3gGuA2+WUj5q03iSY0nmV+4ttY1bkrSFpqtlTnQpvwJc6bfxUspl4PLU3tlT/Z5DkrSZ+7lLUoXcz12SKuSTuyRVKKWU3rWGHUTyP4G/bnHIo8BXQ6jfq95Wn3f77H7l9yv7NeBnDWIctLbXclDncUy6G9eYND3GMRntebY65m+XUvbc95NSyrfuDzA/jPq96m31ebfP7lfepez9b8O1dEzqHZOmxzgmkzcm9/vzbd3Pveu2Btus36veVp93++x+5W3jH6ZBxeKYDM64xqTpMY7JaM/TV9sTMS0jSPJ+KeWpccehbzgmk8cxae7b+uReo/lxB6BNHJPJ45g05JO7JFXIJ3dJqpDJXZIqZHKXpAqZ3CdQkl9P8uMkb407Fq1K8jtJ/mOS/5LkH447HkGS30jyWpK3kvxw3PFMGpP7iCR5PcmdJB9uKD+S5EaSm0nOAZRSbpVSTo4n0gdHyzH5s1LKKeBF4PfGEe+DoOWYXC+lvAj8LvBb44h3kpncR+cCcGR9QZKHgFeBo8BB4ESSg6MP7YF1gfZj8kdrn2s4LtBiTJJ8D3ibbWw9XiuT+4iUUq4C/2tD8dPAzbUn9V8AF4HnRh7cA6rNmGTVvwX+opTyV6OO9UHR9u9JKeVSKeUo8PxoI518Jvfx2gd8tu79IrAvye4krwG/meQPxxPaA+u+YwL8PvDbwD9N8uI4AnuAdft7MpfkfJL/gE/um4x1y1/dXynlS1bndjUhSinngfPjjkPfKKUsAAtjDmNi+eQ+XreBx9e9n1kr0/g4JpPHMemDyX283gNmkzyR5GHgOHBpzDE96ByTyeOY9MHkPiJJ3gDeBQ4kWUxyspSyDJwB3gGuA2+WUj4aZ5wPEsdk8jgmg+PGYZJUIZ/cJalCJndJqpDJXZIqZHKXpAqZ3CWpQiZ3SaqQyV2SKmRyl6QKmdwlqUL/F1Bv7OgPUxFrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# play with one distribution to guess the best percentiles\n",
    "feature = 'Packet Length Mean'\n",
    "plt.figure()\n",
    "frame[feature].hist(log=True)\n",
    "for xc in means[feature]:\n",
    "    plt.axvline(x=xc, c='r')\n",
    "plt.axvline(x=frame[feature].quantile(0.5), c='black')\n",
    "plt.axvline(x=frame[feature].quantile(0.8), c='black')\n",
    "plt.xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}