fireman_data_pec_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8ab622fa-2181-4f28-b1ca-90df96c5b344",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "58c2cf06-902a-4f5a-bf75-181e7f093c0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from src import common\n",
    "from src.outlier_model import OutlierModel\n",
    "import dill as pickle\n",
    "import matplotlib\n",
    "\n",
    "logging.basicConfig(format='%(asctime)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')\n",
    "logger = logging.getLogger(\"TimeSeries\")\n",
    "logger.setLevel(logging.INFO)\n",
    "\n",
    "output_dir = 'pec_output'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e38dff8-a1c2-4afb-9420-d13c2bacf93d",
   "metadata": {},
   "source": [
    "# Dataset preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "727dde3d-d032-4c83-a32f-2fe681d9abd2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "column_types_loc = \"#datasets/PEC/column_types.json\"\n",
    "column_types = common.json_load(column_types_loc)\n",
    "\n",
    "target = column_types[\"target\"]\n",
    "identifier = column_types[\"identifier\"]\n",
    "cat_cols = column_types[\"categorical\"]\n",
    "measurement_label = column_types[\"measurement_label\"]\n",
    "\n",
    "with open(f\"#datasets/PEC/dataset.pkl\", \"rb\") as fh:\n",
    "    data = pickle.load(fh)\n",
    "\n",
    "# assign unique label to each measurement fault and create dictionary for easier analysis\n",
    "fault_dict = {}\n",
    "for label, i in zip(data[measurement_label].unique(), range(len(data[measurement_label].unique()))):\n",
    "    data.loc[(data[measurement_label] == label) & (data[target] == 1), target] = int(i + 1)\n",
    "    fault_dict[label] = int(i + 1)\n",
    "\n",
    "data.drop(columns=[measurement_label], inplace=True)\n",
    "# Fill NA - 0 for numerical and 'NA' for categorical\n",
    "# categorical\n",
    "data[cat_cols] = data[cat_cols].fillna(\"NA\")\n",
    "data[cat_cols] = data[cat_cols].astype(str)\n",
    "# non-categorical\n",
    "non_cat_cols = data.drop(columns=cat_cols + [identifier]).columns.tolist()\n",
    "data[non_cat_cols] = data[non_cat_cols].fillna(0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4534f2d",
   "metadata": {},
   "source": [
    "# Outlier_model\n",
    "* outlier_model code that implementes outlier_model.predict() function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d9c79a5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_index = 0\n",
    "end_index = 1000000\n",
    "data_test = data[start_index:end_index]\n",
    "\n",
    "outlier_key = \"f_c\"\n",
    "m = 250\n",
    "preload_size = 5000\n",
    "std_dev_mult = 4\n",
    "range_mult = 0\n",
    "recent_mult = 1\n",
    "outlier_model = OutlierModel(m=m, std_dev_mult=std_dev_mult, range_mult=range_mult, recent_mult=recent_mult,\n",
    "                             time_series=data_test[:preload_size][outlier_key],\n",
    "                             egress=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "862bf180",
   "metadata": {},
   "source": [
    "# Simulate stream of data\n",
    "## Inverse transform test dataset for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a7475b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "    f_c            P    m_d  m_q       theta   P_ref   V_DC   V_phaseA  \\\n0  50.0  2499.997221  311.0  0.0  312.486512  2500.0  800.0   2.487576   \n1  50.0  2499.997250  311.0  0.0  312.502220  2500.0  800.0   7.372088   \n2  50.0  2499.997280  311.0  0.0  312.517928  2500.0  800.0  12.254782   \n3  50.0  2499.997310  311.0  0.0  312.533636  2500.0  800.0  17.134452   \n4  50.0  2499.997341  311.0  0.0  312.549344  2500.0  800.0  22.009894   \n\n     V_phaseB    V_phaseC  I_phaseA  I_phaseB  I_phaseC  fault     \n0 -270.569073  268.081497 -0.392875 -4.447507  4.840382      0  0  \n1 -272.944264  265.572176 -0.308598 -4.494417  4.803015      0  1  \n2 -275.252112  262.997330 -0.224245 -4.540218  4.764463      0  2  \n3 -277.492044  260.357593 -0.139837 -4.584899  4.724735      0  3  \n4 -279.663510  257.653616 -0.055394 -4.628448  4.683842      0  4  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>f_c</th>\n      <th>P</th>\n      <th>m_d</th>\n      <th>m_q</th>\n      <th>theta</th>\n      <th>P_ref</th>\n      <th>V_DC</th>\n      <th>V_phaseA</th>\n      <th>V_phaseB</th>\n      <th>V_phaseC</th>\n      <th>I_phaseA</th>\n      <th>I_phaseB</th>\n      <th>I_phaseC</th>\n      <th>fault</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>50.0</td>\n      <td>2499.997221</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.486512</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>2.487576</td>\n      <td>-270.569073</td>\n      <td>268.081497</td>\n      <td>-0.392875</td>\n      <td>-4.447507</td>\n      <td>4.840382</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>50.0</td>\n      <td>2499.997250</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.502220</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>7.372088</td>\n      <td>-272.944264</td>\n      <td>265.572176</td>\n      <td>-0.308598</td>\n      <td>-4.494417</td>\n      <td>4.803015</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>50.0</td>\n      <td>2499.997280</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.517928</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>12.254782</td>\n      <td>-275.252112</td>\n      <td>262.997330</td>\n      <td>-0.224245</td>\n      <td>-4.540218</td>\n      <td>4.764463</td>\n      <td>0</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>50.0</td>\n      <td>2499.997310</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.533636</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>17.134452</td>\n      <td>-277.492044</td>\n      <td>260.357593</td>\n      <td>-0.139837</td>\n      <td>-4.584899</td>\n      <td>4.724735</td>\n      <td>0</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>50.0</td>\n      <td>2499.997341</td>\n      <td>311.0</td>\n      <td>0.0</td>\n      <td>312.549344</td>\n      <td>2500.0</td>\n      <td>800.0</td>\n      <td>22.009894</td>\n      <td>-279.663510</td>\n      <td>257.653616</td>\n      <td>-0.055394</td>\n      <td>-4.628448</td>\n      <td>4.683842</td>\n      <td>0</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5317afea",
   "metadata": {},
   "source": [
    "* loading the whole DL model with preprocessors in each iteration is stupid but I wanted to send Alex at least some initial code\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "aade096d",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 10000\n",
      "Current Global index: 20000\n",
      "Current Global index: 30000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:47:47 PM:  Anomaly at Global index: 40341\n",
      "04/21/2022 02:47:47 PM: max_mp: 1689.7735, metric:98.4065: metric-max_mp: 1591.367 range: 1689.7735\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 40000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:47:49 PM:  Anomaly at Global index: 45783\n",
      "04/21/2022 02:47:49 PM: max_mp: 46498.2142, metric:35662.835600000006: metric-max_mp: 10835.378599999996 range: 139.84790000000066\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 50000\n",
      "Current Global index: 60000\n",
      "Current Global index: 70000\n",
      "Current Global index: 80000\n",
      "Current Global index: 90000\n",
      "Current Global index: 100000\n",
      "Current Global index: 110000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:48:16 PM:  Anomaly at Global index: 120038\n",
      "04/21/2022 02:48:16 PM: max_mp: 0.1243, metric:0.0072: metric-max_mp: 0.1171 range: 0.1243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 120000\n",
      "Current Global index: 130000\n",
      "Current Global index: 140000\n",
      "Current Global index: 150000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:48:33 PM:  Anomaly at Global index: 160053\n",
      "04/21/2022 02:48:33 PM: max_mp: 0.1243, metric:0.0072: metric-max_mp: 0.1171 range: 0.1243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 160000\n",
      "Current Global index: 170000\n",
      "Current Global index: 180000\n",
      "Current Global index: 190000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:48:48 PM:  Anomaly at Global index: 200098\n",
      "04/21/2022 02:48:48 PM: max_mp: 0.0112, metric:0.0008: metric-max_mp: 0.0104 range: 0.0112\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 200000\n",
      "Current Global index: 210000\n",
      "Current Global index: 220000\n",
      "Current Global index: 230000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:49:03 PM:  Anomaly at Global index: 240054\n",
      "04/21/2022 02:49:03 PM: max_mp: 1.6339, metric:0.09509999999999999: metric-max_mp: 1.5388 range: 1.6339\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 240000\n",
      "Current Global index: 250000\n",
      "Current Global index: 260000\n",
      "Current Global index: 270000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:49:17 PM:  Anomaly at Global index: 280395\n",
      "04/21/2022 02:49:17 PM: max_mp: 7215.7619, metric:420.2196: metric-max_mp: 6795.5423 range: 7215.7619\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 280000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "04/21/2022 02:49:19 PM:  Anomaly at Global index: 285395\n",
      "04/21/2022 02:49:19 PM: max_mp: 57649.2738, metric:47016.6066: metric-max_mp: 10632.667200000004 range: 75.36929999999847\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Global index: 290000\n",
      "Current Global index: 300000\n",
      "Current Global index: 310000\n",
      "Current Global index: 320000\n"
     ]
    }
   ],
   "source": [
    "fault = False\n",
    "for index, row in data_test[preload_size:].iterrows():\n",
    "    outlier_model.train_one(row[outlier_key])\n",
    "    fault = outlier_model.predict_one(index)\n",
    "    if index % 10000 == 0:\n",
    "        print(f\"Current Global index: {index}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "with open(\"../masters-thesis-graphing/_data/PEC/outlier_model.pkl\", 'wb') as handle:\n",
    "    pickle.dump(outlier_model, handle, protocol=pickle.HIGHEST_PROTOCOL)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3 (ipykernel)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}