Notebooks/EnergyProfilingIntermittent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dc3e4402",
   "metadata": {},
   "source": [
    "# Implementation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sWbXCGozBRNW",
   "metadata": {
    "id": "sWbXCGozBRNW"
   },
   "source": [
    "## Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "kmxpysFu7zjH",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kmxpysFu7zjH",
    "outputId": "db2717d5-22be-4fa8-99fb-3f9ea90e7e1b"
   },
   "outputs": [],
   "source": [
    "# data elaboration functions\n",
    "import pandas as pd\n",
    "from six.moves import collections_abc\n",
    "import string\n",
    "import numpy as np\n",
    "\n",
    "# datetime functions\n",
    "import datetime as dt\n",
    "\n",
    "# file management functions\n",
    "import os\n",
    "import sys\n",
    "import opendatasets as od\n",
    "import pickle\n",
    "from pathlib import Path\n",
    "\n",
    "# plot functions\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# data science functions\n",
    "import matplotlib.pyplot as plt\n",
    "from kneed import KneeLocator\n",
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import silhouette_score\n",
    "from sklearn.preprocessing import StandardScaler, scale\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "import joblib\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "import xgboost as xgb\n",
    "\n",
    "# statistical functions\n",
    "from scipy.stats.mstats import winsorize\n",
    "\n",
    "# configuration file\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "# custom functions\n",
    "from Code.Profiling.Intermittent.intermittent import Intermittent\n",
    "from Code.Utils.utils import Utils\n",
    "from Code.Scoring.kpi import Kpi\n",
    "from Code.Scoring.forecast import Forecasting\n",
    "from Code.Scoring.train import Training\n",
    "from Code.Scoring.train_test import TrainTest\n",
    "from Code.Scoring.scoring import Scoring\n",
    "from Code.Regressors.regressors import Regressors\n",
    "from Code.Plotting.plots import Plots\n",
    "from Configuration.config import cfg_path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dc26b7b",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "458162d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# od.download(\"https://www.kaggle.com/arashnic/building-sites-power-consumption-dataset/download\")\n",
    "root = Path(os.getcwd()).parent\n",
    "dataset_path = os.path.join(root, cfg_path.data_dir.input_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86bb0e13",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "09358d6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ICD10Description</th>\n",
       "      <th>DateOfAccident</th>\n",
       "      <th>holidays</th>\n",
       "      <th>wd_mon</th>\n",
       "      <th>wd_tue</th>\n",
       "      <th>wd_wed</th>\n",
       "      <th>wd_thu</th>\n",
       "      <th>wd_fri</th>\n",
       "      <th>wd_sat</th>\n",
       "      <th>wd_sun</th>\n",
       "      <th>...</th>\n",
       "      <th>month_03</th>\n",
       "      <th>month_04</th>\n",
       "      <th>month_05</th>\n",
       "      <th>month_06</th>\n",
       "      <th>month_07</th>\n",
       "      <th>month_08</th>\n",
       "      <th>month_09</th>\n",
       "      <th>month_10</th>\n",
       "      <th>month_11</th>\n",
       "      <th>month_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Amputation_of_finger</td>\n",
       "      <td>1982-10-24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Amputation_of_finger</td>\n",
       "      <td>1982-10-25</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Amputation_of_finger</td>\n",
       "      <td>1982-10-26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Amputation_of_finger</td>\n",
       "      <td>1982-10-27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Amputation_of_finger</td>\n",
       "      <td>1982-10-28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ICD10Description DateOfAccident  holidays  wd_mon  wd_tue  wd_wed  \\\n",
       "0  Amputation_of_finger     1982-10-24         0       0       0       0   \n",
       "1  Amputation_of_finger     1982-10-25         0       1       0       0   \n",
       "2  Amputation_of_finger     1982-10-26         0       0       1       0   \n",
       "3  Amputation_of_finger     1982-10-27         0       0       0       1   \n",
       "4  Amputation_of_finger     1982-10-28         0       0       0       0   \n",
       "\n",
       "   wd_thu  wd_fri  wd_sat  wd_sun  ...  month_03  month_04  month_05  \\\n",
       "0       0       0       0       1  ...         0         0         0   \n",
       "1       0       0       0       0  ...         0         0         0   \n",
       "2       0       0       0       0  ...         0         0         0   \n",
       "3       0       0       0       0  ...         0         0         0   \n",
       "4       1       0       0       0  ...         0         0         0   \n",
       "\n",
       "   month_06  month_07  month_08  month_09  month_10  month_11  month_12  \n",
       "0         0         0         0         0         1         0         0  \n",
       "1         0         0         0         0         1         0         0  \n",
       "2         0         0         0         0         1         0         0  \n",
       "3         0         0         0         0         1         0         0  \n",
       "4         0         0         0         0         1         0         0  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = pd.read_pickle(os.path.join(\n",
    "    root, cfg_path.data_dir.output_path, 'df_final.pkl'))\n",
    "df_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f23ed7fb",
   "metadata": {},
   "source": [
    "## Parameter setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "find_date, date_col found: ['DateOfAccident']\n"
     ]
    }
   ],
   "source": [
    "id = 'site_id'\n",
    "list_unique_id = ['site_id', 'timestamp']\n",
    "list_temp = ['temp']\n",
    "y = 'value'\n",
    "date_var = Utils.find_date(df_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'value'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\mabellani\\.conda\\envs\\forecasting_energy\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3360\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3361\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3362\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\mabellani\\.conda\\envs\\forecasting_energy\\lib\\site-packages\\pandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\mabellani\\.conda\\envs\\forecasting_energy\\lib\\site-packages\\pandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'value'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\MABELL~1\\AppData\\Local\\Temp/ipykernel_24224/792764010.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Defining working dataframe by removing potential null values in y variable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_final\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m~\u001b[0m\u001b[0mdf_final\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Id available'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mid\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\mabellani\\.conda\\envs\\forecasting_energy\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3456\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3457\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3458\u001b[1;33m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3459\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3460\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\mabellani\\.conda\\envs\\forecasting_energy\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3361\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3362\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3363\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3365\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhasnans\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'value'"
     ]
    }
   ],
   "source": [
    "# Defining working dataframe by removing potential null values in y variable\n",
    "df = df_final.loc[~df_final[y].isnull(),].copy()\n",
    "print('Id available', list(df[id].unique()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute intermittent indicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Winsorizing parameters\n",
    "highest = 0.05\n",
    "lowest = 0.05\n",
    "\n",
    "# Identifying intermittent time series parameters\n",
    "threshold = 250\n",
    "perc = 0.01\n",
    "quant = 0.999"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Choose indicator function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying indicator function ('compute_indicator_values' or 'enhanced_compute_indicator_values')\n",
    "func = 'compute_indicator_values'\n",
    "df_scoring = df.loc[:, [id, y]].groupby(id).apply(lambda x: Intermittent.call_intermittent_function(func,\n",
    "    np.array(x), threshold, perc, quant, highest, lowest)).reset_index(level=id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Profiling intermittent time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_scoring' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mC:\\Users\\MABELL~1\\AppData\\Local\\Temp/ipykernel_21796/557984365.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# Excluding the ids for which the indicators are np.NaN\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdf_scoring_no_nan\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf_scoring\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mlist_nan\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_scoring\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mid\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_scoring_no_nan\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mid\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'List of nan ids'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist_nan\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df_scoring' is not defined"
     ]
    }
   ],
   "source": [
    "# Excluding the ids for which the indicators are np.NaN\n",
    "df_scoring_no_nan = df_scoring.dropna()\n",
    "list_nan = list(set(df_scoring[id]) - set(df_scoring_no_nan[id]))\n",
    "print('List of nan ids', list_nan)\n",
    "\n",
    "# Classify intermittent TS for electricity consumption\n",
    "thres_cv2_constant = 0.06\n",
    "thres_cv2 = 2\n",
    "thres_adi = 3\n",
    "thres_sddi = 6.2\n",
    "min_time_cons = 2\n",
    "type = df_scoring_no_nan['type'].unique()[0]\n",
    "\n",
    "df_profiling = Intermittent.classify_intermittent(df_scoring_no_nan, type, thres_cv2_constant, thres_cv2, thres_adi, thres_sddi, min_time_cons)\n",
    "\n",
    "# Dictionary\n",
    "list_of_profiles = ['regular', 'constant_zero', 'constant', 'spikes',\n",
    "                    'lumpy', 'erratic', 'unforecastable_time', 'unforecastable_quantity']\n",
    "dict_profiling = {}\n",
    "for c in list_of_profiles:\n",
    "    dict_profiling[c] = list(\n",
    "        df_profiling.loc[df_profiling.profile == c, id].unique())\n",
    "    print(id, c, dict_profiling[c])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting profiled series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plotting id: 2 as 1 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 2\n",
      "Successfully created the directory c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 6 as 2 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 6\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 8 as 3 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 8\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 9 as 4 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 9\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 10 as 5 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 10\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 13 as 6 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 13\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 16 as 7 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 16\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 17 as 8 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 17\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 18 as 9 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 18\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 19 as 10 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 19\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 21 as 11 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 21\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 22 as 12 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 22\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 25 as 13 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 25\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 27 as 14 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 27\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 29 as 15 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 29\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 32 as 16 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 32\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 33 as 17 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 33\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 39 as 18 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 39\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 40 as 19 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 40\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 41 as 20 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 41\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 42 as 21 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 42\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 51 as 22 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 51\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 45 as 23 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 45\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 47 as 24 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 47\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 48 as 25 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 48\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 50 as 26 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 50\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 49 as 27 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 49\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 57 as 28 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 57\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 7 as 29 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 7\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 44 as 30 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 44\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 20 as 31 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 20\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 46 as 32 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 46\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 38 as 33 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 38\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 52 as 34 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 52\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 12 as 35 of 35\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 12\n",
      "Creation of the directory failed or already present c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\regular\n",
      "Plotting id: 26 as 1 of 1\n",
      "find_date, date_col found: ['timestamp']\n",
      "sliding_line_plot: plotting Value site_id 26\n",
      "Successfully created the directory c:\\Users\\mabellani\\OneDrive - Microsoft\\Documents\\REPOSITORY\\dstoolkit-forecasting\\Data/Plots\\constant\n"
     ]
    }
   ],
   "source": [
    "for profile in list_of_profiles:\n",
    "    df_to_plot = df.loc[df[id].isin(list(dict_profiling[profile])), ].copy()\n",
    "    count = 1\n",
    "    for i in list(df_to_plot[id].unique()):\n",
    "        print('Plotting id:', i, 'as', count, 'of',\n",
    "              len(list(df_to_plot[id].unique())))\n",
    "        plot = Plots.sliding_line_plot(df_to_plot, y, id, i, chart_title=\"\")\n",
    "        folder_name = os.path.join(root, cfg_path.data_dir.plot_path, profile)\n",
    "        Utils.create_folder_tree(folder_name)\n",
    "        plot.write_html(os.path.join(folder_name, id + '_' + str(i) + '_profile_' + profile + \".html\"))\n",
    "        count = count + 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a binary pickle file \n",
    "f = open(os.path.join(root, cfg_path.data_dir.output_path, 'dict_profiling.pkl'),\"wb\")\n",
    "# write the python object (dict) to pickle file\n",
    "pickle.dump(dict_profiling,f)\n",
    "# close file\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2b8f5b14411d0017ed363cef4929504a7281087d06f1b18c01da6e951b937e80"
  },
  "kernelspec": {
   "display_name": "Python 3.7.7 ('forecasting_energy')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}