notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports and Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# import \n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import os.path\n",
    "from os import path\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.patches as mpatches\n",
    "from sklearn import tree\n",
    "from datetime import datetime\n",
    "import pytz\n",
    "import ast\n",
    "from tqdm import tqdm\n",
    "import copy\n",
    "import pickle \n",
    "import os\n",
    "\n",
    "# Deep Learning\n",
    "#import tensorflow as tf\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchaudio \n",
    "\n",
    "from sklearn.metrics import cohen_kappa_score,f1_score,accuracy_score, precision_score, recall_score, classification_report, roc_auc_score, \\\n",
    "    hamming_loss, confusion_matrix\n",
    "from scipy.special import softmax\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.utils import check_random_state\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "\n",
    "%matplotlib inline\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "plt.rcParams.update({'font.size': 22})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": [
     0,
     1,
     18,
     36,
     52,
     106
    ]
   },
   "outputs": [],
   "source": [
    "# Functions\n",
    "def get_splits(stream_subset, with_user = True):\n",
    "    stream_subset_filtered = stream_subset[[\"user_id\",\"media_id\",'readable_time', 'weekday', 'user_age', \n",
    "                                            'gender_num','device_num','location_num','network_num', 'x_day', \n",
    "                                             'y_day', 'x_time', 'y_time', 'matches']]\n",
    "    if (not with_user):    \n",
    "        stream_subset_filtered = stream_subset_filtered.drop(['user_age', 'gender_num','location_num'],axis=1)\n",
    "\n",
    "    X = stream_subset_filtered.iloc[:,:-1]\n",
    "    y = stream_subset_filtered.iloc[:,-1]\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=1, stratify=y)\n",
    "\n",
    "    X_train = X_train.astype({\"weekday\": int}); X_train = X_train.astype({\"readable_time\": int})\n",
    "    X_test = X_test.astype({\"weekday\": int}); X_test = X_test.astype({\"readable_time\": int})\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "def accuracy_at_k(y_pred_prob, y_test, clf, k = 3):\n",
    "    predictions_array = np.zeros([len(y_test),k],dtype=int)\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    le.classes_ = clf.classes_\n",
    "    acc_k = 0; weighted_acc_k = 0\n",
    "    \n",
    "    for counter in range(len(y_test)):\n",
    "        predictions_array[counter] = np.flip(np.argsort(y_pred_prob[counter])[-k:])\n",
    "    \n",
    "    for n in range(k):\n",
    "        nth_preds = predictions_array[:,n]\n",
    "        nth_preds_labels = le.inverse_transform(nth_preds)\n",
    "        nth_acc = accuracy_score(y_test, nth_preds_labels)\n",
    "        acc_k += nth_acc \n",
    "        weighted_acc_k += (nth_acc/(n+1))\n",
    "    return acc_k, weighted_acc_k\n",
    "\n",
    "# Evaluation scripts\n",
    "def evaluate_model(test_pred_prob, test_classes):\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(test_classes)\n",
    "    test_pred = lb.transform(np.argmax(test_pred_prob, axis=-1))\n",
    "    # Accuracy\n",
    "    accuracy = 100 * accuracy_score(test_classes, test_pred)\n",
    "    print(\"Exact match accuracy is: \" + str(accuracy) + \"%\")\n",
    "    # Area Under the Receiver Operating Characteristic Curve (ROC AUC)\n",
    "    auc_roc = roc_auc_score(test_classes, test_pred_prob)\n",
    "    print(\"Macro Area Under the Curve (AUC) is: \" + str(auc_roc))\n",
    "    auc_roc_micro = roc_auc_score(test_classes, test_pred_prob, average=\"micro\")\n",
    "    print(\"Micro Area Under the Curve (AUC) is: \" + str(auc_roc_micro))\n",
    "    auc_roc_weighted = roc_auc_score(test_classes, test_pred_prob, average=\"weighted\")\n",
    "    print(\"Weighted Area Under the Curve (AUC) is: \" + str(auc_roc_weighted))\n",
    "    return accuracy, auc_roc\n",
    "\n",
    "def create_analysis_report(model_output, groundtruth, LABELS_LIST):\n",
    "    # Create a dataframe where we keep all the evaluations, starting by prediction accuracy\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(groundtruth)\n",
    "    model_output_rounded = lb.transform(np.argmax(model_output, axis=-1))\n",
    "    \n",
    "    accuracies_perclass = sum(model_output_rounded == groundtruth) / len(groundtruth)\n",
    "    results_df = pd.DataFrame(columns=LABELS_LIST)\n",
    "    results_df.index.astype(str, copy=False)\n",
    "    percentage_of_positives_perclass = sum(groundtruth) / len(groundtruth)\n",
    "    results_df.loc[0] = percentage_of_positives_perclass\n",
    "    results_df.loc[1] = accuracies_perclass\n",
    "    results_df.index = ['Ratio of positive samples', 'Model accuracy']\n",
    "    \n",
    "    \"\"\"\n",
    "    # plot the accuracies per class\n",
    "    results_df.T.plot.bar(figsize=(22, 12), fontsize=18)\n",
    "    plt.title('Model accuracy vs the ratio of positive samples per class')\n",
    "    plt.xticks(rotation=45)\n",
    "    plt.savefig(os.path.join(output_path, \"accuracies_vs_positiveRate.pdf\"), format=\"pdf\")\n",
    "    plt.savefig(os.path.join(output_path, \"accuracies_vs_positiveRate.png\"))\n",
    "    \"\"\"\n",
    "    \n",
    "    # Getting the true positive rate perclass\n",
    "    true_positives_ratio_perclass = sum((model_output_rounded == groundtruth) * (groundtruth == 1)) / sum(groundtruth)\n",
    "    results_df.loc[2] = true_positives_ratio_perclass\n",
    "    # Get true negative ratio\n",
    "    true_negative_ratio_perclass = sum((model_output_rounded == groundtruth)\n",
    "                                       * (groundtruth == 0)) / (len(groundtruth) - sum(groundtruth))\n",
    "    results_df.loc[3] = true_negative_ratio_perclass\n",
    "    # compute additional metrics (AUC,f1,recall,precision)\n",
    "    auc_roc_per_label = roc_auc_score(groundtruth, model_output, average=None)\n",
    "    precision_perlabel = precision_score(groundtruth, model_output_rounded, average=None)\n",
    "    recall_perlabel = recall_score(groundtruth, model_output_rounded, average=None)\n",
    "    f1_perlabel = f1_score(groundtruth, model_output_rounded, average=None)\n",
    "    kappa_perlabel = [cohen_kappa_score(groundtruth[:, x], model_output_rounded[:, x]) for x in range(len(LABELS_LIST))]\n",
    "    results_df = results_df.append(\n",
    "        pd.DataFrame([auc_roc_per_label,recall_perlabel, precision_perlabel, f1_perlabel, kappa_perlabel], columns=LABELS_LIST))\n",
    "    results_df.index = ['Ratio of positive samples', 'Model accuracy', 'True positives ratio',\n",
    "                        'True negatives ratio', \"AUC\", \"Recall\", \"Precision\", \"f1-score\", \"Kappa score\"]\n",
    "\n",
    "    \"\"\"\n",
    "    # Creating evaluation plots\n",
    "    plot_true_poisitve_vs_all_positives(model_output_rounded, groundtruth,\n",
    "                                        os.path.join(output_path, 'TruePositive_vs_allPositives'), LABELS_LIST)\n",
    "    plot_output_coocurances(model_output_rounded, os.path.join(output_path, 'output_coocurances'), LABELS_LIST)\n",
    "    plot_false_netgatives_confusion_matrix(model_output_rounded, groundtruth,\n",
    "                                           os.path.join(output_path, 'false_negative_coocurances'), LABELS_LIST)\n",
    "    \"\"\"\n",
    "    results_df['average'] = results_df.mean(numeric_only=True, axis=1)\n",
    "    #results_df.T.to_csv(os.path.join(output_path, \"results_report.csv\"), float_format=\"%.2f\")\n",
    "    return results_df\n",
    "\n",
    "# Formatting filenames within directories [DONE ONCE]\n",
    "def renameSplitFiles(SPLIT_NUMBERS, SPLIT_TYPES, LABELS_NUMBERS ,\n",
    "                     GROUNDTRUTH_PATH = \"/home/mounted/situational_playlist_generator/groundtruth/\"):\n",
    "    for splitNum in SPLIT_NUMBERS: \n",
    "        for splitTyp in SPLIT_TYPES:\n",
    "            for labelNum in LABELS_NUMBERS:\n",
    "                directory = GROUNDTRUTH_PATH + splitTyp + '/' + labelNum+ 'Classes/split_' + splitNum + '/'\n",
    "                for filename in os.listdir(directory):               \n",
    "                    if (\"audio_trainset\" in filename):\n",
    "                        newname = \"[split\" + splitNum + \"]audio_trainset[\" + labelNum + \"label_\" + splitTyp + \"].csv\"\n",
    "                        os.rename(os.path.join(directory, filename),os.path.join(directory, newname))\n",
    "\n",
    "                    elif (\"audio_testset\" in filename):\n",
    "                        newname = \"[split\" + splitNum + \"]audio_testset[\" + labelNum + \"label_\" + splitTyp + \"].csv\"\n",
    "                        os.rename(os.path.join(directory, filename),os.path.join(directory, newname))\n",
    "\n",
    "                    elif (\"trainset\" in filename):\n",
    "                        newname = \"[split\" + splitNum + \"]trainset[\" + labelNum + \"label_\" + splitTyp + \"].csv\"\n",
    "                        os.rename(os.path.join(directory, filename),os.path.join(directory, newname))\n",
    "\n",
    "                    elif (\"testset\" in filename): \n",
    "                        newname = \"[split\" + splitNum + \"]testset[\" + labelNum + \"label_\" + splitTyp + \"].csv\"\n",
    "                        os.rename(os.path.join(directory, filename),os.path.join(directory, newname))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Situation Predictor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# load the splitted dataset and format it for the stream and audio models\n",
    "def loadData_sitPred(labelNum, splitTyp, splitNum, GROUNDTRUTH_PATH):\n",
    "    dataDir = GROUNDTRUTH_PATH + splitTyp + '/' + labelNum+ 'Classes/split_' + splitNum\n",
    "    \n",
    "    trainset = pd.read_csv(dataDir + \"/[split\" + splitNum + \"]trainset[\" \n",
    "                           + labelNum + \"label_\" + splitTyp + \"].csv\")\n",
    "    \n",
    "    testset = pd.read_csv(dataDir + \"/[split\" + splitNum + \"]testset[\" \n",
    "                          + labelNum + \"label_\" + splitTyp + \"].csv\")\n",
    "\n",
    "    # Selecting SitPred data\n",
    "    y_train = trainset.matches\n",
    "    y_test = testset.matches\n",
    "    X_train_streams = trainset.drop([\"user_id\",\"media_id\",\"matches\",\"fold\"],axis=1)\n",
    "    X_test_streams = testset.drop([\"user_id\",\"media_id\",\"matches\",\"fold\"],axis=1)\n",
    "    \n",
    "    \"\"\"\n",
    "    # Save a version of the dataset readable for the audio model training \n",
    "    trainset.rename(columns={\"media_id\": \"song_id\"},inplace=True)\n",
    "    testset.rename(columns={\"media_id\": \"song_id\"},inplace=True)\n",
    "    audio_train_data = trainset[[\"user_id\",\"song_id\",\"matches\"]] \n",
    "    audio_test_data = testset[[\"user_id\",\"song_id\",\"matches\"]] \n",
    "\n",
    "    ## DO I NEED TO SAVE THEM?? (I think for the dataset pipeline??)\n",
    "    audio_train_data.to_csv(\"/home/mounted/groundtruths/audio_trainset[4label_noUseroverlap]\",index=False)\n",
    "    audio_test_data.to_csv(\"/home/mounted/groundtruths/audio_testset[4label_noUseroverlap]\",index=False)\n",
    "    \"\"\"\n",
    "    \n",
    "    return X_train_streams, y_train, X_test_streams, y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read and train Situation Predictor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "code_folding": [
     0,
     3
    ]
   },
   "outputs": [],
   "source": [
    "# Training and Evaluating XGBOOST\n",
    "# Training\n",
    "\n",
    "def train_xgb(X_train_streams, y_train):\n",
    "    print(\"Training XGB model..\")\n",
    "    xgb_model = xgb.XGBClassifier(n_jobs=1).fit(X_train_streams, y_train)\n",
    "    print('Accuracy of xgb classifier on training set: {:.2f}'\n",
    "         .format(xgb_model.score(X_train_streams, y_train)))\n",
    "    return xgb_model\n",
    "\n",
    "# Evaluation \n",
    "def test_xgb(X_test_streams, y_test, xgb_model, label_list, results_path):\n",
    "    print(\"Evaluating XGB model..\")\n",
    "    y_pred = xgb_model.predict(X_test_streams)\n",
    "    report = classification_report(y_test, y_pred,labels = label_list, output_dict=True)\n",
    "    report_df = pd.DataFrame.from_dict(report)\n",
    "    # [TODO] SAVE_REPORT\n",
    "    cm = confusion_matrix(y_test, y_pred, labels=label_list)\n",
    "    cm_df = pd.DataFrame(cm,index=label_list, columns=label_list)\n",
    "    # [TODO] SAVE_CONFUSION\n",
    "    # predict top 3\n",
    "    y_pred_prob = xgb_model.predict_proba(X_test_streams)\n",
    "    acc_k, weighted_acc_k = accuracy_at_k(y_pred_prob, y_test, xgb_model, k=3)\n",
    "    report_df[[\"accuracy@k\"]] = acc_k\n",
    "    report_df[[\"weighted accuracy@k\"]] = weighted_acc_k\n",
    "    print(\"accuracy for 3 top predictions: \" + str(acc_k))\n",
    "    print(\"Weighted accuracy for 3 top predictions: \" + str(weighted_acc_k)) \n",
    "    \n",
    "    np.save(results_path + \"sit_pred_prop.npy\", y_pred_prob)\n",
    "    np.save(results_path + \"sit_pred_test_gt.npy\", y_test)\n",
    "    report_df.to_csv(results_path + \"sit_pred_report.csv\")\n",
    "    cm_df.to_csv(results_path + \"sit_pred_confusion_matrix.csv\")\n",
    "    \n",
    "    return report_df, cm_df, y_pred\n",
    "    #MAKE DATAFRAME. SAVE ALL RESULTS IN CSV\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Autotagger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": [
     0,
     1,
     8,
     13,
     36
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Auto-tagger model\n",
    "class Conv_2d(nn.Module):\n",
    "    def __init__(self, input_channels, output_channels, shape=3, stride=1, pooling=2):\n",
    "        super(Conv_2d, self).__init__()\n",
    "        self.conv = nn.Conv2d(input_channels, output_channels, shape, stride=stride, padding=shape//2)\n",
    "        #self.bn = nn.BatchNorm2d(output_channels)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.mp = nn.MaxPool2d(pooling)\n",
    "    def forward(self, x):\n",
    "        #out = self.mp(self.relu(self.bn(self.conv(x))))\n",
    "        out = self.mp(self.relu(self.conv(x)))\n",
    "        return out\n",
    "\n",
    "class user_autotagger(nn.Module):\n",
    "    def __init__(self, NUM_CLASSES):\n",
    "        super(user_autotagger, self).__init__()\n",
    "        self.a_norming = nn.BatchNorm2d(1)\n",
    "        self.u_norming = nn.BatchNorm1d(256)\n",
    "        self.to_db = torchaudio.transforms.AmplitudeToDB()\n",
    "\n",
    "        self.conv1 = Conv_2d(1,32)\n",
    "        self.conv2 = Conv_2d(32,64)\n",
    "        self.conv3 = Conv_2d(64,128)\n",
    "        self.conv4 = Conv_2d(128,256)\n",
    "        \n",
    "        self.a_fc1 =  nn.Linear(256*6*40, 512)\n",
    "        self.a_fc2 = nn.Linear(512, 256)\n",
    "        self.a_fc3 = nn.Linear(256, 128)       \n",
    "        \n",
    "        self.u_embed1 = nn.Linear(EMBEDDINGS_DIM, 128+64)\n",
    "        self.u_embed2 = nn.Linear(128+64, 128)\n",
    "        \n",
    "        self.merged_fc = nn.Linear(128+128, 128) # 256 -> results of concats\n",
    "        self.drop = nn.Dropout(p=0.3)\n",
    "        self.logits  = nn.Linear(128, NUM_CLASSES)\n",
    "        \n",
    "    def forward(self,audio_input, user_input):\n",
    "        #Audio Branch \n",
    "        audio_db = self.to_db(audio_input)\n",
    "        audio_norm = self.a_norming(audio_db)\n",
    "        \n",
    "        x_audio = self.conv1(audio_norm)\n",
    "        x_audio = self.conv2(x_audio)\n",
    "        x_audio = self.conv3(x_audio)\n",
    "        x_audio = self.conv4(x_audio)\n",
    "\n",
    "        x_audio = x_audio.view(x_audio.size(0), -1)\n",
    "        x_audio = F.relu(self.a_fc1(x_audio))\n",
    "        x_audio = F.relu(self.a_fc2(x_audio))\n",
    "        x_audio = F.relu(self.a_fc3(x_audio))\n",
    "        \n",
    "        #User Branch\n",
    "        #user_norm = self.u_norming(user_input)\n",
    "        #user_norm = user_input ## [Need to figure out BatchNorm on 1D]\n",
    "        #x_user = F.relu(self.u_embed1(user_norm))\n",
    "        x_user = F.relu(self.u_embed1(user_input))\n",
    "        x_user = F.relu(self.u_embed2(x_user))\n",
    "        \n",
    "        #Merged Branch\n",
    "        x_conc = torch.cat((x_audio, x_user), 1)\n",
    "        x_merged = torch.sigmoid(self.merged_fc(x_conc))\n",
    "        x_merged = self.drop(x_merged)\n",
    "        logits = self.logits(x_merged)\n",
    "        output = F.softmax(logits,dim=1)\n",
    "        return output, logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [
     0,
     1,
     2,
     9,
     12
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Defining dataset pipeline \n",
    "class UserAwareDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, labels_csv, mels_folder, label_encoder,  device = 'cpu', transform = None):\n",
    "        self.df = pd.read_csv(labels_csv)\n",
    "        #self.embeds = pd.read_csv(embeds_csv)\n",
    "        self.mels_folder = mels_folder\n",
    "        self.transform = transform\n",
    "        self.device = device\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        track_id = self.df[\"song_id\"][index]\n",
    "        label = label_encoder.transform([self.df[\"matches\"][index]])\n",
    "        label = torch.from_numpy(label)\n",
    "        \n",
    "        # Check it's returnning the expected format [NEED TO LOAD usr_idx_ordered and W out (change that)]\n",
    "        user_id = self.df[\"user_id\"][index]  \n",
    "        usridx = usr_idx_ordered.index(user_id)\n",
    "        user_embeddings = W[usridx,:]\n",
    "        #user_embeddings = self.embeds[self.embeds.user_id == user_id].iloc[:,1:].values.flatten()[0]\n",
    "        #user_embeddings = np.asarray(ast.literal_eval(user_embeddings))\n",
    "        user_embeddings = user_embeddings.astype(np.float32) #.reshape(1,-1)\n",
    "        \n",
    "        # this is to ensure all mels have same shape (padded if missing)\n",
    "        mel_spec = torch.zeros(1,96,646)\n",
    "        try:\n",
    "            loaded_spec = torch.from_numpy(np.load(os.path.join(self.mels_folder, str(track_id)+\".npz\"))['arr_0'])\n",
    "        except:\n",
    "            loaded_spec = torch.from_numpy(np.load(os.path.join(self.mels_folder, str(float(track_id))+\".npz\"))['arr_0'])\n",
    "        if(loaded_spec.dim() == 2):\n",
    "            loaded_spec = torch.unsqueeze(loaded_spec,0)\n",
    "        if (loaded_spec.shape[1] != 96):\n",
    "            loaded_spec = loaded_spec.permute(0,2,1)\n",
    "        mel_spec[:, :, :loaded_spec.shape[2]] = loaded_spec\n",
    "        #mel_spec = torch.unsqueeze(mel_spec, 0)\n",
    "        if self.transform is not None:\n",
    "            \"\"\"MAKE DB HERE? could be faster\"\"\"\n",
    "            mel_spec = self.transform(mel_spec) \n",
    "\n",
    "        return mel_spec, user_embeddings , label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "code_folding": [
     0
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# initiating dataloader \n",
    "def initialize_audio_data(labelNum, splitTyp, splitNum, label_encoder, GROUNDTRUTH_PATH, MELS_PATH):    \n",
    "    dataDir = GROUNDTRUTH_PATH + splitTyp + '/' + labelNum+ 'Classes/split_' + splitNum\n",
    "    trainDataDir = dataDir + \"/[split\" + splitNum + \"]audio_trainset[\" + labelNum + \"label_\" + splitTyp + \"].csv\"\n",
    "    testDataDir = dataDir + \"/[split\" + splitNum + \"]audio_testset[\" + labelNum + \"label_\" + splitTyp + \"].csv\"\n",
    "    \n",
    "    train_instance = UserAwareDataset(trainDataDir, MELS_PATH, label_encoder)\n",
    "    test_instance = UserAwareDataset(testDataDir, MELS_PATH, label_encoder)\n",
    "    \n",
    "    train_loader = torch.utils.data.DataLoader(train_instance,batch_size=32,shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(test_instance,batch_size=32,shuffle=False)\n",
    "    \n",
    "    # Get weights for CE training (the ratio of each class in the dataset)\n",
    "    train_GT = pd.read_csv(dataDir + \"/[split\" + splitNum + \"]trainset[\" \n",
    "                           + labelNum + \"label_\" + splitTyp + \"].csv\")\n",
    "    values = label_encoder.transform(train_GT.matches)\n",
    "    n_values = np.max(values) + 1\n",
    "    train_GT = np.eye(n_values)[values]\n",
    "    POS_WEIGHTS = len(train_GT)/train_GT.sum(axis=0)\n",
    "    POS_WEIGHTS = [np.float32(x) for x in POS_WEIGHTS] # Do I still need this??\n",
    "    POS_WEIGHTS = torch.FloatTensor(POS_WEIGHTS).to(device)\n",
    "    \n",
    "    #validation_instance = UserAwareDataset(\"/home/mounted/implicit_valid_set.csv\",\n",
    "    #                            \"/home/mounted/groundtruths/user_embeds_existing.csv\",\n",
    "    #                           \"/home/mounted/implicit_mels/\")\n",
    "    #valid_loader = torch.utils.data.DataLoader(validation_instance,batch_size=32,shuffle=True)\n",
    "\n",
    "    return train_loader, test_loader, POS_WEIGHTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "code_folding": [
     0
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get autotagger\n",
    "def get_autotagger(labelNum, device, POS_WEIGHTS):\n",
    "    # Define loss and optimizer\n",
    "    autotagger = user_autotagger(int(labelNum))\n",
    "    criterion = nn.CrossEntropyLoss(weight=POS_WEIGHTS)\n",
    "    #criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(autotagger.parameters(), lr=0.001, weight_decay=1e-4)\n",
    "\n",
    "    #Decaying learning rate\n",
    "    #decayRate = 0.98\n",
    "    #optimizer_decayLR = torch.optim.lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=decayRate)\n",
    "    autotagger.to(device)\n",
    "    return autotagger, optimizer, criterion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": [
     0,
     1
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Training loop\n",
    "def train_autotagger(autotagger, train_loader, optimizer, criterion):\n",
    "    for epoch in range(NUM_EPOCHS):  # loop over the dataset multiple times\n",
    "        autotagger.train()\n",
    "        epoch_loss = 0.0\n",
    "        correct = 0\n",
    "        # iterate the training set\n",
    "        with tqdm(train_loader, unit=\"batch\") as tepoch:\n",
    "            for data in tepoch:\n",
    "                tepoch.set_description(f\"Epoch {epoch+1}\")\n",
    "\n",
    "                #audio_in, user_embeds, labels = data\n",
    "                audio_in = data[0].to(device)\n",
    "                user_embeds = data[1].to(device)\n",
    "                labels = torch.squeeze(data[2]).to(device)\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward + backward + optimize\n",
    "                outputs,logits = autotagger(audio_in,user_embeds)\n",
    "                loss = criterion(logits, labels) #Notice, CE in pytorch requires targets as indices\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "\n",
    "                _, predicted_idx = torch.max(outputs.data, 1)\n",
    "                correct += (predicted_idx == labels).sum().item()\n",
    "\n",
    "                # compute epoch loss\n",
    "                epoch_loss += loss.item()\n",
    "                tepoch.set_postfix(loss=loss.item())\n",
    "\n",
    "    print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "code_folding": [
     1
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Evaluation of the model\n",
    "def test_autotagger(autotagger, test_loader, y_test, label_encoder, labels_list, results_path):\n",
    "    #test_GT = pd.read_csv(GROUNDTRUTH_PATH + \"trainset[\" + num_class + \"label_\" + split_type + \"].csv\")\n",
    "    values = label_encoder.transform(y_test)\n",
    "    n_values = np.max(values) + 1\n",
    "    test_labels = np.eye(n_values)[values]\n",
    "    test_gt_idx = np.zeros([len(y_test),1], dtype=float)\n",
    "    test_pred_prob = np.zeros_like(test_labels, dtype=float)\n",
    "    #test_one_hot = np.zeros_like(test_classes, dtype=float)\n",
    "\n",
    "    autotagger.eval()\n",
    "    with torch.no_grad():\n",
    "        for step, (audio_in,user_embeds,labels) in enumerate(test_loader):\n",
    "            audio_in = audio_in.to(device); user_embeds = user_embeds.to(device)\n",
    "            outputs, logits = autotagger(audio_in,user_embeds)\n",
    "            #_, predicted_idx = torch.max(outputs.data, 1)\n",
    "            #_, labels_idx = torch.max(labels, 1)\n",
    "\n",
    "            # save all predictions and GT in on single array (redoing the GT to be aligned with preds)\n",
    "            start_idx = (step * BATCH_SIZE); end_idx = (step * BATCH_SIZE) + labels.size(0)\n",
    "            test_gt_idx[start_idx:end_idx, :] = labels.cpu()\n",
    "            test_pred_prob[start_idx:end_idx, :] = outputs.cpu()\n",
    "\n",
    "    \"\"\"Do you wanna save the user_ids and tracks? \"\"\"\n",
    "    #test_one_hot = np.zeros_like(test_classes, dtype=float)\n",
    "    #test_song_ids = np.zeros([test_classes.shape[0],1])\n",
    "    #test_user_ids = np.zeros([test_classes.shape[0],1])\n",
    "\n",
    "        #test_song_ids[start_idx:end_idx] = test_batch[3].reshape([-1, 1])\n",
    "        #test_user_ids[start_idx:end_idx] = test_batch[4].reshape([-1, 1])\n",
    "\n",
    "        #test_pred_classes = \n",
    "\n",
    "    accuracy_out, auc_roc = evaluate_model(test_pred_prob, test_labels)\n",
    "    results = create_analysis_report(test_pred_prob, test_labels, labels_list)\n",
    "    results[[\"Exact Match\"]] = accuracy_out\n",
    "    \n",
    "    np.save(results_path + \"autotagger_test_gt.npy\", test_labels)\n",
    "    np.save(results_path + \"autotagger_pred_prob.npy\", test_pred_prob)\n",
    "    results.to_csv(results_path + \"autotagger_report.csv\")\n",
    "    return results, test_pred_prob, test_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## JOINT evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "code_folding": [
     1
    ],
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Match the prediction of the test set from the two models [MAKE RESULTS PER LABEL???]\n",
    "def joint_evaluation(y_pred, test_pred_prob,test_labels, label_encoder, labels_list, results_path):\n",
    "    Audio_preds = label_encoder.inverse_transform(np.argmax(test_pred_prob,axis=1))\n",
    "    GT_labels = label_encoder.inverse_transform(np.argmax(test_labels,axis=1))\n",
    "    correct = 0\n",
    "    not_correct_idx = []\n",
    "    for counter in range(len(GT_labels)):\n",
    "        if len({Audio_preds[counter], GT_labels[counter], y_pred[counter]}) == 1: ## WHAT WAS I DOING HERE??\n",
    "            correct+=1\n",
    "        else:\n",
    "            not_correct_idx.append(counter)\n",
    "    joint_accuracy = (correct/len(GT_labels))\n",
    "    print(\"XGBOOST accuracy of mathing predictions: %.4f\" % joint_accuracy)\n",
    "    with open(results_path + \"joint_evaluation.txt\", \"w\") as f:\n",
    "        f.write(str(joint_accuracy))\n",
    "    return joint_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Initialize constants\n",
    "SPLIT_TYPES = [\"ColdUser\" , \"ColdTrack\", \"WarmCase\"]\n",
    "SPLIT_NUMBERS = [\"1\",\"2\",\"3\",\"4\"]\n",
    "LABELS_NUMBERS = [\"4\",\"8\",\"12\"]\n",
    "\n",
    "INPUT_SHAPE = (1, 96, 646)\n",
    "EMBEDDINGS_DIM = 128\n",
    "BATCH_SIZE = 32\n",
    "LABELS_LISTS = {\"4\" : ['gym', 'party', 'sleep', 'work'],\n",
    "                \"8\" : ['dance','gym','morning','night', 'party','running','sleep', 'work'],\n",
    "                \"12\" : ['car', 'club', 'dance','gym','morning','night', \n",
    "                     'party','relax', 'running','sleep', 'train', 'work']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "GROUNDTRUTH_PATH = \"/home/mounted/situational_playlist_generator/groundtruth/\"\n",
    "MELS_PATH = \"/home/mounted/mel_folders/MELS_MERGED/\"\n",
    "RESULTS_PATH = \"/home/mounted/situational_playlist_generator/Results/\"\n",
    "model_save_path = \"/home/mounted/situational_playlist_generator/trained_models/autotagger_\"\n",
    "\n",
    "# What do we do about this??\n",
    "# Load the ids<->indexes\n",
    "with open(\"/home/mounted/situational_playlist_generator/groundtruth/user_idx_ordered.txt\", \"rb\") as fp:\n",
    "    usr_idx_ordered = pickle.load(fp)\n",
    "# Load embeddings matrix (acces with index mapped through the user id)\n",
    "W  = np.load(\"/home/mounted/situational_playlist_generator/groundtruth/users_matrix.npz.npy\")\n",
    "\n",
    "NUM_EPOCHS = 5\n",
    "#min_val_loss = 10**5 #just initialize with random big number \n",
    "#epochs_no_improve = 0\n",
    "#n_epochs_stop = 10\n",
    "\n",
    "for splitNum in SPLIT_NUMBERS: \n",
    "    for splitTyp in SPLIT_TYPES:\n",
    "        for labelNum in LABELS_NUMBERS: \n",
    "            # select right labels list\n",
    "            labels_list = LABELS_LISTS[labelNum]\n",
    "            results_path = RESULTS_PATH + splitTyp + '/' + labelNum+ 'Classes/split_' + splitNum + \"/\"\n",
    "            \n",
    "            #Situation Predictor\n",
    "            X_train_streams, y_train, X_test_streams, y_test = loadData_sitPred(labelNum, \n",
    "                                                                                splitTyp, splitNum, \n",
    "                                                                                GROUNDTRUTH_PATH)\n",
    "            xgb_model = train_xgb(X_train_streams, y_train)\n",
    "            report, cm, sitPred_y_pred = test_xgb(X_test_streams, y_test, xgb_model,labels_list, results_path)\n",
    "            \n",
    "            #Autotagger\n",
    "            \n",
    "            label_encoder = preprocessing.LabelEncoder()\n",
    "            label_encoder.fit(labels_list)\n",
    "            \n",
    "            device = torch.device(\"cuda:2\" if torch.cuda.is_available() else \"cpu\")\n",
    "            print(\"Using device: \" + str(device))\n",
    "            \n",
    "            train_loader, test_loader, POS_WEIGHTS = initialize_audio_data(labelNum, splitTyp, splitNum, \n",
    "                                                                           label_encoder, GROUNDTRUTH_PATH,\n",
    "                                                                           MELS_PATH)\n",
    "            \n",
    "            \n",
    "            autotagger, optimizer, criterion = get_autotagger(labelNum, device, POS_WEIGHTS)\n",
    "            train_autotagger(autotagger, train_loader, optimizer, criterion)\n",
    "            results, autotagger_y_pred_prob, test_labels = test_autotagger(autotagger, test_loader, y_test, \n",
    "                                                         label_encoder, labels_list, results_path)\n",
    "            \n",
    "            joint_results = joint_evaluation(sitPred_y_pred, autotagger_y_pred_prob,\n",
    "                                             test_labels, label_encoder, results_path)\n",
    "            \n",
    "            model_name = model_save_path + splitTyp + '_' + labelNum + 'Classes_split_' + splitNum\n",
    "            torch.save(autotagger.state_dict(),model_name)\n",
    "            torch.cuda.empty_cache()\n",
    "            print(\"================================================================\")\n",
    "            break\n",
    "        break\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}