02-Experiment-Clean-Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2a6d445e",
   "metadata": {},
   "source": [
    "# This file is used to obtain a clean dataset\n",
    "\n",
    "Based on raw data .csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71d62fec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# datafolder = \"C:\\\\Users\\\\xinxi\\\\Purdue\\\\EffectiveSocialLearningProject\\\\DataAnalysis\\\\RawData\\\\Session_Formal\"\n",
    "datafolder = \"..\\\\RawData\\\\Session_Formal\"\n",
    "outputfolder = \"..\\\\Output_Figures\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6487202d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import copy\n",
    "import os\n",
    "import random\n",
    "import math as math\n",
    "import matplotlib.pylab as plt #Typically matplotlib.pylab is imported as plt (shortened name)\n",
    "import pdb\n",
    "from scipy.optimize import minimize\n",
    "import shutil\n",
    "import csv\n",
    "from MyLatexFunctions_SocialLearning import * \n",
    "from itertools import permutations \n",
    "from itertools import combinations\n",
    "import csv\n",
    "\n",
    "\n",
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "22c9e8bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------\n",
      "session0.csv\n",
      "------------\n",
      "session1.csv\n",
      "------------\n",
      "session2.csv\n",
      "------------\n",
      "session3.csv\n",
      "------------\n",
      "session4.csv\n",
      "------------\n",
      "session5.csv\n",
      "------------\n",
      "session6.csv\n",
      "------------\n",
      "session7.csv\n",
      "------------\n",
      "session8.csv\n",
      "------------\n",
      "session9.csv\n",
      "------------\n",
      "session10.csv\n",
      "------------\n",
      "session11.csv\n",
      "------------\n",
      "session12.csv\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'session0.csv': ['pilot', '03/19/2021', 'NC', 'L', 0, 10],\n",
       " 'session1.csv': ['real_1', '04/09/2021', 'NC', 'L', 10, 20],\n",
       " 'session2.csv': ['real_2', '04/09/2021', 'NC', 'L', 20, 30],\n",
       " 'session3.csv': ['real_3', '04/09/2021', 'NC', 'L', 30, 40],\n",
       " 'session4.csv': ['real_4', '04/13/2021', 'NC', 'H', 40, 50],\n",
       " 'session5.csv': ['real_5', '04/13/2021', 'NC', 'H', 50, 60],\n",
       " 'session6.csv': ['real_6', '04/28/2021', 'C', 'H', 60, 70],\n",
       " 'session7.csv': ['real_7', '04/28/2021', 'C', 'H', 70, 80],\n",
       " 'session8.csv': ['real_8', '04/29/2021', 'C', 'L', 80, 90],\n",
       " 'session9.csv': ['real_9', '04/29/2021', 'C', 'L', 90, 100],\n",
       " 'session10.csv': ['real_10', '04/29/2021', 'C', 'L', 100, 110],\n",
       " 'session11.csv': ['real_11', '12/03/2021', 'C', 'L', 110, 120],\n",
       " 'session12.csv': ['real_12', '12/03/2021', 'C', 'L', 120, 130]}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sessions = {\n",
    "#             '2022_02_11_pilot_session1.csv':['pilot_1',125,'DynamicPower_05_Experiment.','T6 END',4], \n",
    "            'session0.csv': ['pilot','03/19/2021', 'NC', 'L' ],\n",
    "            'session1.csv': ['real_1','04/09/2021', 'NC', 'L' ],\n",
    "            'session2.csv': ['real_2','04/09/2021', 'NC', 'L' ],\n",
    "            'session3.csv': ['real_3','04/09/2021', 'NC', 'L' ],\n",
    "            'session4.csv': ['real_4','04/13/2021', 'NC', 'H' ],\n",
    "            'session5.csv': ['real_5','04/13/2021', 'NC', 'H' ],\n",
    "            'session6.csv': ['real_6','04/28/2021', 'C', 'H' ],\n",
    "            'session7.csv': ['real_7','04/28/2021', 'C', 'H' ],\n",
    "            'session8.csv': ['real_8','04/29/2021', 'C', 'L' ],\n",
    "            'session9.csv': ['real_9','04/29/2021', 'C', 'L' ],\n",
    "            'session10.csv': ['real_10','04/29/2021', 'C', 'L' ],\n",
    "            'session11.csv': ['real_11','12/03/2021', 'C', 'L' ],\n",
    "            'session12.csv': ['real_12','12/03/2021', 'C', 'L' ],\n",
    "\n",
    "    \n",
    "           }\n",
    "\n",
    "rows = {}\n",
    "\n",
    "nRows = 0\n",
    "for i,s in enumerate(sessions):\n",
    "    print(\"------------\")\n",
    "    print(s)\n",
    "    data=pd.read_csv(datafolder+'\\\\'+s)\n",
    "    sessions[s].append(nRows)\n",
    "    nRows += data.shape[0]\n",
    "    sessions[s].append(nRows)\n",
    "    treatment =  sessions[s][2] +  sessions[s][3]\n",
    "    if not treatment in rows:\n",
    "        rows[treatment]=np.arange(sessions[s][4],sessions[s][5])\n",
    "    else:\n",
    "        rows[treatment]=np.concatenate([rows[treatment],np.arange(sessions[s][4],sessions[s][5])])\n",
    "\n",
    "sessions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4120ed5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = {'PDCN': ['.player.robot_signal', '.player.robot_type', '.player.robot_period','.player.robot_guess','.subsession.storyline'],\n",
    "          'DCN': [ '.player.sec_spent', '.group.storyline', '.subsession.groupnumber', '.group.id_in_subsession']\n",
    "          }\n",
    "\n",
    "shared_columns = ['.player.current_ball', '.player.type','.player.period', '.player.choice_of_urn',  \n",
    "                 '.player.decision_period','.player.information_set_lastperiod', '.player.information_set_currentperiod',\n",
    "                  '.player.history','.player.time_on_page',\n",
    "                 '.subsession.round_number']\n",
    "\n",
    "additional_columns = ['participant.code','session.code','network','waiting_cost','date','app_name']\n",
    "\n",
    "ncols = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c77a8c4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "session0.csv\n",
      "session1.csv\n",
      "session2.csv\n",
      "session3.csv\n",
      "session4.csv\n",
      "session5.csv\n",
      "session6.csv\n",
      "session7.csv\n",
      "session8.csv\n",
      "session9.csv\n",
      "session10.csv\n",
      "session11.csv\n",
      "session12.csv\n",
      "session0.csv\n",
      "session1.csv\n",
      "session2.csv\n",
      "session3.csv\n",
      "session4.csv\n",
      "session5.csv\n",
      "session6.csv\n",
      "session7.csv\n",
      "session8.csv\n",
      "session9.csv\n",
      "session10.csv\n",
      "session11.csv\n",
      "session12.csv\n"
     ]
    }
   ],
   "source": [
    "# for j,part_name in enumerate(['part2_play.','part3_play.']):\n",
    "result = {}\n",
    "for part_name in columns.keys() :\n",
    "    out = {}\n",
    "    ncols = 40\n",
    "    for col in additional_columns+shared_columns+columns[part_name]:\n",
    "        out[ col]=np.empty((nRows,ncols))\n",
    "        out[ col][:]=np.nan\n",
    "\n",
    "    for i,s in enumerate(sessions):\n",
    "\n",
    "        print(s)\n",
    "        data=pd.read_csv(datafolder+'\\\\'+s)\n",
    "        start_row = sessions[s][4]\n",
    "        end_row = sessions[s][5]\n",
    "    #     for j,part_name in enumerate(['PDCN', 'DCN']):\n",
    "\n",
    "        if part_name == 'PDCN':\n",
    "            ncols = 10\n",
    "        else :\n",
    "            if i == 0:\n",
    "                ncols = 40\n",
    "            else :\n",
    "                ncols = 30\n",
    "        \n",
    "        for col in columns[part_name]+shared_columns:\n",
    "            if i <6 :\n",
    "                colnames = [part_name+'.'+str(x)+col for x in range(1,ncols+1)]\n",
    "            else :\n",
    "                colnames = [part_name+'_b.'+str(x)+col for x in range(1,ncols+1)]\n",
    "            temp=data[colnames].to_numpy()\n",
    "            if type(temp[0][0])==str:\n",
    "                out[ col]=out[ col].astype(str)\n",
    "            out[ col][start_row:end_row,0:ncols]=temp\n",
    "\n",
    "            #sessionID\n",
    "            colnames = ['session.code' for x in range(1,ncols+1)]\n",
    "            temp=data[colnames].to_numpy()\n",
    "            if type(temp[0][0])==str:\n",
    "                out['session.code']=out['session.code'].astype(str)\n",
    "            out[ 'session.code'][start_row:end_row,0:ncols]=temp\n",
    "\n",
    "            #subjectID\n",
    "            colnames = ['participant.code' for x in range(1,ncols+1)]\n",
    "            temp=data[colnames].to_numpy()\n",
    "            if type(temp[0][0])==str:\n",
    "                out[ 'participant.code']=out['participant.code'].astype(str)\n",
    "            out[ 'participant.code'][start_row:end_row,0:ncols]=temp\n",
    "\n",
    "            #treatment:\n",
    "\n",
    "\n",
    "            network = sessions[s][2]\n",
    "            out['network']=out['network'].astype(str)\n",
    "            out['network'][start_row:end_row,0:ncols]=network\n",
    "\n",
    "            waiting_cost = sessions[s][3]\n",
    "            out['waiting_cost']=out['waiting_cost'].astype(str)\n",
    "            out['waiting_cost'][start_row:end_row,0:ncols]=waiting_cost\n",
    "\n",
    "            date = sessions[s][1]\n",
    "            out['date']=out['date'].astype(str)\n",
    "            out['date'][start_row:end_row,0:ncols]=date\n",
    "\n",
    "            app_name = part_name\n",
    "            out['app_name']=out['app_name'].astype(str)\n",
    "            out['app_name'][start_row:end_row,0:ncols]=app_name\n",
    "\n",
    "\n",
    "#         temp = sessions[s][0]\n",
    "#         if type(temp)==str:\n",
    "#             out['session']=out['session'].astype(str)\n",
    "#         out['session'][start_row:end_row,0:ncols]=temp\n",
    "#     if result == {} :\n",
    "#         result.update(out)\n",
    "#     else :\n",
    "#         result = combine_dict(result, out)\n",
    "#     print(result['app_name'])\n",
    "    for col in out:\n",
    "        out[col]=out[col].flatten()\n",
    "    result[part_name] = out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1562eec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "out1 =  pd.DataFrame(result['PDCN'])\n",
    "out2 =  pd.DataFrame(result['DCN'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "58a084db",
   "metadata": {},
   "outputs": [],
   "source": [
    "out = pd.concat([out1, out2])\n",
    "out = out[out['app_name']!='nan']\n",
    "out.to_csv(\"02-Experiment-Clean-Data.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c990c7da",
   "metadata": {},
   "source": [
    "# End of the main file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7707bd01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "session0.csv\n",
      "session1.csv\n",
      "session2.csv\n",
      "session3.csv\n",
      "session4.csv\n",
      "session5.csv\n",
      "session6.csv\n",
      "session7.csv\n",
      "session8.csv\n",
      "session9.csv\n",
      "session10.csv\n",
      "session11.csv\n",
      "session12.csv\n"
     ]
    }
   ],
   "source": [
    "# This one has both apps side by side\n",
    "\n",
    "columns = {'PDCN': ['.player.current_ball', '.player.type', '.player.period', '.player.decision_period', \n",
    "                    '.player.information_set_lastperiod', '.subsession.round_number',\n",
    "          '.player.information_set_currentperiod', '.player.history','.player.time_on_page','.player.robot_type', \n",
    "                    '.player.robot_signal',\n",
    "          '.player.robot_period','.player.robot_guess','.player.choice_of_urn', '.subsession.storyline',],\n",
    "          'DCN': ['.player.payoff', '.player.sec_spent','.player.current_ball','.player.type','.player.period',\n",
    "                 '.player.decision_period','.player.information_set_lastperiod', '.player.information_set_currentperiod',\n",
    "                 '.player.history','.player.time_on_page','.player.choice_of_urn', '.group.storyline', \n",
    "                  '.subsession.round_number','.subsession.groupnumber', '.group.id_in_subsession',\n",
    "                 ]\n",
    "          \n",
    "          \n",
    "          }\n",
    "\n",
    "additional_columns = ['participant.code','session.code','network','waiting_cost','date','app_name']\n",
    "\n",
    "ncols = 40\n",
    "# for j,part_name in enumerate(['part2_play.','part3_play.']):\n",
    "out = {}\n",
    "for part_name in columns.keys() :\n",
    "    for col in additional_columns+columns[part_name]:\n",
    "        out[part_name + '_' + col]=np.empty((nRows,ncols))\n",
    "        out[part_name + '_' + col][:]=np.nan\n",
    "\n",
    "for i,s in enumerate(sessions):\n",
    "    \n",
    "    print(s)\n",
    "    data=pd.read_csv(datafolder+'\\\\'+s)\n",
    "    start_row = sessions[s][4]\n",
    "    end_row = sessions[s][5]\n",
    "#     for j,part_name in enumerate(['PDCN', 'DCN']):\n",
    "    for part_name in columns.keys() :\n",
    "        if part_name == 'PDCN':\n",
    "            ncols = 10\n",
    "        else :\n",
    "            if i == 0:\n",
    "                ncols = 40\n",
    "            else :\n",
    "                ncols = 30\n",
    "        \n",
    "        for col in columns[part_name]:\n",
    "            if i <6 :\n",
    "                colnames = [part_name+'.'+str(x)+col for x in range(1,ncols+1)]\n",
    "            else :\n",
    "                colnames = [part_name+'_b.'+str(x)+col for x in range(1,ncols+1)]\n",
    "            temp=data[colnames].to_numpy()\n",
    "            if type(temp[0][0])==str:\n",
    "                out[part_name + '_' + col]=out[part_name + '_' + col].astype(str)\n",
    "            out[part_name + '_' + col][start_row:end_row,0:ncols]=temp\n",
    "\n",
    "        #sessionID\n",
    "        colnames = ['session.code' for x in range(1,ncols+1)]\n",
    "        temp=data[colnames].to_numpy()\n",
    "        if type(temp[0][0])==str:\n",
    "            out[part_name + '_' +'session.code']=out[part_name + '_' +'session.code'].astype(str)\n",
    "        out[part_name + '_' + 'session.code'][start_row:end_row,0:ncols]=temp\n",
    "\n",
    "        #subjectID\n",
    "        colnames = ['participant.code' for x in range(1,ncols+1)]\n",
    "        temp=data[colnames].to_numpy()\n",
    "        if type(temp[0][0])==str:\n",
    "            out[part_name + '_' + 'participant.code']=out[part_name + '_' +'participant.code'].astype(str)\n",
    "        out[part_name + '_' + 'participant.code'][start_row:end_row,0:ncols]=temp\n",
    "\n",
    "        #treatment:\n",
    "\n",
    "\n",
    "        network = sessions[s][2]\n",
    "        out[part_name + '_' 'network']=out[part_name + '_' +'network'].astype(str)\n",
    "        out[part_name + '_' +'network'][start_row:end_row,0:ncols]=network\n",
    "\n",
    "        waiting_cost = sessions[s][3]\n",
    "        out[part_name + '_' +'waiting_cost']=out[part_name + '_' +'waiting_cost'].astype(str)\n",
    "        out[part_name + '_' +'waiting_cost'][start_row:end_row,0:ncols]=waiting_cost\n",
    "        \n",
    "        date = sessions[s][1]\n",
    "        out[part_name + '_' +'date']=out[part_name + '_' +'date'].astype(str)\n",
    "        out[part_name + '_' +'date'][start_row:end_row,0:ncols]=date\n",
    "        \n",
    "        app_name = part_name\n",
    "        out[part_name + '_' +'app_name']=out[part_name + '_' +'app_name'].astype(str)\n",
    "        out[part_name + '_' +'app_name'][start_row:end_row,0:ncols]=app_name\n",
    "        \n",
    "        \n",
    "#         temp = sessions[s][0]\n",
    "#         if type(temp)==str:\n",
    "#             out['session']=out['session'].astype(str)\n",
    "#         out['session'][start_row:end_row,0:ncols]=temp\n",
    "for col in out:\n",
    "#     print(col)\n",
    "    out[col]=out[col].flatten()\n",
    "out_data = pd.DataFrame(out)\n",
    "\n",
    "out_data.to_csv(\"02-Experiment-Analysis-CleanedData.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b84ceb5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in out:\n",
    "#     print(col)\n",
    "    out[col]=out[col].flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2a3e6e3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PDCN_participant.code': array(['c5epu2kj', 'c5epu2kj', 'c5epu2kj', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'PDCN_session.code': array(['pv7kqlyd', 'pv7kqlyd', 'pv7kqlyd', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'PDCN_network': array(['NC', 'NC', 'NC', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_waiting_cost': array(['L', 'L', 'L', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_date': array(['03/19/2021', '03/19/2021', '03/19/2021', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'PDCN_app_name': array(['PDCN', 'PDCN', 'PDCN', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.player.current_ball': array(['red', 'blue', 'red', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.player.type': array(['A', 'A', 'B', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.player.period': array([ 3.,  5.,  1., ..., nan, nan, nan]),\n",
       " 'PDCN_.player.decision_period': array([ 3.,  5.,  1., ..., nan, nan, nan]),\n",
       " 'PDCN_.player.information_set_lastperiod': array(['A,E,', 'A,E,', 'B,A,C,D,', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.subsession.round_number': array([ 1.,  2.,  3., ..., nan, nan, nan]),\n",
       " 'PDCN_.player.information_set_currentperiod': array(['A,E,', 'A,E,', 'B,A,C,D,', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.player.history': array([\"[1, 3, {'A': 'red', 'E': 'blue'}\",\n",
       "        \"[2, 5, {'A': 'blue', 'E': 'red'}\",\n",
       "        \"[1, 1, {'B': 'red', 'A': 'red', \", ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'PDCN_.player.time_on_page': array([nan, nan, nan, ..., nan, nan, nan]),\n",
       " 'PDCN_.player.robot_type': array(['B,C,D,E,', 'B,C,D,E,', 'A,C,D,E,', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'PDCN_.player.robot_signal': array(['blue,red,blue,blue,', 'blue,blue,blue,red,', 'red,blue,blue,red,',\n",
       "        ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.player.robot_period': array(['4,1,2,3,', '4,4,0,2,', '0,4,2,0,', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'PDCN_.player.robot_guess': array(['Blue,Red,Blue,Red,', 'Blue,Blue,Blue,Red,', 'Red,Blue,Red,Blue,',\n",
       "        ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.player.choice_of_urn': array(['Red', 'Red', 'Red', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'PDCN_.subsession.storyline': array(['Blue', 'Blue', 'Red', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_participant.code': array(['c5epu2kj', 'c5epu2kj', 'c5epu2kj', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'DCN_session.code': array(['pv7kqlyd', 'pv7kqlyd', 'pv7kqlyd', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'DCN_network': array(['NC', 'NC', 'NC', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_waiting_cost': array(['L', 'L', 'L', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_date': array(['03/19/2021', '03/19/2021', '03/19/2021', ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'DCN_app_name': array(['DCN', 'DCN', 'DCN', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.player.payoff': array([138., 138., 139., ...,  nan,  nan,  nan]),\n",
       " 'DCN_.player.sec_spent': array([216.566344, 123.637499, 236.080288, ...,        nan,        nan,\n",
       "               nan]),\n",
       " 'DCN_.player.current_ball': array(['red', 'red', 'red', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.player.type': array(['A', 'A', 'A', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.player.period': array([ 2.,  2.,  1., ..., nan, nan, nan]),\n",
       " 'DCN_.player.decision_period': array([ 2.,  2.,  1., ..., nan, nan, nan]),\n",
       " 'DCN_.player.information_set_lastperiod': array(['AE', 'AE', 'A', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.player.information_set_currentperiod': array(['AE', 'AE', 'AE', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.player.history': array([\"[[1, 2, {'A': 'red', 'E': 'blue'\",\n",
       "        \"[[1, 2, {'A': 'red', 'E': 'blue'\",\n",
       "        \"[[1, 2, {'A': 'red', 'E': 'blue'\", ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'DCN_.player.time_on_page': array([\"['page_instruction', 0, 22.82584\",\n",
       "        \"['page_instruction', 0, 22.82584\",\n",
       "        \"['page_instruction', 0, 22.82584\", ..., 'nan', 'nan', 'nan'],\n",
       "       dtype='<U32'),\n",
       " 'DCN_.player.choice_of_urn': array(['Red', 'Red', 'Blue', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.group.storyline': array(['Red', 'Red', 'Blue', ..., 'nan', 'nan', 'nan'], dtype='<U32'),\n",
       " 'DCN_.subsession.round_number': array([ 1.,  2.,  3., ..., nan, nan, nan]),\n",
       " 'DCN_.subsession.groupnumber': array([ 1.,  1.,  1., ..., nan, nan, nan]),\n",
       " 'DCN_.group.id_in_subsession': array([ 1.,  1.,  1., ..., nan, nan, nan])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# out['PDCN_participant.code'].shape\n",
    "# out['DCN_participant.code'].shape\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "94ea55b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_data = pd.DataFrame(out)\n",
    "# Drop nan\n",
    "# out_data = out_data[~out_data['PDCN_.player.period'].isnull()]\n",
    "out_data.to_csv(\"02-Experiment-Analysis-CleanedData.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b57f045",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       False\n",
       "1       False\n",
       "2       False\n",
       "3       False\n",
       "4       False\n",
       "        ...  \n",
       "5195     True\n",
       "5196     True\n",
       "5197     True\n",
       "5198     True\n",
       "5199     True\n",
       "Name: PDCN_.player.period, Length: 5200, dtype: bool"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# out_data['PDCN_participant.code'].isnull()\n",
    "out_data['PDCN_.player.period'].isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31c05c84",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}