added all the inclass exercises and solutions

Author: timdavidlee

msan694_disco/examples/Week4/ex1.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "from pyspark import SparkContext\n",
+    "sc = SparkContext.getOrCreate()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "business = sc.textFile(\"../Data/filtered_registered_business_sf.csv\")\n",
+    "business_zip_name_pair = business.map(lambda x : x.split(\",\"))\\\n",
+    "                                .map(lambda x : (x[0],x[1])).distinct()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[(u'94103', u'Razaghi A Kiakojouri A'),\n",
+       " (u'94590', u'Julio Cesar De Moraes'),\n",
+       " (u'94103', u'Bernies Pet Shoppe Inc'),\n",
+       " (u'94122', u'Chan Cindy L'),\n",
+       " (u'94124', u'Ho Kwok M')]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "business_zip_name_pair.take(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "supervisor = sc.textFile(\"../Data/supervisor_sf.csv\")\n",
+    "supervisor_zip_id_pair = supervisor.map(lambda x : x.split(\",\"))\\\n",
+    "                                   .map(lambda x : (x[0],x[1])).distinct()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[(u'94118', u'1'),\n",
+       " (u'94134', u'9'),\n",
+       " (u'94118', u'5'),\n",
+       " (u'94134', u'11'),\n",
+       " (u'94111', u'6')]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "supervisor_zip_id_pair.take(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "business_without_supervisor = business_zip_name_pair.subtractByKey(supervisor_zip_id_pair)\\\n",
+    "                                                    .values()\\\n",
+    "                                                    .distinct()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "39422"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "business_without_supervisor.count()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[u'Precision Communication Serv',\n",
+       " u'Schefer Thomas R',\n",
+       " u'Lucid Systems',\n",
+       " u'Jacob Abraham',\n",
+       " u'Daniel Dela Rosa',\n",
+       " u'Sudhir Marahatta',\n",
+       " u'Batista Luis S',\n",
+       " u'\"Wti',\n",
+       " u'Boutin Jacqueline M',\n",
+       " u'Avinesh P Singh']"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "business_without_supervisor.take(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python [Root]",
+   "language": "python",
+   "name": "Python [Root]"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}