16GB T4 on AWS - 3m 25s processing; 7m 52s total

gumdropsteve · Feb 3, 2020 · ca7e60d · ca7e60d
1 parent 3e391f2
commit ca7e60d
Showing 1 changed file with 5 additions and 85 deletions.
diff --git a/download_data.ipynb b/download_data.ipynb
@@ -412,95 +412,16 @@
    "source": [
     "then = time.time()\n",
     "total_time = then - now\n",
-    "\n",
+    "# what was our total runtime?\n",
     "print(f'{int(total_time//60)} min {int(total_time%60)} sec\\nTOTAL TIME: {total_time}')"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Sun Feb  2 11:43:50 2020       \n",
-      "+-----------------------------------------------------------------------------+\n",
-      "| NVIDIA-SMI 418.87.00    Driver Version: 418.87.00    CUDA Version: 10.1     |\n",
-      "|-------------------------------+----------------------+----------------------+\n",
-      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
-      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
-      "|===============================+======================+======================|\n",
-      "|   0  Tesla T4            On   | 00000000:00:1E.0 Off |                    0 |\n",
-      "| N/A   64C    P0    30W /  70W |  13173MiB / 15079MiB |      1%      Default |\n",
-      "+-------------------------------+----------------------+----------------------+\n",
-      "                                                                               \n",
-      "+-----------------------------------------------------------------------------+\n",
-      "| Processes:                                                       GPU Memory |\n",
-      "|  GPU       PID   Type   Process name                             Usage      |\n",
-      "|=============================================================================|\n",
-      "|    0     23229      C   /opt/tljh/user/bin/python                    363MiB |\n",
-      "+-----------------------------------------------------------------------------+\n",
-      "Sun Feb  2 11:43:50 2020       \n",
-      "+-----------------------------------------------------------------------------+\n",
-      "| NVIDIA-SMI 418.87.00    Driver Version: 418.87.00    CUDA Version: 10.1     |\n",
-      "|-------------------------------+----------------------+----------------------+\n",
-      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
-      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
-      "|===============================+======================+======================|\n",
-      "|   0  Tesla T4            On   | 00000000:00:1E.0 Off |                    0 |\n",
-      "| N/A   64C    P0    30W /  70W |  13173MiB / 15079MiB |      0%      Default |\n",
-      "+-------------------------------+----------------------+----------------------+\n",
-      "                                                                               \n",
-      "+-----------------------------------------------------------------------------+\n",
-      "| Processes:                                                       GPU Memory |\n",
-      "|  GPU       PID   Type   Process name                             Usage      |\n",
-      "|=============================================================================|\n",
-      "|    0     23229      C   /opt/tljh/user/bin/python                    363MiB |\n",
-      "+-----------------------------------------------------------------------------+\n"
-     ]
-    }
-   ],
-   "source": [
-    "# see how GPU is doing\n",
-    "!nvidia-smi\n",
-    "# clear up GPU memory\n",
-    "del gdf\n",
-    "# see how GPU is doing\n",
-    "!nvidia-smi"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# EXTRA"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "BlazingContext ready\n",
-      "CPU times: user 11.1 s, sys: 957 ms, total: 12 s\n",
-      "Wall time: 15.3 s\n"
-     ]
-    }
-   ],
-   "source": [
-    "%%time\n",
-    "from blazingsql import BlazingContext\n",
-    "import datashader as ds\n",
-    "from colorcet import fire\n",
-    "from datashader import transfer_functions as tf\n",
-    "# start up BalzingSQL\n",
-    "bc = BlazingContext()"
+    "# EXTRA\n",
+    "The below is optional and is meant to test the capacity of a 16GB Tesla T4 GPU (or whatever you have) when processing this much data."
    ]
   },
   {
@@ -509,7 +430,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# list of direct paths for all months\n",
+    "# list of direct paths for all months (jan - aug was most I was able to process)\n",
     "paths = [f'/home/jupyter-winston/turbo-telegram/data/taxi_0{i}.csv' if i < 10 \n",
     "         else f'/home/jupyter-winston/turbo-telegram/data/taxi_{i}.csv' for i in range(1, 9)]"
    ]
@@ -540,8 +461,7 @@
    ],
    "source": [
     "%%time\n",
-    "# make table from all 12 months at once\n",
-    "# bc.create_table('big_taxi', '/home/jupyter-winston/turbo-telegram/data/taxi_*.csv', header=0)\n",
+    "# make table from all 12 (8*) months at once\n",
     "bc.create_table('big_taxi', paths, header=0)"
    ]
   },