Skip to content

Commit

Permalink
Full 2015 Taxi Data Download & Processing Update
Browse files Browse the repository at this point in the history
[REVIEW] Full 2015 Data Update 

Merge pull request #14 from gumdropsteve/data/download
  • Loading branch information
gumdropsteve authored Feb 3, 2020
2 parents 047234a + 91b59b1 commit a0ebada
Show file tree
Hide file tree
Showing 2 changed files with 675 additions and 0 deletions.
622 changes: 622 additions & 0 deletions download_data.ipynb

Large diffs are not rendered by default.

53 changes: 53 additions & 0 deletions utils/sql_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import os
import sys


def bsql_start():
"""
set up users to use SQL in the RAPIDS AI ecosystem
> try to import BlazingContext from BlazingSQL
> offer to install BlazingSQL if module not found
BlazingSQL prereqs:
> Conda: https://docs.blazingdb.com/docs/install-via-conda#section-conda-prerequisites
> Docker: https://docs.blazingdb.com/docs/install-via-docker#section-docker-hub-prerequisites
latest install scripts:
> Conda: https://docs.blazingdb.com/docs/install-via-conda
> Docker: https://docs.blazingdb.com/docs/install-via-docker
> Source: https://docs.blazingdb.com/docs/build-from-source
"""
# is BlazingSQL installed?
try:
from blazingsql import BlazingContext
# yes, indicate success
return "You've got BlazingSQL set up perfectly!"
# BlazingSQL not found
except ModuleNotFoundError:
# do we want to install BlazingSQL?
print("Unable to locate BlazingSQL. We'll install it now.")
# Install JRE first
os.system("apt-get update")
os.system("apt-get -y install default-jre")
# tag BlazingSQL conda install script
b = "conda install -c blazingsql/label/cuda10.0 -c blazingsql"
b += ' -c rapidsai -c nvidia -c conda-forge -c defaults '
b += "blazingsql python=3.7 cudatoolkit=10.0" # CUDA 10, Python 3.7 (BlazingSQL also supports CUDA 9.2)
# tag python version
py = sys.version.split('.') # e.g. output: ['3', '6', '7 | packaged by cond...
if py[0] == '3': # make sure we're in 3
py = py[1] # focus mid version (3.?)
# are we on python 3.6?
if py == '6':
# adjust to 3.6 install script
b = b.replace('python=3.7', 'python=3.6')
# lmk what's going on?
print('Installing BlazingSQL, this should take some time. This is only need to be done once')
# install BlazingSQL
os.system(b)
# indicate completion
return f"Let's get started with SQL in RAPIDS AI!"


if __name__=='__main__':
# check environment for BlazingSQL
check = bsql_start()
print(check)

0 comments on commit a0ebada

Please sign in to comment.