Merge pull request #8 from tclements/newAWS

New aws

Author: tclements

.travis.yml

@@ -4,7 +4,7 @@ os:
   - linux
   - osx
 julia:
-  - 1.0
+  - 1.5
   - nightly
 notifications:
   email: false
@@ -14,7 +14,7 @@ jobs:
   fast_finish: true
   include:
     - stage: Documentation
-      julia: 1.0
+      julia: 1.5
       script: julia --project=docs -e '
           using Pkg;
           Pkg.develop(PackageSpec(path=pwd()));

Project.toml

@@ -4,9 +4,8 @@ authors = ["Tim Clements <[email protected]>"]
 version = "0.1.0"
 
 [deps]
-AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598"
+AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
 AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
-AWSSDK = "0d499d91-6ae5-5d63-9313-12987b87d5ad"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -17,6 +16,7 @@ SeisIO = "b372bb87-02dd-52bb-bcf6-c30dd83fd342"
 
 [compat]
 julia = "1"
+SeisIO = "1.2"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

README.md

@@ -29,8 +29,7 @@ SCEDC.jl supports direct download and file streaming for continuous data request
 
 ```julia
 # download data using SCEDC on AWS
-using SCEDC, Dates, AWSCore
-aws = aws_config(region="us-west-2")
+using SCEDC, Dates
 bucket = "scedc-pds"
 startdate = Date("2016-07-01")
 enddate = Date("2016-07-01")
@@ -39,20 +38,20 @@ channel = "LH?"
 OUTDIR = "~/data"
 
 # query s3 for matching stations
-filelist = s3query(aws, startdate, enddate=enddate, network=network,channel=channel)
+filelist = s3query(startdate, enddate=enddate, network=network,channel=channel)
 
 # download mseed files to disk 
-ec2download(aws,bucket,filelist,OUTDIR)
+ec2download(bucket,filelist,OUTDIR)
 
 # or stream to an Array of SeisIO.SeisData 
-LHs = ec2stream(aws,bucket,filelist)
+LHs = ec2stream(bucket,filelist)
 ```
 
 To do parallel data transfers, use the `addprocs` function from the `Distributed` module: 
 ```julia
 using Distributed
 addprocs()
-ec2download(aws,bucket,filelist,"~/paralleldir")
+ec2download(bucket,filelist,"~/paralleldir")
 ```
 
 ### Earthquake Catalog Requests 
@@ -69,7 +68,6 @@ SCEDC.jl queries the SCSN earthquake catalog using these possible parameters:
 
 ```julia
 julia> df = catalogquery(
-    aws,
     starttime = DateTime(1937,1,1),
     endtime=DateTime(1937,2,1),
     minlatitude=33.,
@@ -92,7 +90,7 @@ julia> df = catalogquery(
 SCEDC.jl can retrieve phase picks for an earthquake given the event's ID and date. Here is an example from the 1987 Elmore Ranch earthquake with data returned as a `SeisIO.SeisEvent`
 
 ```julia
-julia> phasequery(aws,628016,Date(1987,11,24))
+julia> phasequery(628016,Date(1987,11,24))
 Event 628016: SeisEvent with 70 channels
 
 (.hdr)
@@ -138,7 +136,7 @@ SCEDC.jl can stream waveforms (with associated phase picks) from earthquakes goi
 
 Here is an example of an requesting events from 2019, returned as an `Array` of `SeisEvent` 
 ```julia
-julia> EQ = eventstream(aws,starttime=Date(2019,1,1),endtime=Date(2019,1,2),minmagnitude=2.)
+julia> EQ = eventstream(starttime=Date(2019,1,1),endtime=Date(2019,1,2),minmagnitude=2.)
 julia> EQ[1].data
 SeisIO.Quake.EventTraceData with 2909 channels (3 shown)
     ID: AZ.BZN..BHE                        AZ.BZN..BHN                        AZ.BZN..BHZ                        …

src/SCEDC.jl

@@ -1,8 +1,10 @@
 module SCEDC
 
 using Dates, DelimitedFiles, Distributed, Random
-using AWSCore, AWSS3, AWSSDK,CSV, DataFrames, SeisIO 
+using AWS, AWSS3, CSV, DataFrames, SeisIO 
 using SeisIO.Quake
+@service S3 
+@service Athena
 
 # include modules
 include("catalog.jl")
@@ -12,4 +14,4 @@ include("s3.jl")
 include("athena.jl")
 include("ec2.jl")
 
-end
+end

src/athena.jl

@@ -1,11 +1,10 @@
 export athenasetup, athenaquery
 """
-  athenasetup(aws,bucket)
+  athenasetup(bucket)
 
 Create AWS Athena database and table for SQL queries.
 
 # Arguments
-- `aws::AWSConfig`: AWSConfig configuration dictionary
 - `bucket::String`: Name of AWS bucket to store query results
 """
 function athenasetup(aws::AWSConfig,bucket::String;
@@ -20,9 +19,9 @@ function athenasetup(aws::AWSConfig,bucket::String;
 
     # create database
     println("Creating database '$database' ",now())
-    AWSSDK.Athena.start_query_execution(aws,QueryString="create database $database",
-        ResultConfiguration=["OutputLocation" => "s3://$bucket/queries/"],
-        ClientRequestToken=randstring(32))
+    Athena.start_query_execution("create database $database",
+        Dict("ResultConfiguration"=>Dict("OutputLocation" => "s3://$bucket/queries/"),
+        "ClientRequestToken"=>randstring(32)))
 
     # string to create table
     tablestr = """CREATE EXTERNAL TABLE IF NOT EXISTS $table ( `ms_filename` string,""" *
@@ -36,25 +35,27 @@ function athenasetup(aws::AWSConfig,bucket::String;
 
     # create table
     println("Creating table '$table' ",now())
-    AWSSDK.Athena.start_query_execution(aws,QueryString=tablestr,
-       ResultConfiguration=["OutputLocation" => "s3://$bucket/queries/"],
-       ClientRequestToken=randstring(32),
-       QueryExecutionContext = ["Database" => database])
+    Athena.start_query_execution(tablestr,
+       Dict("ResultConfiguration"=>Dict("OutputLocation" => "s3://$bucket/queries/"),
+       "ClientRequestToken"=>randstring(32),
+       "QueryExecutionContext" => Dict("Database" => database))
+    )
 
     sleep(1)
 
     # update partitions
     println("Repairing table '$table' ",now())
-    output = AWSSDK.Athena.start_query_execution(aws,QueryString="MSCK REPAIR TABLE $table",
-           ResultConfiguration=["OutputLocation" => "s3://$bucket/queries/"],
-           ClientRequestToken=randstring(32),
-           QueryExecutionContext = ["Database" => database])
+    output = Athena.start_query_execution("MSCK REPAIR TABLE $table",
+           Dict("ResultConfiguration"=>Dict("OutputLocation" => "s3://$bucket/queries/"),
+           "ClientRequestToken"=>randstring(32),
+           "QueryExecutionContext"=>Dict("Database" => database))
+    )
 
     # wait until done
     finished = false
     while !finished
-        queryresult = AWSSDK.Athena.get_query_execution(aws,output)
-        if queryresult["QueryExecutionDetail"]["Status"] == "SUCCEEDED"
+        queryresult = Athena.get_query_execution(output["QueryExecutionId"])
+        if queryresult["QueryExecutionDetail"]["Status"]["State"] == "SUCCEEDED"
             finished = true
             println("REPAIR table $table finished ",now())
         elseif queryresult["QueryExecutionDetail"]["Status"]["State"] == "FAILED"
@@ -65,9 +66,11 @@ function athenasetup(aws::AWSConfig,bucket::String;
     end
    return nothing
 end
+athenasetup(a...;b...) = athenasetup(global_aws_config(region="us-west-2"),a...;b...)
+athenasetup(d::Dict,a...;b...) = athenasetup(global_aws_config(region=d[:region]),a...;b...)
 
 """
-  athenaquery(aws,bucket,query)
+  athenaquery(bucket,query)
 
 Use AWS Athena to query SCEDC-pds database.
 
@@ -90,7 +93,6 @@ See https://docs.aws.amazon.com/athena/latest/ug/presto-functions.html for possi
 
 
 # Arguments
-- `aws::AWSConfig`: AWSConfig configuration dictionary
 - `bucket::String`: Name of AWS bucket to store query results
 - `query::String`: SQL query for SCEDC-pds database.
 
@@ -122,16 +124,17 @@ function athenaquery(aws::AWSConfig,bucket::String, query::String;
     end
 
     # query the table
-    output = AWSSDK.Athena.start_query_execution(aws,
-        QueryString="select ms_filename from $table where $query order by ms_filename ",
-        ResultConfiguration=["OutputLocation" => "s3://$bucket/queries/"],
-        ClientRequestToken=randstring(32),
-        QueryExecutionContext = ["Database" => database])
+    output = Athena.start_query_execution(
+        "select ms_filename from $table where $query order by ms_filename ",
+        Dict("ResultConfiguration"=>Dict("OutputLocation" => "s3://$bucket/queries/"),
+        "ClientRequestToken"=>randstring(32),
+        "QueryExecutionContext" => Dict("Database" => database))
+    )
 
     # check when query is finished
     finished = false
     while !finished
-         queryresult = AWSSDK.Athena.get_query_execution(aws,output)
+         queryresult = Athena.get_query_execution(output["QueryExecutionId"])
          if queryresult["QueryExecutionDetail"]["Status"]["State"] == "SUCCEEDED"
              finished = true
              println("QUERY '$query' finished ",now())
@@ -160,3 +163,5 @@ function athenaquery(aws::AWSConfig,bucket::String, query::String;
     end
     return scedcpath.(filelist)
 end
+athenaquery(a...;b...) = athenaquery(global_aws_config(region="us-west-2"),a...;b...)
+athenaquery(d::Dict,a...;b...) = athenaquery(global_aws_config(region=d[:region]),a...;b...)

src/catalog.jl

@@ -55,19 +55,16 @@ function read_catalog(catalog)
 
     # format dates 
     dateformater = Dates.DateFormat("y/m/dTH:M:S.s")
-    df["EVENTTIME"] = DateTime.(parse_hms.(df[!,"YYYY/MM/DD"],df[!,"HH:mm:SS.ss"]),dateformater)
+    df[!,"EVENTTIME"] = DateTime.(parse_hms.(df[!,"YYYY/MM/DD"],df[!,"HH:mm:SS.ss"]),dateformater)
     select!(df,Not([Symbol("YYYY/MM/DD"),Symbol("HH:mm:SS.ss")]))
     return df
 end
 
 """
-    catalogquery(aws)
+    catalogquery()
 
 Use S3 to query SCEDC-pds event catalog. Returns a DataFrame of events.
 
-# Arguments
-- `aws::AWSConfig`: AWSConfig configuration dictionary
-
 # Keywords
 - `starttime::TimeType`: The start time/day of the event query.
 - `endtime::TimeType`: The end time/day of the event query.
@@ -82,10 +79,9 @@ Use S3 to query SCEDC-pds event catalog. Returns a DataFrame of events.
 - `eventtype::String`: Event type: {"eq"=>"earthquake", "qb"=> "quarry blast", 
     "sn"=>"sonic boom', "nt"=>"nuclear blast", "uk"=>"not reported","*"=>"all events"}
 """
-function catalogquery(
-    aws::AWSConfig;
-    starttime::TimeType=Date(1932,1,1),
-    endtime::TimeType=Today(),
+function catalogquery(aws::AWSConfig;
+    starttime::TimeType=today(),
+    endtime::TimeType=today(),
     minlatitude::Real=-90,
     maxlatitude::Real=90,
     minlongitude::Real=-180,
@@ -131,6 +127,8 @@ function catalogquery(
     end
     return alldf   
 end
+catalogquery(a...;b...) = catalogquery(global_aws_config(region="us-west-2"), a...; b...)
+catalogquery(d::Dict, a...;b...) = catalogquery(global_aws_config(region=d[:region]), a...; b...)
 
 function parse_hms(d::String,hms::String)
     # check that date matches HH:MM:SS.s syntax