Merge pull request #282 from 4dn-dcic/ecr

Ecr

awsf/aws_run_workflow_generic.sh

@@ -8,9 +8,10 @@ export ACCESS_KEY=
 export SECRET_KEY=
 export REGION=
 export SINGULARITY_OPTION=
+export TIBANNA_VERSION=
 
 printHelpAndExit() {
-    echo "Usage: ${0##*/} -i JOBID [-m SHUTDOWN_MIN] -j JSON_BUCKET_NAME -l LOGBUCKET [-u SCRIPTS_URL] [-p PASSWORD] [-a ACCESS_KEY] [-s SECRET_KEY] [-r REGION] [-g]"
+    echo "Usage: ${0##*/} -i JOBID [-m SHUTDOWN_MIN] -j JSON_BUCKET_NAME -l LOGBUCKET [-u SCRIPTS_URL] [-p PASSWORD] [-a ACCESS_KEY] [-s SECRET_KEY] [-r REGION] [-g] [-V VERSION]"
     echo "-i JOBID : awsem job id (required)"
     echo "-m SHUTDOWN_MIN : Possibly user can specify SHUTDOWN_MIN to hold it for a while for debugging. (default 'now')"
     echo "-j JSON_BUCKET_NAME : bucket for sending run.json file. This script gets run.json file from this bucket. e.g.: 4dn-aws-pipeline-run-json (required)"
@@ -22,9 +23,10 @@ printHelpAndExit() {
     echo "-s SECRET_KEY : secret key for certian s3 bucket access (if not set, use IAM permission only)"
     echo "-r REGION : region for the profile set for certain s3 bucket access (if not set, use IAM permission only)"
     echo "-g : use singularity"
+    echo "-V TIBANNA_VERSION : tibanna version (used in the run_task lambda that launched this instance)"
     exit "$1"
 }
-while getopts "i:m:j:l:L:u:p:a:s:r:g" opt; do
+while getopts "i:m:j:l:L:u:p:a:s:r:gV:" opt; do
     case $opt in
         i) export JOBID=$OPTARG;;
         m) export SHUTDOWN_MIN=$OPTARG;;  # Possibly user can specify SHUTDOWN_MIN to hold it for a while for debugging.
@@ -37,6 +39,7 @@ while getopts "i:m:j:l:L:u:p:a:s:r:g" opt; do
         s) export SECRET_KEY=$OPTARG;;  # secret key for certian s3 bucket access
         r) export REGION=$OPTARG;;  # region for the profile set for certian s3 bucket access
         g) export SINGULARITY_OPTION=--singularity;;  # use singularity
+        V) export TIBANNA_VERSION=$OPTARG;;  # version of tibanna used in the run_task lambda that launched this instance
         h) printHelpAndExit 0;;
         [?]) printHelpAndExit 1;;
         esac
@@ -59,7 +62,9 @@ export LOGFILE2=$LOCAL_OUTDIR/$JOBID.log
 export LOGJSONFILE=$LOCAL_OUTDIR/$JOBID.log.json
 export STATUS=0
 export ERRFILE=$LOCAL_OUTDIR/$JOBID.error  # if this is found on s3, that means something went wrong.
-export INSTANCE_ID=$(ec2-metadata -i|cut -d' ' -f2)
+export INSTANCE_ID=$(ec2metadata -i|cut -d' ' -f2)
+export INSTANCE_REGION=$(ec2metadata --availability-zone | sed 's/[a-z]$//')
+export AWS_ACCOUNT_ID=$(aws sts get-caller-identity| grep Account | sed 's/[^0-9]//g')
 
 if [[ $LANGUAGE == 'wdl' ]]
 then
@@ -176,6 +181,12 @@ exl chmod +x goofys-latest
 exl echo "user_allow_other" >> /etc/fuse.conf
 export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
+### log into ECR if necessary
+if [[ ! -z "$TIBANNA_VERSION" && "$TIBANNA_VERSION" > '0.18' ]]; then
+  pip install awscli -U;
+  exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com;
+fi
+
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE
 exl date 

requirements-test.txt

@@ -5,4 +5,5 @@ flake8==2.4.1
 pytest==3.0.5
 pytest-cov==2.3.1
 pytest-runner
+pytest-parallel
 mock

tasks.py

@@ -5,30 +5,35 @@
 
 @task
 def test(ctx, watch=False, last_failing=False, no_flake=False, k='',  extra='',
-         ignore='', ignore_pony=False, no_post=False):
+         ignore='', ignore_pony=False, no_post=False, deployment=False):
     """Run the tests.
     Note: --watch requires pytest-xdist to be installed.
     """
-    if not no_flake:
-        flake()
-    args = ['-rxs', ]
-    if k:
-        args.append('-k %s' % k)
-    args.append(extra)
-    if watch:
-        args.append('-f')
+    if deployment:
+        retcode = pytest.main(['--workers', '100', 'tests/tibanna/post_deployment'])
     else:
-        args.append('--cov-report')
-        args.append('xml')
-        args.append('--cov-report')
-        args.append('html')
-    if last_failing:
-        args.append('--lf')
-    if ignore:
+        if not no_flake:
+            flake()
+        args = ['-rxs', ]
+        if k:
+            args.append('-k %s' % k)
+        args.append(extra)
+        if watch:
+            args.append('-f')
+        else:
+            args.append('--cov-report')
+            args.append('xml')
+            args.append('--cov-report')
+            args.append('html')
+        if last_failing:
+            args.append('--lf')
+        if ignore:
+            args.append('--ignore')
+            args.append(ignore)
         args.append('--ignore')
-        args.append(ignore)
-    args.append('tests/tibanna/')
-    retcode = pytest.main(args)
+        args.append('tests/tibanna/post_deployment/')
+        args.append('tests/tibanna/')
+        retcode = pytest.main(args)
     if retcode != 0:
         print("test failed exiting")
         sys.exit(retcode)

test_json/unicorn/shelltest-ecr.json

@@ -0,0 +1,31 @@
+{
+  "args": {
+    "app_name": "shell-test",
+    "input_parameters": {},
+    "language": "shell",
+    "command": "\\$ECHO_COMMAND \"haha\" > /data1/out/shell-test-output; ls -l somefile >> /data1/out/shell-test-output",
+    "container_image": "643366669028.dkr.ecr.us-east-1.amazonaws.com/ubuntu16.04-python35-pip19:v1",
+    "output_target": {
+      "file:///data1/out/shell-test-output": "s3://tibanna-output/shell-test-output"
+    },
+    "secondary_output_target": {},
+    "secondary_files": {},
+    "output_S3_bucket": "soos-4dn-bucket",
+    "app_version": "5",
+    "input_files": {
+        "file:///data1/shell/somefile": "s3://soos-4dn-bucket/4DNFIITTJYNR.mcool.bins.juicerformat.gz"
+    },
+    "input_parameters": {
+    },
+    "input_env": {
+        "ECHO_COMMAND": "echo"
+    }
+  },
+  "config": {
+    "mem": 2,
+    "cpu": 1,
+    "ebs_size": 10,
+    "EBS_optimized": false,
+    "log_bucket": "tibanna-output"
+  }
+}

tests/tibanna/post_deployment/conftest.py

@@ -0,0 +1,60 @@
+import os
+from tibanna.core import API
+
+REGION1 = 'us-east-1'
+REGION2 = 'us-west-1'
+
+DEV_SUFFIX = 'pre'
+DEV_GROUP_SUFFIX = 'testgroup'
+DEV_SFN_REGION1 = 'tibanna_unicorn_' + DEV_GROUP_SUFFIX + '1' + '_' + DEV_SUFFIX  # deployed to us-east-1
+DEV_SFN_REGION2 = 'tibanna_unicorn_' + DEV_GROUP_SUFFIX + '1' + '_' + DEV_SUFFIX  # deployed to us-west-1
+DEV_SFN2_REGION2 = 'tibanna_unicorn_' + DEV_GROUP_SUFFIX + '2' + '_' + DEV_SUFFIX  # deployed to us-west-1
+
+
+def get_test_json(file_name):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    event_file_name = os.path.join(dir_path, 'test_json', file_name)
+    return event_file_name
+
+
+def deploy_sfn1_to_region1():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    API().deploy_unicorn(suffix=DEV_SUFFIX, buckets='tibanna-output,soos-4dn-bucket', usergroup=DEV_GROUP_SUFFIX + '1')
+
+
+def deploy_sfn1_to_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    API().deploy_unicorn(suffix=DEV_SUFFIX, buckets='tibanna-output,soos-4dn-bucket', usergroup=DEV_GROUP_SUFFIX + '1')
+
+
+def deploy_sfn2_to_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    buckets = 'tibanna-output,elasticbeanstalk-fourfront-webdev-files,4dn-tool-evaluation-files'
+    API().deploy_unicorn(suffix=DEV_SUFFIX, buckets=buckets, usergroup=DEV_GROUP_SUFFIX + '2')
+
+
+def cleanup_sfn1_region1():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    API().cleanup(user_group_name=DEV_GROUP_SUFFIX + '1', suffix=DEV_SUFFIX)
+
+
+def cleanup_sfn1_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    API().cleanup(user_group_name=DEV_GROUP_SUFFIX + '1', suffix=DEV_SUFFIX)
+
+
+def cleanup_sfn2_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    API().cleanup(user_group_name=DEV_GROUP_SUFFIX + '2', suffix=DEV_SUFFIX)
+
+
+def pytest_sessionstart(session):
+    deploy_sfn1_to_region1()
+    deploy_sfn1_to_region2()
+    deploy_sfn2_to_region2()
+
+
+def pytest_sessionfinish(session, exitstatus):
+    cleanup_sfn1_region1()
+    cleanup_sfn1_region2()
+    cleanup_sfn2_region2()

tests/tibanna/post_deployment/test_json/4dn_bwa.runonly.v1.json

@@ -0,0 +1,45 @@
+{
+  "_tibanna": {
+    "env": "fourfront-webdev",
+    "run_type": "bwa-mem"
+  },
+  "args": {
+    "app_name": "bwa-mem",
+    "input_parameters": {},
+    "cwl_child_filenames": [],
+    "output_target": {
+      "out_bam": "lalala/out.bam"
+    },
+    "secondary_output_target": {},
+    "cwl_main_filename": "bwa-mem.cwl",
+    "secondary_files": {},
+    "output_S3_bucket": "tibanna-output",
+    "app_version": "5",
+    "cwl_directory_url": "https://raw.githubusercontent.com/4dn-dcic/pipelines-cwl/master/cwl_awsem_v1/",
+    "cwl_version": "v1",
+    "input_files": {
+      "fastq1": {
+        "bucket_name": "4dn-tool-evaluation-files",
+        "object_key": "GM12878_SRR1658581_1pc_1_R1.fastq.h10000"
+      },
+      "fastq2": {
+        "bucket_name": "4dn-tool-evaluation-files",
+        "object_key": "GM12878_SRR1658581_1pc_1_R2.fastq.h10000"
+      },
+      "bwa_index": {
+        "bucket_name": "elasticbeanstalk-fourfront-webdev-files",
+        "object_key": "1f53df95-4cf3-41cc-971d-81bb16c486dd/4DNFIZQZ39L9.bwaIndex.tgz"
+      }
+    },
+    "input_parameters": {
+        "nThreads": 2
+    }
+  },
+  "config": {
+    "ebs_size": 30,
+    "root_ebs_size": 20,
+    "EBS_optimized": true,
+    "instance_type": "t3.large",
+    "log_bucket": "tibanna-output"
+  }
+}

tests/tibanna/post_deployment/test_json/shelltest-ecr.json

@@ -0,0 +1,31 @@
+{
+  "args": {
+    "app_name": "shell-test",
+    "input_parameters": {},
+    "language": "shell",
+    "command": "\\$ECHO_COMMAND \"haha\" > /data1/out/shell-test-output; ls -l somefile >> /data1/out/shell-test-output",
+    "container_image": "643366669028.dkr.ecr.us-east-1.amazonaws.com/ubuntu16.04-python35-pip19:v1",
+    "output_target": {
+      "file:///data1/out/shell-test-output": "s3://tibanna-output/shell-test-output"
+    },
+    "secondary_output_target": {},
+    "secondary_files": {},
+    "output_S3_bucket": "soos-4dn-bucket",
+    "app_version": "5",
+    "input_files": {
+        "file:///data1/shell/somefile": "s3://soos-4dn-bucket/4DNFIITTJYNR.mcool.bins.juicerformat.gz"
+    },
+    "input_parameters": {
+    },
+    "input_env": {
+        "ECHO_COMMAND": "echo"
+    }
+  },
+  "config": {
+    "mem": 2,
+    "cpu": 1,
+    "ebs_size": 10,
+    "EBS_optimized": false,
+    "log_bucket": "tibanna-output"
+  }
+}

tests/tibanna/post_deployment/test_json/shelltest4.json

@@ -0,0 +1,34 @@
+{
+  "args": {
+    "app_name": "shell-test",
+    "input_parameters": {},
+    "language": "shell",
+    "command": "\\$ECHO_COMMAND \"haha\" > /data1/out/shell-test-output; ls -l somefile >> /data1/out/shell-test-output",
+    "container_image": "ubuntu:16.04",
+    "output_target": {
+      "file:///data1/out/shell-test-output": "s3://tibanna-output/shell-test-output"
+    },
+    "secondary_output_target": {},
+    "secondary_files": {},
+    "output_S3_bucket": "soos-4dn-bucket",
+    "app_version": "5",
+    "input_files": {
+        "file:///data1/shell/somefile": "s3://soos-4dn-bucket/4DNFIITTJYNR.mcool.bins.juicerformat.gz"
+    },
+    "input_parameters": {
+    },
+    "input_env": {
+        "ECHO_COMMAND": "echo"
+    }
+  },
+  "config": {
+    "mem": 2,
+    "cpu": 1,
+    "ebs_size": 10,
+    "EBS_optimized": false,
+    "log_bucket": "tibanna-output",
+    "key_name": "4dn-encode",
+    "cloudwatch_dashboard": false,
+    "subnet": "subnet-efb1b3c4"
+  }
+}

tests/tibanna/post_deployment/test_unicorn.py

@@ -0,0 +1,51 @@
+import json
+import os
+import time
+from tibanna.core import API
+from tests.tibanna.post_deployment.conftest import (
+    REGION1,
+    REGION2,
+    DEV_SFN_REGION1,
+    DEV_SFN_REGION2,
+    DEV_SFN2_REGION2,
+    get_test_json,
+)
+
+
+def test_shell():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    res = API().run_workflow(input_json=get_test_json('shelltest4.json'), sfn=DEV_SFN_REGION1)
+    jobid = res['jobid']
+    time.sleep(300)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+
+
+def test_ecr():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    res = API().run_workflow(input_json=get_test_json('shelltest-ecr.json'), sfn=DEV_SFN_REGION1)
+    jobid = res['jobid']
+    time.sleep(300)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+
+
+def test_shell_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    res = API().run_workflow(input_json=get_test_json('shelltest4.json'), sfn=DEV_SFN_REGION2)
+    jobid = res['jobid']
+    time.sleep(300)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+
+
+def test_cw_metrics_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    res = API().run_workflow(input_json=get_test_json('4dn_bwa.runonly.v1.json'), sfn=DEV_SFN2_REGION2)
+    jobid = res['jobid']
+    time.sleep(60 * 20)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+    prj = json.loads(API().log(job_id=jobid, postrunjson=True))
+    assert prj['metrics']['max_mem_utilization_percent']
+    assert prj['metrics']['max_cpu_utilization_percent']

tibanna/_version.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.17.3"
+__version__ = "0.18.0"

tibanna/check_task.py

@@ -75,7 +75,7 @@ def run(self):
             eh = AWSEMErrorHandler()
             if 'custom_errors' in self.input_json['args']:
                 eh.add_custom_errors(self.input_json['args']['custom_errors'])
-            log = API().log(job_id=jobid)
+            log = API().log(job_id=jobid, logbucket=bucket_name)
             ex = eh.parse_log(log)
             if ex:
                 msg_aug = str(ex) + ". For more info - " + eh.general_awsem_check_log_msg(jobid)