From d93237991ab4c2047489ef2701e2ba0b6fd1fd56 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 21:46:25 +0000
Subject: [PATCH 01/26] ecr test

---
 test_json/unicorn/shelltest-ecr.json | 31 ++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 test_json/unicorn/shelltest-ecr.json

diff --git a/test_json/unicorn/shelltest-ecr.json b/test_json/unicorn/shelltest-ecr.json
new file mode 100644
index 000000000..31770a7b2
--- /dev/null
+++ b/test_json/unicorn/shelltest-ecr.json
@@ -0,0 +1,31 @@
+{
+  "args": {
+    "app_name": "shell-test",
+    "input_parameters": {},
+    "language": "shell",
+    "command": "\\$ECHO_COMMAND \"haha\" > /data1/out/shell-test-output; ls -l somefile >> /data1/out/shell-test-output",
+    "container_image": "643366669028.dkr.ecr.us-east-1.amazonaws.com/ubuntu16.04-python35-pip19:v1",
+    "output_target": {
+      "file:///data1/out/shell-test-output": "s3://tibanna-output/shell-test-output"
+    },
+    "secondary_output_target": {},
+    "secondary_files": {},
+    "output_S3_bucket": "soos-4dn-bucket",
+    "app_version": "5",
+    "input_files": {
+        "file:///data1/shell/somefile": "s3://soos-4dn-bucket/4DNFIITTJYNR.mcool.bins.juicerformat.gz"
+    },
+    "input_parameters": {
+    },
+    "input_env": {
+        "ECHO_COMMAND": "echo"
+    }
+  },
+  "config": {
+    "mem": 2,
+    "cpu": 1,
+    "ebs_size": 10,
+    "EBS_optimized": false,
+    "log_bucket": "tibanna-output"
+  }
+}

From b637c4bcb0547825e3dceaab341de34839df0644 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 21:52:21 +0000
Subject: [PATCH 02/26] test awsf for ecr login

---
 awsf/aws_run_workflow_generic.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 88533f7ac..66437260e 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -176,6 +176,11 @@ exl chmod +x goofys-latest
 exl echo "user_allow_other" >> /etc/fuse.conf
 export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
+### log into ECR if necessary
+export AWS_REGION='us-east-1'  # hardcoded for testing
+export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
+aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-1.amazonaws.com
+
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE
 exl date 

From 19251dc68e99b591b5710b6d7aeae31aeea8f7d2 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 21:57:18 +0000
Subject: [PATCH 03/26] minor fix for awsf test

---
 awsf/aws_run_workflow_generic.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 66437260e..d479e1d00 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -179,7 +179,7 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 ### log into ECR if necessary
 export AWS_REGION='us-east-1'  # hardcoded for testing
 export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
-aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-1.amazonaws.com
+aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE

From d61973398d9888e173bcd3c1f794429da2f6497f Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 22:09:44 +0000
Subject: [PATCH 04/26] minor

---
 awsf/aws_run_workflow_generic.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index d479e1d00..204408206 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -179,7 +179,8 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 ### log into ECR if necessary
 export AWS_REGION='us-east-1'  # hardcoded for testing
 export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
-aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
+exl echo "aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
+exl aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE

From 2d5bc206ec16bacfffad155db904822c855e6c7c Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 22:32:32 +0000
Subject: [PATCH 05/26] minor

---
 awsf/aws_run_workflow_generic.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 204408206..2a8db51e1 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -179,6 +179,7 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 ### log into ECR if necessary
 export AWS_REGION='us-east-1'  # hardcoded for testing
 export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
+pip install awscli -U
 exl echo "aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
 exl aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
 

From 0783a3e23da36c5c900a3eb3c69ed5b29c23528b Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 22:49:28 +0000
Subject: [PATCH 06/26] minor

---
 awsf/aws_run_workflow_generic.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 2a8db51e1..12c3fceb2 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -181,7 +181,7 @@ export AWS_REGION='us-east-1'  # hardcoded for testing
 export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
 pip install awscli -U
 exl echo "aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
-exl aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
+exl "aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE

From 0f73ee744159a49a6002fe3401dfcbe801d72e62 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Wed, 6 May 2020 22:55:55 +0000
Subject: [PATCH 07/26] minor

---
 awsf/aws_run_workflow_generic.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 12c3fceb2..8105c63c9 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -180,8 +180,8 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 export AWS_REGION='us-east-1'  # hardcoded for testing
 export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
 pip install awscli -U
-exl echo "aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
-exl "aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
+exl echo "docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com $(aws ecr get-login-password --region $AWS_REGION)"
+exl docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com $(aws ecr get-login-password --region $AWS_REGION)
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE

From e67e3cc0c2db7fbba08762ad8313d91d177ba568 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 17:02:54 +0000
Subject: [PATCH 08/26] minor

---
 awsf/aws_run_workflow_generic.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 8105c63c9..a2da0d332 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -180,8 +180,8 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 export AWS_REGION='us-east-1'  # hardcoded for testing
 export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
 pip install awscli -U
-exl echo "docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com $(aws ecr get-login-password --region $AWS_REGION)"
-exl docker login --username AWS --password-stdin $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com $(aws ecr get-login-password --region $AWS_REGION)
+exl echo "docker login --username AWS --password $(aws ecr get-login-password --region $AWS_REGION) $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
+exl docker login --username AWS --password $(aws ecr get-login-password --region $AWS_REGION) $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE

From e652d17efd45472a72ce80ee762aba1158b1631e Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 19:28:26 +0000
Subject: [PATCH 09/26] region/acc id autodeterined by instance

---
 awsf/aws_run_workflow_generic.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index a2da0d332..6d731bffd 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -60,6 +60,8 @@ export LOGJSONFILE=$LOCAL_OUTDIR/$JOBID.log.json
 export STATUS=0
 export ERRFILE=$LOCAL_OUTDIR/$JOBID.error  # if this is found on s3, that means something went wrong.
 export INSTANCE_ID=$(ec2-metadata -i|cut -d' ' -f2)
+export INSTANCE_REGION=$(ec2-metadata --availability-zone | sed 's/[a-z]$//')
+export AWS_ACCOUNT_ID=$(aws sts get-caller-identity|grep Account | sed 's/[^0-9]//g')
 
 if [[ $LANGUAGE == 'wdl' ]]
 then
@@ -177,11 +179,8 @@ exl echo "user_allow_other" >> /etc/fuse.conf
 export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
 ### log into ECR if necessary
-export AWS_REGION='us-east-1'  # hardcoded for testing
-export AWS_ACCOUNT_NUMBER='643366669028'  # hardcoded for testing
 pip install awscli -U
-exl echo "docker login --username AWS --password $(aws ecr get-login-password --region $AWS_REGION) $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com"
-exl docker login --username AWS --password $(aws ecr get-login-password --region $AWS_REGION) $AWS_ACCOUNT_NUMBER.dkr.ecr.$AWS_REGION.amazonaws.com
+exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE

From 428d45ac20aa0758af63a3af0a6e59f9619b17c0 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 19:49:36 +0000
Subject: [PATCH 10/26] more logging for debugging

---
 awsf/aws_run_workflow_generic.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 6d731bffd..44fa35356 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -61,7 +61,7 @@ export STATUS=0
 export ERRFILE=$LOCAL_OUTDIR/$JOBID.error  # if this is found on s3, that means something went wrong.
 export INSTANCE_ID=$(ec2-metadata -i|cut -d' ' -f2)
 export INSTANCE_REGION=$(ec2-metadata --availability-zone | sed 's/[a-z]$//')
-export AWS_ACCOUNT_ID=$(aws sts get-caller-identity|grep Account | sed 's/[^0-9]//g')
+export AWS_ACCOUNT_ID=$(aws sts get-caller-identity| grep Account | sed 's/[^0-9]//g')
 
 if [[ $LANGUAGE == 'wdl' ]]
 then
@@ -180,6 +180,7 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
 ### log into ECR if necessary
 pip install awscli -U
+exl echo "docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com"
 exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
 
 ### download data & reference files from s3

From e23fa65c3fcccfd0a40fd9715208512902d34bb0 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 20:00:41 +0000
Subject: [PATCH 11/26] minor fix

---
 awsf/aws_run_workflow_generic.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 44fa35356..57c93065a 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -59,8 +59,8 @@ export LOGFILE2=$LOCAL_OUTDIR/$JOBID.log
 export LOGJSONFILE=$LOCAL_OUTDIR/$JOBID.log.json
 export STATUS=0
 export ERRFILE=$LOCAL_OUTDIR/$JOBID.error  # if this is found on s3, that means something went wrong.
-export INSTANCE_ID=$(ec2-metadata -i|cut -d' ' -f2)
-export INSTANCE_REGION=$(ec2-metadata --availability-zone | sed 's/[a-z]$//')
+export INSTANCE_ID=$(ec2metadata -i|cut -d' ' -f2)
+export INSTANCE_REGION=$(ec2metadata --availability-zone | sed 's/[a-z]$//')
 export AWS_ACCOUNT_ID=$(aws sts get-caller-identity| grep Account | sed 's/[^0-9]//g')
 
 if [[ $LANGUAGE == 'wdl' ]]

From 52eed50ccc31da066b3b10f36a601329d903e2a1 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 20:39:54 +0000
Subject: [PATCH 12/26] iam role for ecr added

---
 tibanna/iam_utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tibanna/iam_utils.py b/tibanna/iam_utils.py
index 6686b444f..36f41d8d5 100644
--- a/tibanna/iam_utils.py
+++ b/tibanna/iam_utils.py
@@ -128,7 +128,8 @@ def policy_arn_list_for_role(self, role_type):
                                         'desc_stepfunction', 'cw_dashboard']
         check_task_custom_policy_types = ['cloudwatch_metric', 'cloudwatch', 'bucket', 'ec2_desc',
                                           'termination']
-        arnlist = {'ec2': [self.policy_arn(_) for _ in ['bucket', 'cloudwatch_metric']],
+        arnlist = {'ec2': [self.policy_arn(_) for _ in ['bucket', 'cloudwatch_metric']] +
+                          ['arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly'],
                    # 'stepfunction': [self.policy_arn(_) for _ in ['lambdainvoke']],
                    'stepfunction': ['arn:aws:iam::aws:policy/service-role/AWSLambdaRole'],
                    self.run_task_lambda_name: [self.policy_arn(_) for _ in run_task_custom_policy_types] +

From 0f235682b4645eedca0f3b3903ffc15d7626235b Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 20:49:15 +0000
Subject: [PATCH 13/26] version update

---
 tibanna/_version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tibanna/_version.py b/tibanna/_version.py
index 385ebea0f..65013b90a 100644
--- a/tibanna/_version.py
+++ b/tibanna/_version.py
@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.17.3"
+__version__ = "0.18.0"

From 2ff77e89f1a496ee50577b0ee5faeed4dfe77b8c Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Thu, 7 May 2020 20:52:42 +0000
Subject: [PATCH 14/26] disabled logging w/ password

---
 awsf/aws_run_workflow_generic.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 57c93065a..362c94b0c 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -180,7 +180,6 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
 ### log into ECR if necessary
 pip install awscli -U
-exl echo "docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com"
 exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
 
 ### download data & reference files from s3

From 026e5c95f078456b7edaa3a3fbde4e1512b63ef5 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 00:12:02 +0000
Subject: [PATCH 15/26] check task trying to list step functions fixed

---
 tibanna/check_task.py |  2 +-
 tibanna/core.py       | 73 +++++++++++++++++++++++--------------------
 2 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/tibanna/check_task.py b/tibanna/check_task.py
index 02f393713..8515a4103 100644
--- a/tibanna/check_task.py
+++ b/tibanna/check_task.py
@@ -75,7 +75,7 @@ def run(self):
             eh = AWSEMErrorHandler()
             if 'custom_errors' in self.input_json['args']:
                 eh.add_custom_errors(self.input_json['args']['custom_errors'])
-            log = API().log(job_id=jobid)
+            log = API().log(job_id=jobid, logbucket=bucket_name)
             ex = eh.parse_log(log)
             if ex:
                 msg_aug = str(ex) + ". For more info - " + eh.general_awsem_check_log_msg(jobid)
diff --git a/tibanna/core.py b/tibanna/core.py
index 91c22b874..43cb94134 100644
--- a/tibanna/core.py
+++ b/tibanna/core.py
@@ -426,7 +426,7 @@ def kill_all(self, sfn=None):
             else:
                 break
 
-    def log(self, exec_arn=None, job_id=None, exec_name=None, sfn=None, postrunjson=False, runjson=False, quiet=False):
+    def log(self, exec_arn=None, job_id=None, exec_name=None, sfn=None, postrunjson=False, runjson=False, logbucket=None, quiet=False):
         if postrunjson:
             suffix = '.postrun.json'
         elif runjson:
@@ -441,40 +441,45 @@ def log(self, exec_arn=None, job_id=None, exec_name=None, sfn=None, postrunjson=
         if exec_arn:
             desc = sf.describe_execution(executionArn=exec_arn)
             job_id = str(json.loads(desc['input'])['jobid'])
-            logbucket = str(json.loads(desc['input'])['config']['log_bucket'])
+            if not logbucket:
+                logbucket = str(json.loads(desc['input'])['config']['log_bucket'])
         elif job_id:
-            # first try dynanmodb to get logbucket
-            ddres = dict()
-            try:
-                dd = boto3.client('dynamodb')
-                ddres = dd.query(TableName=DYNAMODB_TABLE,
-                                 KeyConditions={'Job Id': {'AttributeValueList': [{'S': job_id}],
-                                                           'ComparisonOperator': 'EQ'}})
-            except Exception as e:
-                pass
-            if 'Items' in ddres:
-                logbucket = ddres['Items'][0]['Log Bucket']['S']
-            else:
-                # search through executions to get logbucket
-                stateMachineArn = STEP_FUNCTION_ARN(sfn)
-                res = sf.list_executions(stateMachineArn=stateMachineArn)
-                while True:
-                    if 'executions' not in res or not res['executions']:
-                        break
-                    breakwhile = False
-                    for exc in res['executions']:
-                        desc = sf.describe_execution(executionArn=exc['executionArn'])
-                        if job_id == str(json.loads(desc['input'])['jobid']):
-                            logbucket = str(json.loads(desc['input'])['config']['log_bucket'])
-                            breakwhile = True
-                            break
-                    if breakwhile:
-                        break
-                    if 'nextToken' in res:
-                        res = sf.list_executions(nextToken=res['nextToken'],
-                                                 stateMachineArn=stateMachineArn)
-                    else:
-                        break
+            if not logbucket:
+                # first try dynanmodb to get logbucket
+                ddres = dict()
+                try:
+                    dd = boto3.client('dynamodb')
+                    ddres = dd.query(TableName=DYNAMODB_TABLE,
+                                     KeyConditions={'Job Id': {'AttributeValueList': [{'S': job_id}],
+                                                               'ComparisonOperator': 'EQ'}})
+                except Exception as e:
+                    pass
+                if 'Items' in ddres:
+                    logbucket = ddres['Items'][0]['Log Bucket']['S']
+                else:
+                    # search through executions to get logbucket
+                    stateMachineArn = STEP_FUNCTION_ARN(sfn)
+                    try:
+                        res = sf.list_executions(stateMachineArn=stateMachineArn)
+                        while True:
+                            if 'executions' not in res or not res['executions']:
+                                break
+                            breakwhile = False
+                            for exc in res['executions']:
+                                desc = sf.describe_execution(executionArn=exc['executionArn'])
+                                if job_id == str(json.loads(desc['input'])['jobid']):
+                                    logbucket = str(json.loads(desc['input'])['config']['log_bucket'])
+                                    breakwhile = True
+                                    break
+                            if breakwhile:
+                                break
+                            if 'nextToken' in res:
+                                res = sf.list_executions(nextToken=res['nextToken'],
+                                                         stateMachineArn=stateMachineArn)
+                            else:
+                                break
+                    except:
+                        raise Exception("Cannot retrieve job. Try again later.")
         else:
             raise Exception("Either job_id, exec_arn or exec_name must be provided.")
         try:

From fcb1ebc6300fbf3a30945ba5b63427efa9c2d4a8 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 00:22:41 +0000
Subject: [PATCH 16/26] bucket access denied error added to known awsem errors

---
 tibanna/exceptions.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tibanna/exceptions.py b/tibanna/exceptions.py
index c6c45fa2e..975a41443 100644
--- a/tibanna/exceptions.py
+++ b/tibanna/exceptions.py
@@ -39,7 +39,10 @@ def _ErrorList(self):
             # not enough disk space
             self.AWSEMError('Not enough space', '.+No space left on device'),
             # CWL missing input error
-            self.AWSEMError('CWL missing input', 'Missing required input parameter\n.+\n', True)
+            self.AWSEMError('CWL missing input', 'Missing required input parameter\n.+\n', True),
+            # Bucket access error
+            self.AWSEMError('Bucket/file access denied', 'An error occurred (AccessDenied) when calling' + 
+                                                         ' the ListObjectsV2 operation: Access Denied')
         ]
 
     def parse_log(self, log):

From 824ca2f8d570404a91d09e219cfa583d30a74027 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 00:31:21 +0000
Subject: [PATCH 17/26] bucket access denied error added to known awsem errors

---
 tibanna/exceptions.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tibanna/exceptions.py b/tibanna/exceptions.py
index 975a41443..debbfbff0 100644
--- a/tibanna/exceptions.py
+++ b/tibanna/exceptions.py
@@ -41,8 +41,7 @@ def _ErrorList(self):
             # CWL missing input error
             self.AWSEMError('CWL missing input', 'Missing required input parameter\n.+\n', True),
             # Bucket access error
-            self.AWSEMError('Bucket/file access denied', 'An error occurred (AccessDenied) when calling' + 
-                                                         ' the ListObjectsV2 operation: Access Denied')
+            self.AWSEMError('Bucket/file access denied', 'when calling the ListObjectsV2 operation: Access Denied')
         ]
 
     def parse_log(self, log):

From 941fd65ff1dc19c74fc5957a4dac931f49f43e86 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 00:52:18 +0000
Subject: [PATCH 18/26] tibanna version dependency for ecr login

---
 awsf/aws_run_workflow_generic.sh | 11 ++++++++---
 tibanna/ec2_utils.py             |  2 ++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 362c94b0c..9c73ee261 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -8,9 +8,10 @@ export ACCESS_KEY=
 export SECRET_KEY=
 export REGION=
 export SINGULARITY_OPTION=
+export TIBANNA_VERSION=
 
 printHelpAndExit() {
-    echo "Usage: ${0##*/} -i JOBID [-m SHUTDOWN_MIN] -j JSON_BUCKET_NAME -l LOGBUCKET [-u SCRIPTS_URL] [-p PASSWORD] [-a ACCESS_KEY] [-s SECRET_KEY] [-r REGION] [-g]"
+    echo "Usage: ${0##*/} -i JOBID [-m SHUTDOWN_MIN] -j JSON_BUCKET_NAME -l LOGBUCKET [-u SCRIPTS_URL] [-p PASSWORD] [-a ACCESS_KEY] [-s SECRET_KEY] [-r REGION] [-g] [-V VERSION]"
     echo "-i JOBID : awsem job id (required)"
     echo "-m SHUTDOWN_MIN : Possibly user can specify SHUTDOWN_MIN to hold it for a while for debugging. (default 'now')"
     echo "-j JSON_BUCKET_NAME : bucket for sending run.json file. This script gets run.json file from this bucket. e.g.: 4dn-aws-pipeline-run-json (required)"
@@ -22,6 +23,7 @@ printHelpAndExit() {
     echo "-s SECRET_KEY : secret key for certian s3 bucket access (if not set, use IAM permission only)"
     echo "-r REGION : region for the profile set for certain s3 bucket access (if not set, use IAM permission only)"
     echo "-g : use singularity"
+    echo "-V : tibanna version"
     exit "$1"
 }
 while getopts "i:m:j:l:L:u:p:a:s:r:g" opt; do
@@ -37,6 +39,7 @@ while getopts "i:m:j:l:L:u:p:a:s:r:g" opt; do
         s) export SECRET_KEY=$OPTARG;;  # secret key for certian s3 bucket access
         r) export REGION=$OPTARG;;  # region for the profile set for certian s3 bucket access
         g) export SINGULARITY_OPTION=--singularity;;  # use singularity
+        V) export TIBANNA_VERSION=$OPTARG;;  # version of tibanna used in the run_task that launched this instance
         h) printHelpAndExit 0;;
         [?]) printHelpAndExit 1;;
         esac
@@ -179,8 +182,10 @@ exl echo "user_allow_other" >> /etc/fuse.conf
 export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
 ### log into ECR if necessary
-pip install awscli -U
-exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
+if [[ ! -z "$TIBANNA_VERSION" && "$TIBANNA_VERSION" > '0.18' ]]; then
+  pip install awscli -U
+  exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
+fi
 
 ### download data & reference files from s3
 exl cat $DOWNLOAD_COMMAND_FILE
diff --git a/tibanna/ec2_utils.py b/tibanna/ec2_utils.py
index ab3d86ad8..ec4ae131e 100644
--- a/tibanna/ec2_utils.py
+++ b/tibanna/ec2_utils.py
@@ -34,6 +34,7 @@
 )
 from .base import SerializableObject
 from .nnested_array import flatten, run_on_nested_arrays1
+from ._version import __version__
 from Benchmark import run as B
 from Benchmark.classes import get_instance_types, instance_list
 from Benchmark.byteformat import B2GB
@@ -657,6 +658,7 @@ def create_userdata(self, profile=None):
             str += " -a {access_key} -s {secret_key} -r {region}".format(region=AWS_REGION, **profile)
         if hasattr(cfg, 'singularity') and cfg.singularity:
             str += " -g"
+        str += " -V {version}".format(version=__version__)
         str += "\n"
         print(str)
         return(str)

From cf6e19d67757a10c6b71efc5e290a433a12de07c Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 01:06:15 +0000
Subject: [PATCH 19/26] minor

---
 awsf/aws_run_workflow_generic.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index 9c73ee261..ce83e755c 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -183,7 +183,7 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 
 ### log into ECR if necessary
 if [[ ! -z "$TIBANNA_VERSION" && "$TIBANNA_VERSION" > '0.18' ]]; then
-  pip install awscli -U
+  pip install awscli -U;
   exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
 fi
 

From e69faa084e27856e8ee10a9c1012a5fda9fd9e7a Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 01:42:07 +0000
Subject: [PATCH 20/26] fix

---
 awsf/aws_run_workflow_generic.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index ce83e755c..fb3875b1c 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -23,10 +23,10 @@ printHelpAndExit() {
     echo "-s SECRET_KEY : secret key for certian s3 bucket access (if not set, use IAM permission only)"
     echo "-r REGION : region for the profile set for certain s3 bucket access (if not set, use IAM permission only)"
     echo "-g : use singularity"
-    echo "-V : tibanna version"
+    echo "-V TIBANNA_VERSION : tibanna version (used in the run_task lambda that launched this instance)"
     exit "$1"
 }
-while getopts "i:m:j:l:L:u:p:a:s:r:g" opt; do
+while getopts "i:m:j:l:L:u:p:a:s:r:gV:" opt; do
     case $opt in
         i) export JOBID=$OPTARG;;
         m) export SHUTDOWN_MIN=$OPTARG;;  # Possibly user can specify SHUTDOWN_MIN to hold it for a while for debugging.
@@ -39,7 +39,7 @@ while getopts "i:m:j:l:L:u:p:a:s:r:g" opt; do
         s) export SECRET_KEY=$OPTARG;;  # secret key for certian s3 bucket access
         r) export REGION=$OPTARG;;  # region for the profile set for certian s3 bucket access
         g) export SINGULARITY_OPTION=--singularity;;  # use singularity
-        V) export TIBANNA_VERSION=$OPTARG;;  # version of tibanna used in the run_task that launched this instance
+        V) export TIBANNA_VERSION=$OPTARG;;  # version of tibanna used in the run_task lambda that launched this instance
         h) printHelpAndExit 0;;
         [?]) printHelpAndExit 1;;
         esac

From 00ad71d3bc7e0ae56cc3204b5d5ba04d7b5ee5b0 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 02:42:48 +0000
Subject: [PATCH 21/26] fix

---
 awsf/aws_run_workflow_generic.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awsf/aws_run_workflow_generic.sh b/awsf/aws_run_workflow_generic.sh
index fb3875b1c..646c586d4 100755
--- a/awsf/aws_run_workflow_generic.sh
+++ b/awsf/aws_run_workflow_generic.sh
@@ -184,7 +184,7 @@ export GOOFYS_COMMAND='./goofys-latest -o allow_other -o nonempty'
 ### log into ECR if necessary
 if [[ ! -z "$TIBANNA_VERSION" && "$TIBANNA_VERSION" > '0.18' ]]; then
   pip install awscli -U;
-  exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com
+  exl docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com;
 fi
 
 ### download data & reference files from s3

From 3e9d9b7fa71361df6a29a3af6268e7e4bb1bf761 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 18:49:09 +0000
Subject: [PATCH 22/26] post-deployment test

---
 requirements-test.txt                         |  1 +
 tasks.py                                      | 43 +++++++------
 tests/tibanna/post_deployment/__init__.py     |  0
 tests/tibanna/post_deployment/conftest.py     | 61 +++++++++++++++++++
 .../test_json/4dn_bwa.runonly.v1.json         | 45 ++++++++++++++
 .../test_json/shelltest-ecr.json              | 31 ++++++++++
 .../post_deployment/test_json/shelltest4.json | 34 +++++++++++
 tests/tibanna/post_deployment/test_unicorn.py | 51 ++++++++++++++++
 8 files changed, 246 insertions(+), 20 deletions(-)
 create mode 100644 tests/tibanna/post_deployment/__init__.py
 create mode 100644 tests/tibanna/post_deployment/conftest.py
 create mode 100644 tests/tibanna/post_deployment/test_json/4dn_bwa.runonly.v1.json
 create mode 100644 tests/tibanna/post_deployment/test_json/shelltest-ecr.json
 create mode 100644 tests/tibanna/post_deployment/test_json/shelltest4.json
 create mode 100644 tests/tibanna/post_deployment/test_unicorn.py

diff --git a/requirements-test.txt b/requirements-test.txt
index 97e7aa622..c7cf87b06 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -5,4 +5,5 @@ flake8==2.4.1
 pytest==3.0.5
 pytest-cov==2.3.1
 pytest-runner
+pytest-parallel
 mock
diff --git a/tasks.py b/tasks.py
index afb8969e8..80e6156c4 100644
--- a/tasks.py
+++ b/tasks.py
@@ -5,30 +5,33 @@
 
 @task
 def test(ctx, watch=False, last_failing=False, no_flake=False, k='',  extra='',
-         ignore='', ignore_pony=False, no_post=False):
+         ignore='', ignore_pony=False, no_post=False, deployment=False):
     """Run the tests.
     Note: --watch requires pytest-xdist to be installed.
     """
-    if not no_flake:
-        flake()
-    args = ['-rxs', ]
-    if k:
-        args.append('-k %s' % k)
-    args.append(extra)
-    if watch:
-        args.append('-f')
+    if deployment:
+        retcode = pytest.main(['--workers', '100', 'tests/tibanna/post_deployment'])
     else:
-        args.append('--cov-report')
-        args.append('xml')
-        args.append('--cov-report')
-        args.append('html')
-    if last_failing:
-        args.append('--lf')
-    if ignore:
-        args.append('--ignore')
-        args.append(ignore)
-    args.append('tests/tibanna/')
-    retcode = pytest.main(args)
+        if not no_flake:
+            flake()
+        args = ['-rxs', ]
+        if k:
+            args.append('-k %s' % k)
+        args.append(extra)
+        if watch:
+            args.append('-f')
+        else:
+            args.append('--cov-report')
+            args.append('xml')
+            args.append('--cov-report')
+            args.append('html')
+        if last_failing:
+            args.append('--lf')
+        if ignore:
+            args.append('--ignore')
+            args.append(ignore)
+        args.append('tests/tibanna/')
+        retcode = pytest.main(args)
     if retcode != 0:
         print("test failed exiting")
         sys.exit(retcode)
diff --git a/tests/tibanna/post_deployment/__init__.py b/tests/tibanna/post_deployment/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/tibanna/post_deployment/conftest.py b/tests/tibanna/post_deployment/conftest.py
new file mode 100644
index 000000000..471021319
--- /dev/null
+++ b/tests/tibanna/post_deployment/conftest.py
@@ -0,0 +1,61 @@
+import os
+from tibanna.core import API
+
+REGION1 = 'us-east-1'
+REGION2 = 'us-west-1'
+
+DEV_SUFFIX = 'pre'
+DEV_GROUP_SUFFIX = 'testgroup'
+DEV_SFN_REGION1 = 'tibanna_unicorn_' + DEV_GROUP_SUFFIX + '1' + '_' + DEV_SUFFIX  # deployed to us-east-1
+DEV_SFN_REGION2 = 'tibanna_unicorn_' + DEV_GROUP_SUFFIX + '1' + '_' + DEV_SUFFIX  # deployed to us-west-1
+DEV_SFN2_REGION2 = 'tibanna_unicorn_' + DEV_GROUP_SUFFIX + '2' + '_' + DEV_SUFFIX  # deployed to us-west-1
+
+
+def get_test_json(file_name):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    event_file_name = os.path.join(dir_path, 'test_json', file_name)
+    return event_file_name
+
+
+def deploy_sfn1_to_region1():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    API().deploy_unicorn(suffix=DEV_SUFFIX, buckets='tibanna-output,soos-4dn-bucket', usergroup=DEV_GROUP_SUFFIX + '1')
+    
+
+def deploy_sfn1_to_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    API().deploy_unicorn(suffix=DEV_SUFFIX, buckets='tibanna-output,soos-4dn-bucket', usergroup=DEV_GROUP_SUFFIX + '1')
+
+
+def deploy_sfn2_to_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    buckets = 'tibanna-output,elasticbeanstalk-fourfront-webdev-files,4dn-tool-evaluation-files'
+    API().deploy_unicorn(suffix=DEV_SUFFIX, buckets=buckets, usergroup=DEV_GROUP_SUFFIX + '2')
+
+
+def cleanup_sfn1_region1():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    API().cleanup(user_group_name=DEV_GROUP_SUFFIX + '1', suffix=DEV_SUFFIX)
+
+
+def cleanup_sfn1_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    API().cleanup(user_group_name=DEV_GROUP_SUFFIX + '1', suffix=DEV_SUFFIX)
+
+
+def cleanup_sfn2_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    API().cleanup(user_group_name=DEV_GROUP_SUFFIX + '2', suffix=DEV_SUFFIX)
+
+
+def pytest_sessionstart(session):
+    #deploy_sfn1_to_region1()
+    #deploy_sfn1_to_region2()
+    #deploy_sfn2_to_region2()
+    pass
+
+def pytest_sessionfinish(session, exitstatus):
+    #cleanup_sfn1_region1()
+    #cleanup_sfn1_region2()
+    #cleanup_sfn2_region2()
+    pass
diff --git a/tests/tibanna/post_deployment/test_json/4dn_bwa.runonly.v1.json b/tests/tibanna/post_deployment/test_json/4dn_bwa.runonly.v1.json
new file mode 100644
index 000000000..ad2cf2de8
--- /dev/null
+++ b/tests/tibanna/post_deployment/test_json/4dn_bwa.runonly.v1.json
@@ -0,0 +1,45 @@
+{
+  "_tibanna": {
+    "env": "fourfront-webdev",
+    "run_type": "bwa-mem"
+  },
+  "args": {
+    "app_name": "bwa-mem",
+    "input_parameters": {},
+    "cwl_child_filenames": [],
+    "output_target": {
+      "out_bam": "lalala/out.bam"
+    },
+    "secondary_output_target": {},
+    "cwl_main_filename": "bwa-mem.cwl",
+    "secondary_files": {},
+    "output_S3_bucket": "tibanna-output",
+    "app_version": "5",
+    "cwl_directory_url": "https://raw.githubusercontent.com/4dn-dcic/pipelines-cwl/master/cwl_awsem_v1/",
+    "cwl_version": "v1",
+    "input_files": {
+      "fastq1": {
+        "bucket_name": "4dn-tool-evaluation-files",
+        "object_key": "GM12878_SRR1658581_1pc_1_R1.fastq.h10000"
+      },
+      "fastq2": {
+        "bucket_name": "4dn-tool-evaluation-files",
+        "object_key": "GM12878_SRR1658581_1pc_1_R2.fastq.h10000"
+      },
+      "bwa_index": {
+        "bucket_name": "elasticbeanstalk-fourfront-webdev-files",
+        "object_key": "1f53df95-4cf3-41cc-971d-81bb16c486dd/4DNFIZQZ39L9.bwaIndex.tgz"
+      }
+    },
+    "input_parameters": {
+        "nThreads": 2
+    }
+  },
+  "config": {
+    "ebs_size": 30,
+    "root_ebs_size": 20,
+    "EBS_optimized": true,
+    "instance_type": "t3.large",
+    "log_bucket": "tibanna-output"
+  }
+}
diff --git a/tests/tibanna/post_deployment/test_json/shelltest-ecr.json b/tests/tibanna/post_deployment/test_json/shelltest-ecr.json
new file mode 100644
index 000000000..31770a7b2
--- /dev/null
+++ b/tests/tibanna/post_deployment/test_json/shelltest-ecr.json
@@ -0,0 +1,31 @@
+{
+  "args": {
+    "app_name": "shell-test",
+    "input_parameters": {},
+    "language": "shell",
+    "command": "\\$ECHO_COMMAND \"haha\" > /data1/out/shell-test-output; ls -l somefile >> /data1/out/shell-test-output",
+    "container_image": "643366669028.dkr.ecr.us-east-1.amazonaws.com/ubuntu16.04-python35-pip19:v1",
+    "output_target": {
+      "file:///data1/out/shell-test-output": "s3://tibanna-output/shell-test-output"
+    },
+    "secondary_output_target": {},
+    "secondary_files": {},
+    "output_S3_bucket": "soos-4dn-bucket",
+    "app_version": "5",
+    "input_files": {
+        "file:///data1/shell/somefile": "s3://soos-4dn-bucket/4DNFIITTJYNR.mcool.bins.juicerformat.gz"
+    },
+    "input_parameters": {
+    },
+    "input_env": {
+        "ECHO_COMMAND": "echo"
+    }
+  },
+  "config": {
+    "mem": 2,
+    "cpu": 1,
+    "ebs_size": 10,
+    "EBS_optimized": false,
+    "log_bucket": "tibanna-output"
+  }
+}
diff --git a/tests/tibanna/post_deployment/test_json/shelltest4.json b/tests/tibanna/post_deployment/test_json/shelltest4.json
new file mode 100644
index 000000000..50c1d955c
--- /dev/null
+++ b/tests/tibanna/post_deployment/test_json/shelltest4.json
@@ -0,0 +1,34 @@
+{
+  "args": {
+    "app_name": "shell-test",
+    "input_parameters": {},
+    "language": "shell",
+    "command": "\\$ECHO_COMMAND \"haha\" > /data1/out/shell-test-output; ls -l somefile >> /data1/out/shell-test-output",
+    "container_image": "ubuntu:16.04",
+    "output_target": {
+      "file:///data1/out/shell-test-output": "s3://tibanna-output/shell-test-output"
+    },
+    "secondary_output_target": {},
+    "secondary_files": {},
+    "output_S3_bucket": "soos-4dn-bucket",
+    "app_version": "5",
+    "input_files": {
+        "file:///data1/shell/somefile": "s3://soos-4dn-bucket/4DNFIITTJYNR.mcool.bins.juicerformat.gz"
+    },
+    "input_parameters": {
+    },
+    "input_env": {
+        "ECHO_COMMAND": "echo"
+    }
+  },
+  "config": {
+    "mem": 2,
+    "cpu": 1,
+    "ebs_size": 10,
+    "EBS_optimized": false,
+    "log_bucket": "tibanna-output",
+    "key_name": "4dn-encode",
+    "cloudwatch_dashboard": false,
+    "subnet": "subnet-efb1b3c4"
+  }
+}
diff --git a/tests/tibanna/post_deployment/test_unicorn.py b/tests/tibanna/post_deployment/test_unicorn.py
new file mode 100644
index 000000000..f5e0932ac
--- /dev/null
+++ b/tests/tibanna/post_deployment/test_unicorn.py
@@ -0,0 +1,51 @@
+import json
+import os
+import time
+from tibanna.core import API
+from tests.tibanna.post_deployment.conftest import (
+    REGION1,
+    REGION2,
+    DEV_SFN_REGION1,
+    DEV_SFN_REGION2,
+    DEV_SFN2_REGION2,
+    get_test_json,
+)
+
+
+def test_shell():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    res = API().run_workflow(input_json=get_test_json('shelltest4.json'), sfn=DEV_SFN_REGION1)
+    jobid = res['jobid']
+    time.sleep(300)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+
+
+def test_ecr():
+    os.environ['AWS_DEFAULT_REGION'] = REGION1
+    res = API().run_workflow(input_json=get_test_json('shelltest-ecr.json'), sfn=DEV_SFN_REGION1)
+    jobid = res['jobid']
+    time.sleep(300)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+
+
+def test_shell_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    res = API().run_workflow(input_json=get_test_json('shelltest4.json'), sfn=DEV_SFN_REGION2)
+    jobid = res['jobid']
+    time.sleep(300)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+
+
+def test_cw_metrics_region2():
+    os.environ['AWS_DEFAULT_REGION'] = REGION2
+    res = API().run_workflow(input_json=get_test_json('4dn_bwa.runonly.v1.json'), sfn=DEV_SFN2_REGION2)
+    jobid = res['jobid']
+    time.sleep(60 * 20)
+    status = API().check_status(job_id=jobid)
+    assert status == 'SUCCEEDED'
+    prj = json.loads(log(job_id=jobid, postrunjson=True))
+    assert prj['metrics']['max_mem_utilization_percent']
+    assert prj['metrics']['max_cpu_utilization_percent']

From 7ce82b70546b8f5b18b255f8c378437124307779 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 18:54:37 +0000
Subject: [PATCH 23/26] flake

---
 tests/tibanna/post_deployment/conftest.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/tests/tibanna/post_deployment/conftest.py b/tests/tibanna/post_deployment/conftest.py
index 471021319..5d95efb28 100644
--- a/tests/tibanna/post_deployment/conftest.py
+++ b/tests/tibanna/post_deployment/conftest.py
@@ -20,7 +20,7 @@ def get_test_json(file_name):
 def deploy_sfn1_to_region1():
     os.environ['AWS_DEFAULT_REGION'] = REGION1
     API().deploy_unicorn(suffix=DEV_SUFFIX, buckets='tibanna-output,soos-4dn-bucket', usergroup=DEV_GROUP_SUFFIX + '1')
-    
+
 
 def deploy_sfn1_to_region2():
     os.environ['AWS_DEFAULT_REGION'] = REGION2
@@ -49,13 +49,12 @@ def cleanup_sfn2_region2():
 
 
 def pytest_sessionstart(session):
-    #deploy_sfn1_to_region1()
-    #deploy_sfn1_to_region2()
-    #deploy_sfn2_to_region2()
-    pass
+    deploy_sfn1_to_region1()
+    deploy_sfn1_to_region2()
+    deploy_sfn2_to_region2()
+
 
 def pytest_sessionfinish(session, exitstatus):
-    #cleanup_sfn1_region1()
-    #cleanup_sfn1_region2()
-    #cleanup_sfn2_region2()
-    pass
+    cleanup_sfn1_region1()
+    cleanup_sfn1_region2()
+    cleanup_sfn2_region2()

From b271447d5cb4918172fee3b57bc558b513db5d1d Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 18:56:17 +0000
Subject: [PATCH 24/26] flake

---
 tests/tibanna/post_deployment/test_unicorn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/tibanna/post_deployment/test_unicorn.py b/tests/tibanna/post_deployment/test_unicorn.py
index f5e0932ac..2db0ab131 100644
--- a/tests/tibanna/post_deployment/test_unicorn.py
+++ b/tests/tibanna/post_deployment/test_unicorn.py
@@ -46,6 +46,6 @@ def test_cw_metrics_region2():
     time.sleep(60 * 20)
     status = API().check_status(job_id=jobid)
     assert status == 'SUCCEEDED'
-    prj = json.loads(log(job_id=jobid, postrunjson=True))
+    prj = json.loads(API().log(job_id=jobid, postrunjson=True))
     assert prj['metrics']['max_mem_utilization_percent']
     assert prj['metrics']['max_cpu_utilization_percent']

From 933e5329477fdf5a8d70969f37dc0badb83f7382 Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 19:00:36 +0000
Subject: [PATCH 25/26] skip post-deployment test for non-post-deployment tests

---
 tasks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tasks.py b/tasks.py
index 80e6156c4..c2a7b0fab 100644
--- a/tasks.py
+++ b/tasks.py
@@ -30,6 +30,8 @@ def test(ctx, watch=False, last_failing=False, no_flake=False, k='',  extra='',
         if ignore:
             args.append('--ignore')
             args.append(ignore)
+        args.append('--ignore')
+        args.append('tests/post_deployment/')
         args.append('tests/tibanna/')
         retcode = pytest.main(args)
     if retcode != 0:

From c7bdaa6b71376b7d83d78d79585d9795a61eb70d Mon Sep 17 00:00:00 2001
From: SooLee <duplexa@gmail.com>
Date: Fri, 8 May 2020 19:02:01 +0000
Subject: [PATCH 26/26] skip post-deployment test for non-post-deployment tests

---
 tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tasks.py b/tasks.py
index c2a7b0fab..2f07104a3 100644
--- a/tasks.py
+++ b/tasks.py
@@ -31,7 +31,7 @@ def test(ctx, watch=False, last_failing=False, no_flake=False, k='',  extra='',
             args.append('--ignore')
             args.append(ignore)
         args.append('--ignore')
-        args.append('tests/post_deployment/')
+        args.append('tests/tibanna/post_deployment/')
         args.append('tests/tibanna/')
         retcode = pytest.main(args)
     if retcode != 0: