Skip to content

Commit 548a32e

Browse files
committed
various fixes
1 parent 4fe706c commit 548a32e

File tree

5 files changed

+50
-38
lines changed

5 files changed

+50
-38
lines changed

awsf3-docker/run.sh

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,9 @@ done
2727

2828
export RUN_JSON_FILE_NAME=$JOBID.run.json
2929
export POSTRUN_JSON_FILE_NAME=$JOBID.postrun.json
30-
export EBS_DIR=/data1 ## WARNING: also hardcoded in aws_decode_run_json.py
31-
export LOCAL_OUTDIR=$EBS_DIR/out
30+
export EBS_DIR=/data1
31+
export MOUNT_DIR_PREFIX=/mnt
32+
export LOCAL_OUTDIR=$EBS_DIR/out
3233
export LOCAL_INPUT_DIR=$EBS_DIR/input ## WARNING: also hardcoded in aws_decode_run_json.py
3334
export LOCAL_WF_TMPDIR=$EBS_DIR/tmp
3435
export MD5FILE=$JOBID.md5sum.txt
@@ -71,7 +72,7 @@ fi
7172

7273
# EBS_DIR cannot be directly mounted to docker container since it's already a mount point for EBS,
7374
# so mount /mnt/data1/ instead and create a symlink.
74-
ln -s /mnt/$EBS_DIR $EBS_DIR
75+
ln -s $MOUNT_DIR_PREFIX/$EBS_DIR $EBS_DIR
7576

7677
# Transferring profile info
7778
ln -s /home/ubuntu/.aws /root/.aws
@@ -152,12 +153,12 @@ send_log
152153
### download data & reference files from s3
153154
exl echo
154155
exl echo "## Downloading data & reference files from S3"
155-
exl date
156+
exl date
156157
exl mkdir -p $LOCAL_INPUT_DIR
157158
exl cat $DOWNLOAD_COMMAND_FILE
158-
exle source $DOWNLOAD_COMMAND_FILE
159+
exle source $DOWNLOAD_COMMAND_FILE
159160
exl date
160-
send_log
161+
send_log
161162

162163

163164
### mount input buckets
@@ -198,7 +199,7 @@ exl echo "## $(docker info | grep 'Total Memory')"
198199
exl echo
199200
send_log
200201
cwd0=$(pwd)
201-
cd $LOCAL_WFDIR
202+
cd $LOCAL_WFDIR
202203
mkdir -p $LOCAL_WF_TMPDIR
203204
if [[ $LANGUAGE == 'wdl_v1' || $LANGUAGE == 'wdl' ]]
204205
then
@@ -227,13 +228,15 @@ else
227228
exl echo "Error: CWL draft3 is no longer supported. Please switch to v1"
228229
handle_error 1
229230
fi
230-
exlj cwltool --enable-dev --non-strict --no-read-only --no-match-user --outdir $LOCAL_OUTDIR --tmp-outdir-prefix $LOCAL_WF_TMPDIR --tmpdir-prefix $LOCAL_WF_TMPDIR $PRESERVED_ENV_OPTION $SINGULARITY_OPTION $MAIN_CWL $cwd0/$INPUT_YML_FILE
231+
# cwltool cannot recognize symlinks and end up copying output from tmp directory intead of moving.
232+
# To prevent this, use the original directory name here.
233+
exlj cwltool --enable-dev --non-strict --no-read-only --no-match-user --outdir $MOUNT_DIR_PREFIX$LOCAL_OUTDIR --tmp-outdir-prefix $MOUNT_DIR_PREFIX$LOCAL_WF_TMPDIR --tmpdir-prefix $MOUNT_DIR_PREFIX$LOCAL_WF_TMPDIR $PRESERVED_ENV_OPTION $SINGULARITY_OPTION $MAIN_CWL $cwd0/$INPUT_YML_FILE
231234
handle_error $?
232235
fi
233236
cd $cwd0
234237
exl echo
235238
exl echo "Finished running the command/workflow"
236-
send_log
239+
send_log
237240

238241
### copy output files to s3
239242
exl echo
@@ -282,14 +285,14 @@ fi
282285
exl awsf3 update_postrun_json_upload_output -i $POSTRUN_JSON_FILE_NAME $LOGJSON_OPTION -m $LOCAL_OUTDIR/$MD5FILE -o $POSTRUN_JSON_FILE_NAME -L $LANGUAGE
283286
exl awsf3 upload_postrun_json -i $POSTRUN_JSON_FILE_NAME
284287
send_log
285-
288+
286289
### updating status
287290
exl echo
288291
exl echo "## Checking the job status (0 means success)"
289292
## if STATUS is 21,0,0,1 JOB_STATUS is 21,0,0,1. If STATUS is 0,0,0,0,0,0, JOB_STATUS is 0.
290293
if [ $(echo $STATUS| sed 's/0//g' | sed 's/,//g') ]; then export JOB_STATUS=$STATUS ; else export JOB_STATUS=0; fi
291294
exl echo "JOB_STATUS=$JOB_STATUS"
292-
# This env variable (JOB_STATUS) will be read by aws_update_run_json.py and the result will go into $POSTRUN_JSON_FILE_NAME.
295+
# This env variable (JOB_STATUS) will be read by aws_update_run_json.py and the result will go into $POSTRUN_JSON_FILE_NAME.
293296

294297
# update & upload postrun json
295298
exl echo

tibanna/__main__.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,6 @@ def args(self):
157157
{'flag': ["-d", "--shutdown-min"],
158158
'help': "use a specified shutdown mininutes for the rerun"},
159159
{'flag': ["-b", "--ebs-size"],
160-
'type': int,
161160
'help': "use a specified ebs size for the rerun (GB)"},
162161
{'flag': ["-T", "--ebs-type"],
163162
'help': "use a specified ebs type for the rerun (gp2 vs io1)"},
@@ -203,7 +202,6 @@ def args(self):
203202
{'flag': ["-d", "--shutdown-min"],
204203
'help': "use a specified shutdown mininutes for the rerun"},
205204
{'flag': ["-b", "--ebs-size"],
206-
'type': int,
207205
'help': "use a specified ebs size for the rerun (GB)"},
208206
{'flag': ["-T", "--ebs-type"],
209207
'help': "use a specified ebs type for the rerun (gp2 vs io1)"},

tibanna/_version.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
"""Version information."""
22

33
# The following line *must* be the last in the module, exactly as formatted:
4-
__version__ = "1.0.4"
4+
__version__ = "1.0.5"

tibanna/core.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,10 @@ def rerun(self, exec_arn=None, job_id=None, sfn=None,
677677
if shutdown_min:
678678
override_config['shutdown_min'] = shutdown_min
679679
if ebs_size:
680-
override_config['ebs_size'] = int(ebs_size)
680+
if ebs_size.endswith('x'):
681+
override_config['ebs_size'] = ebs_size
682+
else:
683+
override_config['ebs_size'] = int(ebs_size)
681684
if overwrite_input_extra:
682685
override_config['overwrite_input_extra'] = overwrite_input_extra
683686
if key_name:
@@ -1221,4 +1224,3 @@ def create_ami(self, build_from_scratch=True, source_image_to_copy_from=None, so
12211224
args.update({'base_region': source_image_region})
12221225

12231226
return AMI(**args).create_ami_for_tibanna(make_public=make_public)
1224-

tibanna/iam_utils.py

Lines changed: 31 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
AWS_ACCOUNT_NUMBER,
88
AWS_REGION,
99
LAMBDA_TYPE,
10+
SFN_TYPE,
1011
RUN_TASK_LAMBDA_NAME,
1112
CHECK_TASK_LAMBDA_NAME
1213
)
@@ -20,6 +21,7 @@ class IAM(object):
2021
account_id = AWS_ACCOUNT_NUMBER
2122
region = AWS_REGION
2223
lambda_type = LAMBDA_TYPE # lambda_type : '' for unicorn, 'pony' for pony, 'zebra' for zebra
24+
sfn_type = SFN_TYPE # sfn type : 'unicorn' for unicorn, 'pony' for pony, 'zebra' for zebra
2325
run_task_lambda_name = RUN_TASK_LAMBDA_NAME
2426
check_task_lambda_name = CHECK_TASK_LAMBDA_NAME
2527

@@ -48,6 +50,7 @@ def __init__(self, user_group_tag, bucket_names='', no_randomize=True):
4850
random_tag = str(int(random.random() * 10000))
4951
self.user_group_name = self.user_group_tag + '_' + random_tag
5052
self.tibanna_policy_prefix = self.prefix + self.user_group_name
53+
self.tibanna_sfn_name = self.prefix + self.sfn_type + '_' + self.user_group_name
5154

5255
# bucket names
5356
self.bucket_names = bucket_names
@@ -63,7 +66,8 @@ def iam_group_name(self):
6366
@property
6467
def policy_types(self):
6568
return ['bucket', 'termination', 'list', 'cloudwatch', 'passrole', 'lambdainvoke',
66-
'desc_stepfunction', 'cloudwatch_metric', 'cw_dashboard', 'dynamodb', 'ec2_desc']
69+
'cloudwatch_metric', 'cw_dashboard', 'dynamodb', 'ec2_desc',
70+
'executions']
6771

6872
def policy_arn(self, policy_type):
6973
return 'arn:aws:iam::' + self.account_id + ':policy/' + self.policy_name(policy_type)
@@ -75,11 +79,11 @@ def policy_suffix(self, policy_type):
7579
'cloudwatch': 'cloudwatchlogs',
7680
'passrole': 'iam_passrole_s3',
7781
'lambdainvoke': 'lambdainvoke',
78-
'desc_stepfunction': 'desc_sts',
7982
'cloudwatch_metric': 'cw_metric',
8083
'cw_dashboard': 'cw_dashboard',
8184
'dynamodb': 'dynamodb',
82-
'ec2_desc': 'ec2_desc'}
85+
'ec2_desc': 'ec2_desc',
86+
'executions': 'executions'}
8387
if policy_type not in suffices:
8488
raise Exception("policy %s must be one of %s." % (policy_type, str(self.policy_types)))
8589
return suffices[policy_type]
@@ -94,11 +98,11 @@ def policy_definition(self, policy_type):
9498
'cloudwatch': self.policy_cloudwatchlogs,
9599
'passrole': self.policy_iam_passrole_s3,
96100
'lambdainvoke': self.policy_lambdainvoke,
97-
'desc_stepfunction': self.policy_desc_stepfunction,
98101
'cloudwatch_metric': self.policy_cloudwatch_metric,
99102
'cw_dashboard': self.policy_cw_dashboard,
100103
'dynamodb': self.policy_dynamodb,
101-
'ec2_desc': self.policy_ec2_desc_policy}
104+
'ec2_desc': self.policy_ec2_desc_policy,
105+
'executions': self.policy_executions}
102106
if policy_type not in definitions:
103107
raise Exception("policy %s must be one of %s." % (policy_type, str(self.policy_types)))
104108
return definitions[policy_type]
@@ -128,7 +132,7 @@ def role_service(self, role_type):
128132

129133
def policy_arn_list_for_role(self, role_type):
130134
run_task_custom_policy_types = ['list', 'cloudwatch', 'passrole', 'bucket', 'dynamodb',
131-
'desc_stepfunction', 'cw_dashboard']
135+
'executions', 'cw_dashboard']
132136
check_task_custom_policy_types = ['cloudwatch_metric', 'cloudwatch', 'bucket', 'ec2_desc',
133137
'termination', 'dynamodb']
134138
arnlist = {'ec2': [self.policy_arn(_) for _ in ['bucket', 'cloudwatch_metric']] +
@@ -268,18 +272,32 @@ def policy_lambdainvoke(self):
268272
return policy
269273

270274
@property
271-
def policy_desc_stepfunction(self):
272-
execution_arn_prefix = 'arn:aws:states:' + self.region + ':' + self.account_id + ':execution:'
273-
resource = execution_arn_prefix + self.tibanna_policy_prefix + ':*'
275+
def policy_executions(self):
276+
execution_arn_prefix = 'arn:aws:states:' + self.region + ':' + self.account_id + ':stateMachine:'
277+
sfn_arn_prefix = 'arn:aws:states:' + self.region + ':' + self.account_id + ':execution:'
278+
resources = [execution_arn_prefix + self.tibanna_sfn_name,
279+
sfn_arn_prefix + self.tibanna_sfn_name + ':*']
274280
policy = {
275281
"Version": "2012-10-17",
276282
"Statement": [
277283
{
278284
"Effect": "Allow",
279285
"Action": [
280-
"states:DescribeExecution"
286+
"states:StartExecution",
287+
"states:StopExecution",
288+
"states:ListExecutions",
289+
"states:DescribeExecution",
290+
"states:GetExecutionHistory",
291+
"states:DescribeStateMachineForExecution",
292+
"states:DescribeStateMachine"
281293
],
282-
"Resource": resource
294+
"Resource": resources
295+
},
296+
{
297+
"Sid": "VisualEditor1",
298+
"Effect": "Allow",
299+
"Action": "states:ListStateMachines",
300+
"Resource": "*"
283301
}
284302
]
285303
}
@@ -421,16 +439,6 @@ def create_user_group(self, verbose=False):
421439
# deleting a group would require users to be detached from the group.
422440
self.detach_policies_from_group()
423441
group = self.iam.Group(self.iam_group_name)
424-
response = group.attach_policy(
425-
PolicyArn='arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess'
426-
)
427-
if verbose:
428-
logger.debug("response from IAM attach_policy :" + str(response))
429-
response = group.attach_policy(
430-
PolicyArn='arn:aws:iam::aws:policy/AWSStepFunctionsConsoleFullAccess'
431-
)
432-
if verbose:
433-
logger.debug("response from IAM attach_policy :" + str(response))
434442
response = group.attach_policy(
435443
PolicyArn='arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
436444
)
@@ -441,7 +449,8 @@ def create_user_group(self, verbose=False):
441449
)
442450
if verbose:
443451
logger.debug("response from IAM attach_policy :" + str(response))
444-
custom_policy_types = ['bucket', 'ec2_desc', 'cloudwatch_metric', 'dynamodb', 'termination']
452+
custom_policy_types = ['bucket', 'ec2_desc', 'cloudwatch_metric', 'dynamodb',
453+
'termination', 'executions']
445454
for pn in [self.policy_name(pt) for pt in custom_policy_types]:
446455
response = group.attach_policy(
447456
PolicyArn='arn:aws:iam::' + self.account_id + ':policy/' + pn

0 commit comments

Comments
 (0)