diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..0137a09 --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,37 @@ +name: Python package + +on: [push] +env: + POETRY_VERSION: 1.0 + +jobs: + testing: + runs-on: ubuntu-latest + strategy: + matrix: + PYTHON_VERSION: [3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{matrix.PYTHON_VERSION}} + uses: actions/setup-python@v1 + with: + python-version: ${{matrix.PYTHON_VERSION}} + - name: Install poetry + run: | + python -m pip install --upgrade pip + pip install poetry==${{env.POETRY_VERSION}} + - name: Install deps + run: | + poetry install + - name: Run pytest + run: | + poetry run python -m pytest --cov=pyfgaws --cov-branch + - name: Style checking + run: | + poetry run black --line-length 99 --check pyfgaws + - name: Run lint + run: | + poetry run flake8 --config=ci/flake8.cfg pyfgaws + - name: Run mypy + run: | + poetry run mypy -p pyfgaws --config=ci/mypy.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fdfce86 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# PyCharm +.idea + +# Python compiled & optimized files +*.pyc +*.pyo + +# MyPy Cache directory +.mypy_cache + +# for develop installs +*.egg-info + +# venv set up +.venv +dist/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..197e38b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2020 Fulcrum Genomics LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/ci/check.sh b/ci/check.sh new file mode 100755 index 0000000..91e0ebf --- /dev/null +++ b/ci/check.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +function banner() { + echo + echo "================================================================================" + echo $* + echo "================================================================================" + echo +} + +##################################################################### +# Takes two parameters, a "name" and a "command". +# Runs the command and prints out whether it succeeded or failed, and +# also tracks a list of failed steps in $failures. +##################################################################### +function run() { + local name=$1 + local cmd=$2 + + banner "Running $name [$cmd]" + set +e + $cmd + exit_code=$? + set -e + + if [[ $exit_code == 0 ]]; then + echo Passed $name: "[$cmd]" + else + echo Failed $name: "[$cmd]" + if [ -z "$failures" ]; then + failures="$failures $name" + else + failures="$failures, $name" + fi + fi +} + +parent=$(cd $(dirname $0) && pwd -P) + +banner "Executing in conda environment ${CONDA_DEFAULT_ENV} in directory ${root}" +run "Unit Tests" "python -m pytest -vv -r sx pyfgaws" +run "Style Checking" "black --line-length 99 --check pyfgaws" +run "Linting" "flake8 --config=$parent/flake8.cfg pyfgaws" +run "Type Checking" "mypy -p pyfgaws --config $parent/mypy.ini" + +if [ -z "$failures" ]; then + banner "Checks Passed" +else + banner "Checks Failed with failures in: $failures" + exit 1 +fi diff --git a/ci/flake8.cfg b/ci/flake8.cfg new file mode 100644 index 0000000..64f2e3f --- /dev/null +++ b/ci/flake8.cfg @@ -0,0 +1,6 @@ +# flake8 config file + +[flake8] +max_line_length = 100 +show-source = true +ignore = E701 W504 W503 diff --git a/ci/mypy.ini b/ci/mypy.ini new file mode 100644 index 0000000..6e117f5 --- /dev/null +++ b/ci/mypy.ini @@ -0,0 +1,6 @@ +[mypy] +strict_optional = False +ignore_missing_imports = True +disallow_untyped_decorators = False +follow_imports = "silent" +disallow_untyped_defs = True \ No newline at end of file diff --git a/conda-requirements.txt b/conda-requirements.txt new file mode 100644 index 0000000..acf9c18 --- /dev/null +++ b/conda-requirements.txt @@ -0,0 +1,3 @@ +# Setup python version and poetry +python=3.8.2 +poetry=1.0.5 diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..6cb3ef1 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,924 @@ +[[package]] +category = "dev" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +name = "appdirs" +optional = false +python-versions = "*" +version = "1.4.4" + +[[package]] +category = "dev" +description = "Atomic file writes." +marker = "sys_platform == \"win32\"" +name = "atomicwrites" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.4.0" + +[[package]] +category = "main" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + +[[package]] +category = "dev" +description = "The uncompromising code formatter." +name = "black" +optional = false +python-versions = ">=3.6" +version = "19.10b0" + +[package.dependencies] +appdirs = "*" +attrs = ">=18.1.0" +click = ">=6.5" +pathspec = ">=0.6,<1" +regex = "*" +toml = ">=0.9.4" +typed-ast = ">=1.4.0" + +[package.extras] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] + +[[package]] +category = "main" +description = "The AWS SDK for Python" +name = "boto3" +optional = false +python-versions = "*" +version = "1.13.20" + +[package.dependencies] +botocore = ">=1.16.20,<1.17.0" +jmespath = ">=0.7.1,<1.0.0" +s3transfer = ">=0.3.0,<0.4.0" + +[[package]] +category = "main" +description = "Type annotations for boto3 1.13.20, generated by mypy-boto3-buider 1.0.9" +name = "boto3-stubs" +optional = false +python-versions = ">=3.6" +version = "1.13.20.0" + +[package.dependencies] +mypy-boto3 = "1.13.20.0" + +[package.dependencies.mypy-boto3-batch] +optional = true +version = "1.13.20.0" + +[package.dependencies.mypy-boto3-logs] +optional = true +version = "1.13.20.0" + +[package.extras] +accessanalyzer = ["mypy-boto3-accessanalyzer (1.13.20.0)"] +acm = ["mypy-boto3-acm (1.13.20.0)"] +acm-pca = ["mypy-boto3-acm-pca (1.13.20.0)"] +alexaforbusiness = ["mypy-boto3-alexaforbusiness (1.13.20.0)"] +amplify = ["mypy-boto3-amplify (1.13.20.0)"] +apigateway = ["mypy-boto3-apigateway (1.13.20.0)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (1.13.20.0)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (1.13.20.0)"] +appconfig = ["mypy-boto3-appconfig (1.13.20.0)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (1.13.20.0)"] +application-insights = ["mypy-boto3-application-insights (1.13.20.0)"] +appmesh = ["mypy-boto3-appmesh (1.13.20.0)"] +appstream = ["mypy-boto3-appstream (1.13.20.0)"] +appsync = ["mypy-boto3-appsync (1.13.20.0)"] +athena = ["mypy-boto3-athena (1.13.20.0)"] +autoscaling = ["mypy-boto3-autoscaling (1.13.20.0)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (1.13.20.0)"] +backup = ["mypy-boto3-backup (1.13.20.0)"] +batch = ["mypy-boto3-batch (1.13.20.0)"] +budgets = ["mypy-boto3-budgets (1.13.20.0)"] +ce = ["mypy-boto3-ce (1.13.20.0)"] +chime = ["mypy-boto3-chime (1.13.20.0)"] +cloud9 = ["mypy-boto3-cloud9 (1.13.20.0)"] +clouddirectory = ["mypy-boto3-clouddirectory (1.13.20.0)"] +cloudformation = ["mypy-boto3-cloudformation (1.13.20.0)"] +cloudfront = ["mypy-boto3-cloudfront (1.13.20.0)"] +cloudhsm = ["mypy-boto3-cloudhsm (1.13.20.0)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (1.13.20.0)"] +cloudsearch = ["mypy-boto3-cloudsearch (1.13.20.0)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (1.13.20.0)"] +cloudtrail = ["mypy-boto3-cloudtrail (1.13.20.0)"] +cloudwatch = ["mypy-boto3-cloudwatch (1.13.20.0)"] +codebuild = ["mypy-boto3-codebuild (1.13.20.0)"] +codecommit = ["mypy-boto3-codecommit (1.13.20.0)"] +codedeploy = ["mypy-boto3-codedeploy (1.13.20.0)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (1.13.20.0)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (1.13.20.0)"] +codepipeline = ["mypy-boto3-codepipeline (1.13.20.0)"] +codestar = ["mypy-boto3-codestar (1.13.20.0)"] +codestar-connections = ["mypy-boto3-codestar-connections (1.13.20.0)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (1.13.20.0)"] +cognito-identity = ["mypy-boto3-cognito-identity (1.13.20.0)"] +cognito-idp = ["mypy-boto3-cognito-idp (1.13.20.0)"] +cognito-sync = ["mypy-boto3-cognito-sync (1.13.20.0)"] +comprehend = ["mypy-boto3-comprehend (1.13.20.0)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (1.13.20.0)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (1.13.20.0)"] +config = ["mypy-boto3-config (1.13.20.0)"] +connect = ["mypy-boto3-connect (1.13.20.0)"] +connectparticipant = ["mypy-boto3-connectparticipant (1.13.20.0)"] +cur = ["mypy-boto3-cur (1.13.20.0)"] +dataexchange = ["mypy-boto3-dataexchange (1.13.20.0)"] +datapipeline = ["mypy-boto3-datapipeline (1.13.20.0)"] +datasync = ["mypy-boto3-datasync (1.13.20.0)"] +dax = ["mypy-boto3-dax (1.13.20.0)"] +detective = ["mypy-boto3-detective (1.13.20.0)"] +devicefarm = ["mypy-boto3-devicefarm (1.13.20.0)"] +directconnect = ["mypy-boto3-directconnect (1.13.20.0)"] +discovery = ["mypy-boto3-discovery (1.13.20.0)"] +dlm = ["mypy-boto3-dlm (1.13.20.0)"] +dms = ["mypy-boto3-dms (1.13.20.0)"] +docdb = ["mypy-boto3-docdb (1.13.20.0)"] +ds = ["mypy-boto3-ds (1.13.20.0)"] +dynamodb = ["mypy-boto3-dynamodb (1.13.20.0)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (1.13.20.0)"] +ebs = ["mypy-boto3-ebs (1.13.20.0)"] +ec2 = ["mypy-boto3-ec2 (1.13.20.0)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (1.13.20.0)"] +ecr = ["mypy-boto3-ecr (1.13.20.0)"] +ecs = ["mypy-boto3-ecs (1.13.20.0)"] +efs = ["mypy-boto3-efs (1.13.20.0)"] +eks = ["mypy-boto3-eks (1.13.20.0)"] +elastic-inference = ["mypy-boto3-elastic-inference (1.13.20.0)"] +elasticache = ["mypy-boto3-elasticache (1.13.20.0)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (1.13.20.0)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (1.13.20.0)"] +elb = ["mypy-boto3-elb (1.13.20.0)"] +elbv2 = ["mypy-boto3-elbv2 (1.13.20.0)"] +emr = ["mypy-boto3-emr (1.13.20.0)"] +es = ["mypy-boto3-es (1.13.20.0)"] +essential = ["mypy-boto3-cloudformation (1.13.20.0)", "mypy-boto3-dynamodb (1.13.20.0)", "mypy-boto3-ec2 (1.13.20.0)", "mypy-boto3-lambda (1.13.20.0)", "mypy-boto3-rds (1.13.20.0)", "mypy-boto3-s3 (1.13.20.0)", "mypy-boto3-sqs (1.13.20.0)"] +events = ["mypy-boto3-events (1.13.20.0)"] +firehose = ["mypy-boto3-firehose (1.13.20.0)"] +fms = ["mypy-boto3-fms (1.13.20.0)"] +forecast = ["mypy-boto3-forecast (1.13.20.0)"] +forecastquery = ["mypy-boto3-forecastquery (1.13.20.0)"] +frauddetector = ["mypy-boto3-frauddetector (1.13.20.0)"] +fsx = ["mypy-boto3-fsx (1.13.20.0)"] +gamelift = ["mypy-boto3-gamelift (1.13.20.0)"] +glacier = ["mypy-boto3-glacier (1.13.20.0)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (1.13.20.0)"] +glue = ["mypy-boto3-glue (1.13.20.0)"] +greengrass = ["mypy-boto3-greengrass (1.13.20.0)"] +groundstation = ["mypy-boto3-groundstation (1.13.20.0)"] +guardduty = ["mypy-boto3-guardduty (1.13.20.0)"] +health = ["mypy-boto3-health (1.13.20.0)"] +iam = ["mypy-boto3-iam (1.13.20.0)"] +imagebuilder = ["mypy-boto3-imagebuilder (1.13.20.0)"] +importexport = ["mypy-boto3-importexport (1.13.20.0)"] +inspector = ["mypy-boto3-inspector (1.13.20.0)"] +iot = ["mypy-boto3-iot (1.13.20.0)"] +iot-data = ["mypy-boto3-iot-data (1.13.20.0)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (1.13.20.0)"] +iot1click-devices = ["mypy-boto3-iot1click-devices (1.13.20.0)"] +iot1click-projects = ["mypy-boto3-iot1click-projects (1.13.20.0)"] +iotanalytics = ["mypy-boto3-iotanalytics (1.13.20.0)"] +iotevents = ["mypy-boto3-iotevents (1.13.20.0)"] +iotevents-data = ["mypy-boto3-iotevents-data (1.13.20.0)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (1.13.20.0)"] +iotsitewise = ["mypy-boto3-iotsitewise (1.13.20.0)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (1.13.20.0)"] +kafka = ["mypy-boto3-kafka (1.13.20.0)"] +kendra = ["mypy-boto3-kendra (1.13.20.0)"] +kinesis = ["mypy-boto3-kinesis (1.13.20.0)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (1.13.20.0)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (1.13.20.0)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (1.13.20.0)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (1.13.20.0)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (1.13.20.0)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (1.13.20.0)"] +kms = ["mypy-boto3-kms (1.13.20.0)"] +lakeformation = ["mypy-boto3-lakeformation (1.13.20.0)"] +lambda = ["mypy-boto3-lambda (1.13.20.0)"] +lex-models = ["mypy-boto3-lex-models (1.13.20.0)"] +lex-runtime = ["mypy-boto3-lex-runtime (1.13.20.0)"] +license-manager = ["mypy-boto3-license-manager (1.13.20.0)"] +lightsail = ["mypy-boto3-lightsail (1.13.20.0)"] +logs = ["mypy-boto3-logs (1.13.20.0)"] +machinelearning = ["mypy-boto3-machinelearning (1.13.20.0)"] +macie = ["mypy-boto3-macie (1.13.20.0)"] +managedblockchain = ["mypy-boto3-managedblockchain (1.13.20.0)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (1.13.20.0)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (1.13.20.0)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (1.13.20.0)"] +mediaconnect = ["mypy-boto3-mediaconnect (1.13.20.0)"] +mediaconvert = ["mypy-boto3-mediaconvert (1.13.20.0)"] +medialive = ["mypy-boto3-medialive (1.13.20.0)"] +mediapackage = ["mypy-boto3-mediapackage (1.13.20.0)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (1.13.20.0)"] +mediastore = ["mypy-boto3-mediastore (1.13.20.0)"] +mediastore-data = ["mypy-boto3-mediastore-data (1.13.20.0)"] +mediatailor = ["mypy-boto3-mediatailor (1.13.20.0)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (1.13.20.0)"] +mgh = ["mypy-boto3-mgh (1.13.20.0)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (1.13.20.0)"] +mobile = ["mypy-boto3-mobile (1.13.20.0)"] +mq = ["mypy-boto3-mq (1.13.20.0)"] +mturk = ["mypy-boto3-mturk (1.13.20.0)"] +neptune = ["mypy-boto3-neptune (1.13.20.0)"] +networkmanager = ["mypy-boto3-networkmanager (1.13.20.0)"] +opsworks = ["mypy-boto3-opsworks (1.13.20.0)"] +opsworkscm = ["mypy-boto3-opsworkscm (1.13.20.0)"] +organizations = ["mypy-boto3-organizations (1.13.20.0)"] +outposts = ["mypy-boto3-outposts (1.13.20.0)"] +personalize = ["mypy-boto3-personalize (1.13.20.0)"] +personalize-events = ["mypy-boto3-personalize-events (1.13.20.0)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (1.13.20.0)"] +pi = ["mypy-boto3-pi (1.13.20.0)"] +pinpoint = ["mypy-boto3-pinpoint (1.13.20.0)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (1.13.20.0)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (1.13.20.0)"] +polly = ["mypy-boto3-polly (1.13.20.0)"] +pricing = ["mypy-boto3-pricing (1.13.20.0)"] +qldb = ["mypy-boto3-qldb (1.13.20.0)"] +qldb-session = ["mypy-boto3-qldb-session (1.13.20.0)"] +quicksight = ["mypy-boto3-quicksight (1.13.20.0)"] +ram = ["mypy-boto3-ram (1.13.20.0)"] +rds = ["mypy-boto3-rds (1.13.20.0)"] +rds-data = ["mypy-boto3-rds-data (1.13.20.0)"] +redshift = ["mypy-boto3-redshift (1.13.20.0)"] +rekognition = ["mypy-boto3-rekognition (1.13.20.0)"] +resource-groups = ["mypy-boto3-resource-groups (1.13.20.0)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (1.13.20.0)"] +robomaker = ["mypy-boto3-robomaker (1.13.20.0)"] +route53 = ["mypy-boto3-route53 (1.13.20.0)"] +route53domains = ["mypy-boto3-route53domains (1.13.20.0)"] +route53resolver = ["mypy-boto3-route53resolver (1.13.20.0)"] +s3 = ["mypy-boto3-s3 (1.13.20.0)"] +s3control = ["mypy-boto3-s3control (1.13.20.0)"] +sagemaker = ["mypy-boto3-sagemaker (1.13.20.0)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (1.13.20.0)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (1.13.20.0)"] +savingsplans = ["mypy-boto3-savingsplans (1.13.20.0)"] +schemas = ["mypy-boto3-schemas (1.13.20.0)"] +sdb = ["mypy-boto3-sdb (1.13.20.0)"] +secretsmanager = ["mypy-boto3-secretsmanager (1.13.20.0)"] +securityhub = ["mypy-boto3-securityhub (1.13.20.0)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (1.13.20.0)"] +service-quotas = ["mypy-boto3-service-quotas (1.13.20.0)"] +servicecatalog = ["mypy-boto3-servicecatalog (1.13.20.0)"] +servicediscovery = ["mypy-boto3-servicediscovery (1.13.20.0)"] +ses = ["mypy-boto3-ses (1.13.20.0)"] +sesv2 = ["mypy-boto3-sesv2 (1.13.20.0)"] +shield = ["mypy-boto3-shield (1.13.20.0)"] +signer = ["mypy-boto3-signer (1.13.20.0)"] +sms = ["mypy-boto3-sms (1.13.20.0)"] +sms-voice = ["mypy-boto3-sms-voice (1.13.20.0)"] +snowball = ["mypy-boto3-snowball (1.13.20.0)"] +sns = ["mypy-boto3-sns (1.13.20.0)"] +sqs = ["mypy-boto3-sqs (1.13.20.0)"] +ssm = ["mypy-boto3-ssm (1.13.20.0)"] +sso = ["mypy-boto3-sso (1.13.20.0)"] +sso-oidc = ["mypy-boto3-sso-oidc (1.13.20.0)"] +stepfunctions = ["mypy-boto3-stepfunctions (1.13.20.0)"] +storagegateway = ["mypy-boto3-storagegateway (1.13.20.0)"] +sts = ["mypy-boto3-sts (1.13.20.0)"] +support = ["mypy-boto3-support (1.13.20.0)"] +swf = ["mypy-boto3-swf (1.13.20.0)"] +synthetics = ["mypy-boto3-synthetics (1.13.20.0)"] +textract = ["mypy-boto3-textract (1.13.20.0)"] +transcribe = ["mypy-boto3-transcribe (1.13.20.0)"] +transfer = ["mypy-boto3-transfer (1.13.20.0)"] +translate = ["mypy-boto3-translate (1.13.20.0)"] +waf = ["mypy-boto3-waf (1.13.20.0)"] +waf-regional = ["mypy-boto3-waf-regional (1.13.20.0)"] +wafv2 = ["mypy-boto3-wafv2 (1.13.20.0)"] +workdocs = ["mypy-boto3-workdocs (1.13.20.0)"] +worklink = ["mypy-boto3-worklink (1.13.20.0)"] +workmail = ["mypy-boto3-workmail (1.13.20.0)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (1.13.20.0)"] +workspaces = ["mypy-boto3-workspaces (1.13.20.0)"] +xray = ["mypy-boto3-xray (1.13.20.0)"] + +[[package]] +category = "main" +description = "Low-level, data-driven core of boto 3." +name = "botocore" +optional = false +python-versions = "*" +version = "1.16.20" + +[package.dependencies] +docutils = ">=0.10,<0.16" +jmespath = ">=0.7.1,<1.0.0" +python-dateutil = ">=2.1,<3.0.0" + +[package.dependencies.urllib3] +python = "<3.4.0 || >=3.5.0" +version = ">=1.20,<1.26" + +[[package]] +category = "dev" +description = "Composable command line interface toolkit" +name = "click" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "7.1.2" + +[[package]] +category = "main" +description = "Cross-platform colored terminal text." +marker = "sys_platform == \"win32\"" +name = "colorama" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.4.3" + +[[package]] +category = "dev" +description = "Code coverage measurement for Python" +name = "coverage" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "5.1" + +[package.extras] +toml = ["toml"] + +[[package]] +category = "main" +description = "Effortless argument parser" +name = "defopt" +optional = false +python-versions = ">=3.5" +version = "6.0" + +[package.dependencies] +colorama = ">=0.3.4" +docutils = ">=0.10" +sphinxcontrib-napoleon = ">=0.7.0" + +[[package]] +category = "main" +description = "Docutils -- Python Documentation Utilities" +name = "docutils" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.15.2" + +[[package]] +category = "dev" +description = "the modular source code checker: pep8 pyflakes and co" +name = "flake8" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +version = "3.8.2" + +[package.dependencies] +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.6.0a1,<2.7.0" +pyflakes = ">=2.2.0,<2.3.0" + +[[package]] +category = "main" +description = "JSON Matching Expressions" +name = "jmespath" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "0.10.0" + +[[package]] +category = "dev" +description = "McCabe checker, plugin for flake8" +name = "mccabe" +optional = false +python-versions = "*" +version = "0.6.1" + +[[package]] +category = "dev" +description = "More routines for operating on iterables, beyond itertools" +name = "more-itertools" +optional = false +python-versions = ">=3.5" +version = "8.3.0" + +[[package]] +category = "dev" +description = "Optional static typing for Python" +name = "mypy" +optional = false +python-versions = ">=3.5" +version = "0.770" + +[package.dependencies] +mypy-extensions = ">=0.4.3,<0.5.0" +typed-ast = ">=1.4.0,<1.5.0" +typing-extensions = ">=3.7.4" + +[package.extras] +dmypy = ["psutil (>=4.0)"] + +[[package]] +category = "main" +description = "Type annotations for boto3 1.13.20 master module, generated by mypy-boto3-buider 1.0.9" +name = "mypy-boto3" +optional = false +python-versions = ">=3.6" +version = "1.13.20.0" + +[package.dependencies] +boto3 = "*" + +[[package]] +category = "main" +description = "Type annotations for boto3.Batch 1.13.20 service, generated by mypy-boto3-buider 1.0.9" +name = "mypy-boto3-batch" +optional = false +python-versions = ">=3.6" +version = "1.13.20.0" + +[[package]] +category = "main" +description = "Type annotations for boto3.CloudWatchLogs 1.13.20 service, generated by mypy-boto3-buider 1.0.9" +name = "mypy-boto3-logs" +optional = false +python-versions = ">=3.6" +version = "1.13.20.0" + +[[package]] +category = "dev" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +name = "mypy-extensions" +optional = false +python-versions = "*" +version = "0.4.3" + +[[package]] +category = "main" +description = "Generate Random Names for anything" +name = "namegenerator" +optional = false +python-versions = "*" +version = "1.0.6" + +[[package]] +category = "dev" +description = "Core utilities for Python packages" +name = "packaging" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.4" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +category = "dev" +description = "Utility library for gitignore style pattern matching of file paths." +name = "pathspec" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.8.0" + +[[package]] +category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.13.1" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +category = "main" +description = "A collection of helpful Python tools!" +name = "pockets" +optional = false +python-versions = "*" +version = "0.9.1" + +[package.dependencies] +six = ">=1.5.2" + +[[package]] +category = "dev" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.8.1" + +[[package]] +category = "dev" +description = "Python style guide checker" +name = "pycodestyle" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.6.0" + +[[package]] +category = "dev" +description = "passive checker of Python programs" +name = "pyflakes" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.2.0" + +[[package]] +category = "dev" +description = "Python parsing module" +name = "pyparsing" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.7" + +[[package]] +category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" +optional = false +python-versions = ">=3.5" +version = "5.4.2" + +[package.dependencies] +atomicwrites = ">=1.0" +attrs = ">=17.4.0" +colorama = "*" +more-itertools = ">=4.0.0" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.5.0" +wcwidth = "*" + +[package.extras] +checkqa-mypy = ["mypy (v0.761)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +category = "dev" +description = "Pytest plugin for measuring coverage." +name = "pytest-cov" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.9.0" + +[package.dependencies] +coverage = ">=4.4" +pytest = ">=3.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "pytest-xdist", "virtualenv"] + +[[package]] +category = "main" +description = "Extensions to the standard Python datetime module" +name = "python-dateutil" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +version = "2.8.1" + +[package.dependencies] +six = ">=1.5" + +[[package]] +category = "dev" +description = "Alternative regular expression module, to replace re." +name = "regex" +optional = false +python-versions = "*" +version = "2020.5.14" + +[[package]] +category = "main" +description = "An Amazon S3 Transfer Manager" +name = "s3transfer" +optional = false +python-versions = "*" +version = "0.3.3" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[[package]] +category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +version = "1.15.0" + +[[package]] +category = "main" +description = "Sphinx \"napoleon\" extension." +name = "sphinxcontrib-napoleon" +optional = false +python-versions = "*" +version = "0.7" + +[package.dependencies] +pockets = ">=0.3" +six = ">=1.5.2" + +[[package]] +category = "dev" +description = "Python Library for Tom's Obvious, Minimal Language" +name = "toml" +optional = false +python-versions = "*" +version = "0.10.1" + +[[package]] +category = "dev" +description = "a fork of Python 2 and 3 ast modules with type comment support" +name = "typed-ast" +optional = false +python-versions = "*" +version = "1.4.1" + +[[package]] +category = "dev" +description = "Backported and Experimental Type Hints for Python 3.5+" +name = "typing-extensions" +optional = false +python-versions = "*" +version = "3.7.4.2" + +[[package]] +category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +marker = "python_version != \"3.4\"" +name = "urllib3" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "1.25.9" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] + +[[package]] +category = "dev" +description = "Measures the displayed width of unicode strings in a terminal" +name = "wcwidth" +optional = false +python-versions = "*" +version = "0.2.2" + +[metadata] +content-hash = "fd261a1a0a6420670811e9c2d6899f3b6ca760c90fed41df26f5b9d6cb09b9c0" +python-versions = "^3.8" + +[metadata.files] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] +black = [ + {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, + {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, +] +boto3 = [ + {file = "boto3-1.13.20-py2.py3-none-any.whl", hash = "sha256:f59d0bd230ed3a4b932c5c4e497a0e0ff3c93b46b7e8cde54efb6fe10c8266ba"}, + {file = "boto3-1.13.20.tar.gz", hash = "sha256:26f8564b46d009b8f4c6470a6d6cde147b282a197339c7e31cbb0fe9fd9e5f5d"}, +] +boto3-stubs = [ + {file = "boto3-stubs-1.13.20.0.tar.gz", hash = "sha256:28f24401bff4480c6b9d715b3c1ec8d40758b577bc71bec6db8792f6ebf917bc"}, +] +botocore = [ + {file = "botocore-1.16.20-py2.py3-none-any.whl", hash = "sha256:990f3fc33dec746829740b1a9e1fe86183cdc96aedba6a632ccfcbae03e097cc"}, + {file = "botocore-1.16.20.tar.gz", hash = "sha256:d4cc47ac989a7f1d2992ef7679fb423a7966f687becf623a291a555a2d7ce1c0"}, +] +click = [ + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, +] +colorama = [ + {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, + {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, +] +coverage = [ + {file = "coverage-5.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65"}, + {file = "coverage-5.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2"}, + {file = "coverage-5.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04"}, + {file = "coverage-5.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6"}, + {file = "coverage-5.1-cp27-cp27m-win32.whl", hash = "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796"}, + {file = "coverage-5.1-cp27-cp27m-win_amd64.whl", hash = "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730"}, + {file = "coverage-5.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0"}, + {file = "coverage-5.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a"}, + {file = "coverage-5.1-cp35-cp35m-macosx_10_12_x86_64.whl", hash = "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf"}, + {file = "coverage-5.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9"}, + {file = "coverage-5.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768"}, + {file = "coverage-5.1-cp35-cp35m-win32.whl", hash = "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2"}, + {file = "coverage-5.1-cp35-cp35m-win_amd64.whl", hash = "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7"}, + {file = "coverage-5.1-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0"}, + {file = "coverage-5.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019"}, + {file = "coverage-5.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c"}, + {file = "coverage-5.1-cp36-cp36m-win32.whl", hash = "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1"}, + {file = "coverage-5.1-cp36-cp36m-win_amd64.whl", hash = "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7"}, + {file = "coverage-5.1-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355"}, + {file = "coverage-5.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489"}, + {file = "coverage-5.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd"}, + {file = "coverage-5.1-cp37-cp37m-win32.whl", hash = "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e"}, + {file = "coverage-5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a"}, + {file = "coverage-5.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55"}, + {file = "coverage-5.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c"}, + {file = "coverage-5.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef"}, + {file = "coverage-5.1-cp38-cp38-win32.whl", hash = "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24"}, + {file = "coverage-5.1-cp38-cp38-win_amd64.whl", hash = "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0"}, + {file = "coverage-5.1-cp39-cp39-win32.whl", hash = "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4"}, + {file = "coverage-5.1-cp39-cp39-win_amd64.whl", hash = "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e"}, + {file = "coverage-5.1.tar.gz", hash = "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052"}, +] +defopt = [ + {file = "defopt-6.0.tar.gz", hash = "sha256:a8fcd5560c845dceafd495d7dc97da355b1fdaa1881b5b83cf0fa4e9c7534285"}, +] +docutils = [ + {file = "docutils-0.15.2-py2-none-any.whl", hash = "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827"}, + {file = "docutils-0.15.2-py3-none-any.whl", hash = "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0"}, + {file = "docutils-0.15.2.tar.gz", hash = "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"}, +] +flake8 = [ + {file = "flake8-3.8.2-py2.py3-none-any.whl", hash = "sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5"}, + {file = "flake8-3.8.2.tar.gz", hash = "sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634"}, +] +jmespath = [ + {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, + {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +more-itertools = [ + {file = "more-itertools-8.3.0.tar.gz", hash = "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be"}, + {file = "more_itertools-8.3.0-py3-none-any.whl", hash = "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982"}, +] +mypy = [ + {file = "mypy-0.770-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:a34b577cdf6313bf24755f7a0e3f3c326d5c1f4fe7422d1d06498eb25ad0c600"}, + {file = "mypy-0.770-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:86c857510a9b7c3104cf4cde1568f4921762c8f9842e987bc03ed4f160925754"}, + {file = "mypy-0.770-cp35-cp35m-win_amd64.whl", hash = "sha256:a8ffcd53cb5dfc131850851cc09f1c44689c2812d0beb954d8138d4f5fc17f65"}, + {file = "mypy-0.770-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:7687f6455ec3ed7649d1ae574136835a4272b65b3ddcf01ab8704ac65616c5ce"}, + {file = "mypy-0.770-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3beff56b453b6ef94ecb2996bea101a08f1f8a9771d3cbf4988a61e4d9973761"}, + {file = "mypy-0.770-cp36-cp36m-win_amd64.whl", hash = "sha256:15b948e1302682e3682f11f50208b726a246ab4e6c1b39f9264a8796bb416aa2"}, + {file = "mypy-0.770-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:b90928f2d9eb2f33162405f32dde9f6dcead63a0971ca8a1b50eb4ca3e35ceb8"}, + {file = "mypy-0.770-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:c56ffe22faa2e51054c5f7a3bc70a370939c2ed4de308c690e7949230c995913"}, + {file = "mypy-0.770-cp37-cp37m-win_amd64.whl", hash = "sha256:8dfb69fbf9f3aeed18afffb15e319ca7f8da9642336348ddd6cab2713ddcf8f9"}, + {file = "mypy-0.770-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:219a3116ecd015f8dca7b5d2c366c973509dfb9a8fc97ef044a36e3da66144a1"}, + {file = "mypy-0.770-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7ec45a70d40ede1ec7ad7f95b3c94c9cf4c186a32f6bacb1795b60abd2f9ef27"}, + {file = "mypy-0.770-cp38-cp38-win_amd64.whl", hash = "sha256:f91c7ae919bbc3f96cd5e5b2e786b2b108343d1d7972ea130f7de27fdd547cf3"}, + {file = "mypy-0.770-py3-none-any.whl", hash = "sha256:3b1fc683fb204c6b4403a1ef23f0b1fac8e4477091585e0c8c54cbdf7d7bb164"}, + {file = "mypy-0.770.tar.gz", hash = "sha256:8a627507ef9b307b46a1fea9513d5c98680ba09591253082b4c48697ba05a4ae"}, +] +mypy-boto3 = [ + {file = "mypy-boto3-1.13.20.0.tar.gz", hash = "sha256:f3236b792af85132629e6e4d6ebb5bd30a58b69ade3671bcec8b48530f734332"}, +] +mypy-boto3-batch = [ + {file = "mypy-boto3-batch-1.13.20.0.tar.gz", hash = "sha256:ec77774d909019a0196b8612d822bc839dea749e024e4e288e0f25b93f83a550"}, +] +mypy-boto3-logs = [ + {file = "mypy-boto3-logs-1.13.20.0.tar.gz", hash = "sha256:fc90256fe42a10d082b76c896979ddaf0870c818e80b60486eb2a0acf7a3b780"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +namegenerator = [ + {file = "namegenerator-1.0.6.tar.gz", hash = "sha256:144759c62d771e5b589514a0aa332d0b967818c0a24b7824e48a905357c58bfa"}, +] +packaging = [ + {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, + {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, +] +pathspec = [ + {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, + {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +pockets = [ + {file = "pockets-0.9.1-py2.py3-none-any.whl", hash = "sha256:68597934193c08a08eb2bf6a1d85593f627c22f9b065cc727a4f03f669d96d86"}, + {file = "pockets-0.9.1.tar.gz", hash = "sha256:9320f1a3c6f7a9133fe3b571f283bcf3353cd70249025ae8d618e40e9f7e92b3"}, +] +py = [ + {file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"}, + {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, +] +pycodestyle = [ + {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, + {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, +] +pyflakes = [ + {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, + {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-5.4.2-py3-none-any.whl", hash = "sha256:95c710d0a72d91c13fae35dce195633c929c3792f54125919847fdcdf7caa0d3"}, + {file = "pytest-5.4.2.tar.gz", hash = "sha256:eb2b5e935f6a019317e455b6da83dd8650ac9ffd2ee73a7b657a30873d67a698"}, +] +pytest-cov = [ + {file = "pytest-cov-2.9.0.tar.gz", hash = "sha256:b6a814b8ed6247bd81ff47f038511b57fe1ce7f4cc25b9106f1a4b106f1d9322"}, + {file = "pytest_cov-2.9.0-py2.py3-none-any.whl", hash = "sha256:c87dfd8465d865655a8213859f1b4749b43448b5fae465cb981e16d52a811424"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, + {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, +] +regex = [ + {file = "regex-2020.5.14-cp27-cp27m-win32.whl", hash = "sha256:e565569fc28e3ba3e475ec344d87ed3cd8ba2d575335359749298a0899fe122e"}, + {file = "regex-2020.5.14-cp27-cp27m-win_amd64.whl", hash = "sha256:d466967ac8e45244b9dfe302bbe5e3337f8dc4dec8d7d10f5e950d83b140d33a"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:27ff7325b297fb6e5ebb70d10437592433601c423f5acf86e5bc1ee2919b9561"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ea55b80eb0d1c3f1d8d784264a6764f931e172480a2f1868f2536444c5f01e01"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:c9bce6e006fbe771a02bda468ec40ffccbf954803b470a0345ad39c603402577"}, + {file = "regex-2020.5.14-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:d881c2e657c51d89f02ae4c21d9adbef76b8325fe4d5cf0e9ad62f850f3a98fd"}, + {file = "regex-2020.5.14-cp36-cp36m-win32.whl", hash = "sha256:99568f00f7bf820c620f01721485cad230f3fb28f57d8fbf4a7967ec2e446994"}, + {file = "regex-2020.5.14-cp36-cp36m-win_amd64.whl", hash = "sha256:70c14743320a68c5dac7fc5a0f685be63bc2024b062fe2aaccc4acc3d01b14a1"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a7c37f048ec3920783abab99f8f4036561a174f1314302ccfa4e9ad31cb00eb4"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:89d76ce33d3266173f5be80bd4efcbd5196cafc34100fdab814f9b228dee0fa4"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:51f17abbe973c7673a61863516bdc9c0ef467407a940f39501e786a07406699c"}, + {file = "regex-2020.5.14-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:ce5cc53aa9fbbf6712e92c7cf268274eaff30f6bd12a0754e8133d85a8fb0f5f"}, + {file = "regex-2020.5.14-cp37-cp37m-win32.whl", hash = "sha256:8044d1c085d49673aadb3d7dc20ef5cb5b030c7a4fa253a593dda2eab3059929"}, + {file = "regex-2020.5.14-cp37-cp37m-win_amd64.whl", hash = "sha256:c2062c7d470751b648f1cacc3f54460aebfc261285f14bc6da49c6943bd48bdd"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux1_i686.whl", hash = "sha256:329ba35d711e3428db6b45a53b1b13a0a8ba07cbbcf10bbed291a7da45f106c3"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:579ea215c81d18da550b62ff97ee187b99f1b135fd894a13451e00986a080cad"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:3a9394197664e35566242686d84dfd264c07b20f93514e2e09d3c2b3ffdf78fe"}, + {file = "regex-2020.5.14-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ce367d21f33e23a84fb83a641b3834dd7dd8e9318ad8ff677fbfae5915a239f7"}, + {file = "regex-2020.5.14-cp38-cp38-win32.whl", hash = "sha256:1386e75c9d1574f6aa2e4eb5355374c8e55f9aac97e224a8a5a6abded0f9c927"}, + {file = "regex-2020.5.14-cp38-cp38-win_amd64.whl", hash = "sha256:7e61be8a2900897803c293247ef87366d5df86bf701083b6c43119c7c6c99108"}, + {file = "regex-2020.5.14.tar.gz", hash = "sha256:ce450ffbfec93821ab1fea94779a8440e10cf63819be6e176eb1973a6017aff5"}, +] +s3transfer = [ + {file = "s3transfer-0.3.3-py2.py3-none-any.whl", hash = "sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13"}, + {file = "s3transfer-0.3.3.tar.gz", hash = "sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db"}, +] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] +sphinxcontrib-napoleon = [ + {file = "sphinxcontrib-napoleon-0.7.tar.gz", hash = "sha256:407382beed396e9f2d7f3043fad6afda95719204a1e1a231ac865f40abcbfcf8"}, + {file = "sphinxcontrib_napoleon-0.7-py2.py3-none-any.whl", hash = "sha256:711e41a3974bdf110a484aec4c1a556799eb0b3f3b897521a018ad7e2db13fef"}, +] +toml = [ + {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, + {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, +] +typed-ast = [ + {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"}, + {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb"}, + {file = "typed_ast-1.4.1-cp35-cp35m-win32.whl", hash = "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919"}, + {file = "typed_ast-1.4.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01"}, + {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, + {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, + {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, + {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, + {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, + {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, + {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, + {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, + {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, + {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, + {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, +] +typing-extensions = [ + {file = "typing_extensions-3.7.4.2-py2-none-any.whl", hash = "sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392"}, + {file = "typing_extensions-3.7.4.2-py3-none-any.whl", hash = "sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5"}, + {file = "typing_extensions-3.7.4.2.tar.gz", hash = "sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae"}, +] +urllib3 = [ + {file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"}, + {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, +] +wcwidth = [ + {file = "wcwidth-0.2.2-py2.py3-none-any.whl", hash = "sha256:b651b6b081476420e4e9ae61239ac4c1b49d0c5ace42b2e81dc2ff49ed50c566"}, + {file = "wcwidth-0.2.2.tar.gz", hash = "sha256:3de2e41158cb650b91f9654cbf9a3e053cee0719c9df4ddc11e4b568669e9829"}, +] diff --git a/pyfgaws/__init__.py b/pyfgaws/__init__.py new file mode 100644 index 0000000..5c84ed5 --- /dev/null +++ b/pyfgaws/__init__.py @@ -0,0 +1,3 @@ +from pyfgaws.core import logging + +logging.setup_logging() diff --git a/pyfgaws/__main__.py b/pyfgaws/__main__.py new file mode 100644 index 0000000..826094b --- /dev/null +++ b/pyfgaws/__main__.py @@ -0,0 +1,56 @@ +"""Main entry point for all pyfgaws tools.""" + +import logging +import sys +from typing import Any +from typing import Callable +from typing import Dict +from typing import List + +import defopt +import json + +from pyfgaws.batch.tools import run_job +from pyfgaws.batch.tools import watch_job +from pyfgaws.batch import Status +from pyfgaws.logs.tools import watch_logs +from mypy_boto3.batch.type_defs import KeyValuePairTypeDef as BatchKeyValuePairTypeDef # noqa + + +# The list of tools to expose on the command line +TOOLS: List[Callable] = sorted([run_job, watch_job, watch_logs], key=lambda f: f.__name__) + + +def _parse_key_value_pair_type(string: str) -> BatchKeyValuePairTypeDef: + """Parses a simple dictionary that must contain two key-value pairs.""" + pair = json.loads(string) + assert isinstance(pair, dict), "argument is not a list" + assert "name" in pair, "name not found in key-value pair" + assert "value" in pair, "name not found in key-value pair" + assert len(pair.keys()) == 2, "Found extra keys in key-value pair: " + ", ".join(pair.keys()) + return pair # type: ignore + + +def _parse_batch_status(string: str) -> Status: + raise Exception + + +def _parsers() -> Dict[type, Callable[[str], Any]]: + """Returns the custom parsers for defopt""" + return { + Dict[str, Any]: lambda string: json.loads(string), + BatchKeyValuePairTypeDef: _parse_key_value_pair_type, + Status: _parse_batch_status, + } + + +def main(argv: List[str] = sys.argv[1:]) -> None: + logger = logging.getLogger(__name__) + if len(argv) != 0 and all(arg not in argv for arg in ["-h", "--help"]): + logger.info("Running command: fgaws-tools " + " ".join(argv)) + try: + defopt.run(funcs=TOOLS, argv=argv, parsers=_parsers()) + logger.info("Completed successfully.") + except Exception as e: + logger.info("Failed on command: " + " ".join(argv)) + raise e diff --git a/pyfgaws/batch/__init__.py b/pyfgaws/batch/__init__.py new file mode 100644 index 0000000..dce684a --- /dev/null +++ b/pyfgaws/batch/__init__.py @@ -0,0 +1,3 @@ +from pyfgaws.batch.api import BatchJob, Status + +__all__ = ("BatchJob", "Status") diff --git a/pyfgaws/batch/api.py b/pyfgaws/batch/api.py new file mode 100644 index 0000000..cde8965 --- /dev/null +++ b/pyfgaws/batch/api.py @@ -0,0 +1,355 @@ +""" +Utility methods for interacting with AWS Batch +---------------------------------------------- +""" +import copy +import enum +import logging +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + +import botocore +import namegenerator +from botocore.waiter import Waiter as BotoWaiter +from mypy_boto3 import batch +from mypy_boto3.batch.type_defs import ArrayPropertiesTypeDef # noqa +from mypy_boto3.batch.type_defs import ContainerDetailTypeDef # noqa +from mypy_boto3.batch.type_defs import ContainerOverridesTypeDef # noqa +from mypy_boto3.batch.type_defs import DescribeJobsResponseTypeDef # noqa +from mypy_boto3.batch.type_defs import JobDependencyTypeDef # noqa +from mypy_boto3.batch.type_defs import JobDetailTypeDef # noqa +from mypy_boto3.batch.type_defs import JobTimeoutTypeDef # noqa +from mypy_boto3.batch.type_defs import JobTimeoutTypeDef # noqa +from mypy_boto3.batch.type_defs import KeyValuePairTypeDef # noqa +from mypy_boto3.batch.type_defs import NodeOverridesTypeDef # noqa +from mypy_boto3.batch.type_defs import ResourceRequirementTypeDef # noqa +from mypy_boto3.batch.type_defs import RetryStrategyTypeDef # noqa +from mypy_boto3.batch.type_defs import SubmitJobResponseTypeDef # noqa + + +@enum.unique +class Status(enum.Enum): + Submitted = "SUBMITTED" + Pending = "PENDING" + Runnable = "RUNNABLE" + Starting = "STARTING" + Running = "RUNNING" + Succeeded = "SUCCEEDED" + Failed = "FAILED" + + +def get_latest_job_definition_arn(client: batch.Client, job_definition: str) -> str: + """Retrieves the latest job definition ARN. + + Args: + client: the AWS batch client + job_definition: the AWS batch job definition name + + Returns: + the latest job definition ARN. + """ + response = client.describe_job_definitions(jobDefinitionName=job_definition) + latest = max(response["jobDefinitions"], key=lambda d: d["revision"]) + return latest["jobDefinitionArn"] + + +class BatchJob: + """Stores information about a batch job. + + The following arguments override the provided container overrides: `cpus`, `mem_mb` ( + overrides `memory`), `command`, `instance_type`, `environment`, and `resourceRequirements`. + + Attributes: + client: the batch client to use + queue: the nae of the AWS batch queue + job_definition: the ARN for the AWS batch job definition, or the name of the job definition + to get the latest revision + name: the name of the job, otherwise one will be automatically generated + cpus: the number of CPUs to request + mem_mb: the amount of memory to request (in megabytes) + command: the command to use + instance_type: the instance type to use + environment: the environment variables to use + resource_requirements: a list of resource requirements for the job (type and amount) + array_properties: the array properties for this job + depends_on: the list of jobs to depend on + parameters: additional parameters passed to the job that replace parameter substitution + placeholders that are set in the job definition. + container_overrides: the container overrides that specify the name of a container in the + specified job definition and the overrides it should receive + node_overrides: list of node overrides that specify the node range to target and the + container overrides for that node range. + retry_strategy: the retry strategy to use for failed jobs from the `submit_job` operation. + timeout: the timeout configuration + logger: logger to write status messages + """ + + def __init__( + self, + client: batch.Client, + queue: str, + job_definition: str, + name: Optional[str] = None, + cpus: Optional[int] = None, + mem_mb: Optional[int] = None, + command: Optional[List[str]] = None, + instance_type: Optional[str] = None, + environment: Optional[List[KeyValuePairTypeDef]] = None, + resource_requirements: Optional[List[ResourceRequirementTypeDef]] = None, + array_properties: Optional[ArrayPropertiesTypeDef] = None, + depends_on: Optional[List[JobDependencyTypeDef]] = None, + parameters: Optional[Dict[str, str]] = None, + container_overrides: Optional[ContainerOverridesTypeDef] = None, + node_overrides: Optional[NodeOverridesTypeDef] = None, + retry_strategy: Optional[RetryStrategyTypeDef] = None, + timeout: Optional[JobTimeoutTypeDef] = None, + logger: Optional[logging.Logger] = None, + ) -> None: + + self.client: batch.Client = client + + # Get the latest job definition ARN if not given + if job_definition.startswith("arn:aws:batch:"): + if logger is not None: + logger.info(f"Using provided job definition '{job_definition}'") + self.job_definition_arn = job_definition + else: + if logger is not None: + logger.info(f"Retrieving the latest job definition for {job_definition}") + self.job_definition_arn = get_latest_job_definition_arn( + client=self.client, job_definition=job_definition + ) + if logger is not None: + logger.info(f"Retrieved latest job definition '{job_definition}'") + + # Main arguments + self.name: str = namegenerator.gen() if name is None else name + self.queue: str = queue + self.array_properties: ArrayPropertiesTypeDef = ( + {} if array_properties is None else array_properties + ) + self.depends_on: List[JobDependencyTypeDef] = ( + [] if depends_on is None else copy.deepcopy(depends_on) + ) + self.parameters = {} if parameters is None else copy.deepcopy(parameters) + self.container_overrides = ( + {} if container_overrides is None else copy.deepcopy(container_overrides) + ) + self.node_overrides: NodeOverridesTypeDef = {} if node_overrides is None else node_overrides + self.retry_strategy: RetryStrategyTypeDef = {} if retry_strategy is None else retry_strategy + self.timeout: JobTimeoutTypeDef = {} if timeout is None else timeout + + # Add to container overrides + if cpus is not None: + self.container_overrides["vcpus"] = cpus + if mem_mb is not None: + self.container_overrides["memory"] = mem_mb + if command is not None: + self.container_overrides["command"] = copy.deepcopy(command) + if instance_type is not None: + self.container_overrides["instanceType"] = instance_type + if environment is not None: + self.container_overrides["environment"] = copy.deepcopy(environment) + if resource_requirements is not None: + self.container_overrides["resourceRequirements"] = copy.deepcopy(resource_requirements) + + self.job_id: Optional[str] = None + + @classmethod + def from_id(cls, client: batch.Client, job_id: str) -> "BatchJob": + """"Builds a batch job from the given ID. + + Will lookup the job to retrieve job information. + + Args: + client: the AWS batch client + job_id: the job identifier + """ + jobs_response = client.describe_jobs(jobs=[job_id]) + jobs = jobs_response["jobs"] + assert len(jobs) <= 1, "More than one job described" + assert len(jobs) > 0, "No jobs found" + job_info = jobs[0] + + # Treat container overrides specially + container: ContainerDetailTypeDef = job_info.get("container", {}) + container_overrides: ContainerOverridesTypeDef = {} + if "vpus" in container: + container_overrides["vcpus"] = container["vcpus"] + if "memory" in container: + container_overrides["memory"] = container["memory"] + if "command" in container: + container_overrides["command"] = container["command"] + if "instanceType" in container: + container_overrides["instanceType"] = container["instanceType"] + if "environment" in container: + container_overrides["environment"] = container["environment"] + if "resourceRequirements" in container: + container_overrides["resourceRequirements"] = container["resourceRequirements"] + + job: BatchJob = BatchJob( + client=client, + name=job_info["jobName"], + queue=job_info["jobQueue"], + job_definition=job_info["jobDefinition"], + array_properties=job_info.get("arrayProperties", {}), + depends_on=job_info.get("dependsOn", []), + parameters=job_info.get("parameters", {}), + container_overrides=container_overrides, + retry_strategy=job_info.get("retryStrategy", {}), + timeout=job_info.get("timeout", {}), + ) + + job.job_id = job_id + + return job + + @property + def stream(self) -> Optional[str]: + """The log stream for the job, if available.""" + return self.describe_job()["container"].get("logStreamName") + + def submit(self) -> SubmitJobResponseTypeDef: + """Submits this job.""" + return self.client.submit_job( + jobName=self.name, + jobQueue=self.queue, + arrayProperties=self.array_properties, + dependsOn=self.depends_on, + jobDefinition=self.job_definition_arn, + parameters=self.parameters, + containerOverrides=self.container_overrides, + nodeOverrides=self.node_overrides, + retryStrategy=self.retry_strategy, + timeout=self.timeout, + ) + + def _reason(self, reason: Optional[str] = None) -> str: + """The default reason for cancelling or terminating a job""" + return reason if reason is not None else "manually initiated" + + def cancel_job(self, reason: Optional[str] = None) -> None: + """Cancels the given AWS Batch job. Does nothing if running or finished.""" + assert self.job_id is not None, "Cannot cancel a job that has not been submitted" + self.client.cancel_job(jobId=self.job_id, reason=self._reason(reason=reason)) + + def terminate_job(self, reason: Optional[str] = None) -> None: + """Terminates this job.""" + assert self.job_id is not None, "Cannot terminate a job that has not been submitted" + self.client.terminate_job(jobId=self.job_id, reason=self._reason(reason=reason)) + + def get_status(self) -> Optional[Status]: + """Gets the status of this job""" + if self.job_id is None: + return None + else: + return Status(self.describe_job()["status"]) + + def describe_job(self) -> JobDetailTypeDef: + """Gets detauled information about this job.""" + jobs_response = self.client.describe_jobs(jobs=[self.job_id]) + job_statuses = jobs_response["jobs"] + assert len(job_statuses) == 1 + job = job_statuses[0] + assert ( + job["jobName"] == self.name + ), f"""Job name mismatched: {self.name} != {job["jobName"]}""" + assert ( + job["jobId"] == self.job_id + ), f"""Job id mismatched: {self.job_id} != {job["jobId"]}""" + assert ( + job["jobQueue"] == self.queue + ), f"""Job queue mismatched: {self.queue} != {job["jobQueue"]}""" + return job_statuses[0] + + def wait_on( + self, + status_to_state: Dict[Status, bool], + max_attempts: Optional[int] = None, + delay: Optional[int] = None, + after_success: bool = False, + ) -> batch.type_defs.JobDetailTypeDef: + """Waits for the given states with associated success or failure. + + If some states are missing from the input mapping, then all statuses after the last + successful input status are treated as success or failure based on `after_success` + + Args: + status_to_state: mapping of status to success (true) or failure (false) state + max_attempts: the maximum # of attempts until reaching the given state. + delay: the delay before waiting + """ + assert len(status_to_state) > 0, "No statuses given" + assert any(value for value in status_to_state.values()), "No statuses with success set." + + _status_to_state = copy.deepcopy(status_to_state) + # get the last status in the given mapping + last_success_status = None + for status in Status: + if _status_to_state.get(status, False): + last_success_status = status + + # for all statuses after last_success_status, set to failure + set_to_failure = False + for status in Status: + if status == last_success_status: + set_to_failure = True + elif set_to_failure: + _status_to_state[status] = after_success + + name = "Waiter for statues: [" + ",".join(s.value for s in _status_to_state) + "]" + config: Dict[str, Any] = {"version": 2} + waiter_body: Dict[str, Any] = { + "delay": 1 if delay is None else delay, + "operation": "DescribeJobs", + "maxAttempts": sys.maxsize if max_attempts is None else max_attempts, + "acceptors": [ + { + "argument": "jobs[].status", + "expected": f"{status.value}", + "matcher": "pathAll", + "state": f"""{"success" if state else "failure"}""", + } + for status, state in _status_to_state.items() + ], + } + config["waiters"] = {name: waiter_body} + model: botocore.waiter.WaiterModel = botocore.waiter.WaiterModel(config) + waiter: BotoWaiter = botocore.waiter.create_waiter_with_client(name, model, self.client) + waiter.wait(jobs=[self.job_id]) + return self.describe_job() + + def wait_on_running( + self, max_attempts: Optional[int] = None, delay: Optional[int] = None + ) -> batch.type_defs.JobDetailTypeDef: + """Waits for the given states with associated success or failure. + + Args: + max_attempts: the maximum # of attempts until reaching the given state. + delay: the delay before waiting + """ + return self.wait_on( + status_to_state={Status.Running: True}, + max_attempts=max_attempts, + delay=delay, + after_success=True, + ) + + def wait_on_complete( + self, max_attempts: Optional[int] = None, delay: Optional[int] = None + ) -> batch.type_defs.JobDetailTypeDef: + """Waits for the given states with associated success or failure. + + Args: + max_attempts: the maximum # of attempts until reaching the given state. + delay: the delay before waiting + """ + return self.wait_on( + status_to_state={Status.Succeeded: True, Status.Failed: True}, + max_attempts=max_attempts, + delay=delay, + after_success=False, + ) diff --git a/pyfgaws/batch/tests/__init__.py b/pyfgaws/batch/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyfgaws/batch/tests/test_api.py b/pyfgaws/batch/tests/test_api.py new file mode 100644 index 0000000..7849416 --- /dev/null +++ b/pyfgaws/batch/tests/test_api.py @@ -0,0 +1,162 @@ +"""Tests for :module:`~pyfgaws.batch.api`""" + +from typing import List +from typing import Optional + +import botocore.session +import pytest +from botocore.stub import Stubber +from mypy_boto3.batch import Client +from mypy_boto3.batch.type_defs import DescribeJobDefinitionsResponseTypeDef # noqa +from mypy_boto3.batch.type_defs import DescribeJobsResponseTypeDef # noqa +from mypy_boto3.batch.type_defs import SubmitJobResponseTypeDef # noqa +from py._path.local import LocalPath as TmpDir + +from pyfgaws.batch import BatchJob +from pyfgaws.batch import Status +from pyfgaws.tests import stubbed_client + + +def stubbed_client_submit_job( + submit_job_response: SubmitJobResponseTypeDef, + describe_job_definitions_response: Optional[DescribeJobDefinitionsResponseTypeDef] = None, +) -> Client: + client = botocore.session.get_session().create_client("batch", region_name="us-east-1") + stubber = Stubber(client) + + if describe_job_definitions_response is not None: + stubber.add_response( + method="describe_job_definitions", service_response=describe_job_definitions_response + ) + + stubber.add_response(method="submit_job", service_response=submit_job_response) + + stubber.activate() + return client + + +def stubbed_client_describe_jobs(service_responses: List[DescribeJobsResponseTypeDef]) -> Client: + return stubbed_client( + service_name="batch", method="describe_jobs", service_responses=service_responses + ) + + +def valid_describe_job_definitions_response() -> DescribeJobDefinitionsResponseTypeDef: + return { + "jobDefinitions": [ + { + "type": "container", + "jobDefinitionName": "job-definition-name", + "jobDefinitionArn": "arn:aws:batch:some-arn-1", + "revision": 1, + }, + { + "type": "container", + "jobDefinitionName": "job-definition-name", + "jobDefinitionArn": "arn:aws:batch:some-arn-2", + "revision": 2, + }, + { + "type": "container", + "jobDefinitionName": "job-definition-name", + "jobDefinitionArn": "arn:aws:batch:some-arn-3", + "revision": 3, + }, + ], + "nextToken": "next-token", + } + + +def valid_submit_job_response() -> SubmitJobResponseTypeDef: + return {"jobName": "job-name", "jobId": "job-id"} + + +def test_submit_job(tmpdir: TmpDir) -> None: + for job_definition in ["job-definition-name", "arn:aws:batch:some-arn"]: + client: Client + submit_job_response: SubmitJobResponseTypeDef = valid_submit_job_response() + describe_job_definitions_response: Optional[DescribeJobDefinitionsResponseTypeDef] = None + + if not job_definition.startswith("arn:aws:batch:"): + describe_job_definitions_response = valid_describe_job_definitions_response() + + client = stubbed_client_submit_job( + submit_job_response=submit_job_response, + describe_job_definitions_response=describe_job_definitions_response, + ) + + job = BatchJob( + client=client, name="job-name", queue="job-queue", job_definition=job_definition + ) + + response = job.submit() + + assert response == submit_job_response + + +def build_describe_jobs_response(status: Status) -> DescribeJobsResponseTypeDef: + return { + "jobs": [ + { + "jobName": "job-name", + "jobId": "job-id", + "jobQueue": "job-queue", + "jobDefinition": "arn:aws:batch:some-arn", + "startedAt": 1, + "status": status.value, + } + ] + } + + +def build_describe_jobs_responses(*status: Status) -> List[DescribeJobsResponseTypeDef]: + assert all(s in Status for s in status) + return [build_describe_jobs_response(status=s) for s in status] + + +@pytest.mark.parametrize( + "statuses", + [ + [Status.Succeeded], + # [Status.Failed], + # [ + # Status.Submitted, + # Status.Pending, + # Status.Runnable, + # Status.Runnable, + # Status.Running, + # Status.Succeeded, + # ], + # [ + # Status.Submitted, + # Status.Pending, + # Status.Runnable, + # Status.Runnable, + # Status.Running, + # Status.Failed, + # ], + ], +) +def test_wait_for_job(statuses: List[Status]) -> None: + service_responses: List[DescribeJobsResponseTypeDef] = [] + # add a response for the `describe_jobs` in `BatchJob.from_id` + service_responses.append(build_describe_jobs_response(status=Status.Submitted)) + + # add the expected responses + service_responses.extend(build_describe_jobs_responses(*statuses)) + + # add a three more terminal responses, since we have a waiter that's job-exists and then + # job-running, before job-complete, with a final `describe_jobs` after it completes + last_response = service_responses[-1] + service_responses.append(last_response) + service_responses.append(last_response) + service_responses.append(last_response) + + assert len(service_responses) > 1 + client: Client = stubbed_client_describe_jobs(service_responses=service_responses) + + job: BatchJob = BatchJob.from_id(client=client, job_id="job-id") + assert job.job_id is not None, str(job) + response = job.wait_on_complete(delay=0) + + assert response == service_responses[-1]["jobs"][0], str(response) diff --git a/pyfgaws/batch/tools.py b/pyfgaws/batch/tools.py new file mode 100644 index 0000000..e143755 --- /dev/null +++ b/pyfgaws/batch/tools.py @@ -0,0 +1,174 @@ +""" +Command-line tools for interacting with AWS Batch +------------------------------------------------- +""" + +import logging +import threading +import time +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + +import boto3 +from mypy_boto3 import batch +from mypy_boto3 import logs +from mypy_boto3.batch.type_defs import DescribeJobsResponseTypeDef # noqa +from mypy_boto3.batch.type_defs import KeyValuePairTypeDef # noqa +from mypy_boto3.batch.type_defs import SubmitJobResponseTypeDef # noqa + +from pyfgaws.batch import BatchJob +from pyfgaws.batch import Status +from pyfgaws.logs import DEFAULT_POLLING_INTERVAL as DEFAULT_LOGS_POLLING_INTERVAL +from pyfgaws.logs import Log + + +def _log_it(region_name: str, job: BatchJob, logger: logging.Logger) -> None: + """Creates a background thread to print out CloudWatch logs. + + Args: + region_name: the AWS region + job: the AWS batch job + logger: the logger to which logs should be printed + """ + # Create a background thread + logs_thread = threading.Thread( + target=_watch_logs, args=(region_name, job, logger), daemon=True + ) + logs_thread.start() + + +def watch_job(*, job_id: str, region_name: Optional[str] = None, print_logs: bool = True,) -> None: + """Watches an AWS batch job. + + Args: + job_id: the AWS batch job identifier + region_name: the AWS region + print_logs: true to print CloudWatch logs, false otherwise + """ + logger = logging.getLogger(__name__) + + client: batch.Client = boto3.client( + service_name="batch", region_name=region_name # type: ignore + ) + + # Create the job + job: BatchJob = BatchJob.from_id(client=client, job_id=job_id) + + logger.info(f"Watching job with name '{job.name}' and id '{job.job_id}'") + if print_logs: + _log_it(region_name=region_name, job=job, logger=logger) + + job.wait_on_complete() + logger.info( + f"Job completed with name '{job.name}', id '{job.job_id}', and status '{job.get_status()}'" + ) + + +def run_job( + *, + job_definition: str, + name: Optional[str] = None, + region_name: Optional[str] = None, + print_logs: bool = True, + queue: Optional[str] = None, + cpus: Optional[int] = None, + mem_mb: Optional[int] = None, + command: List[str] = [], + parameters: Optional[Dict[str, Any]] = None, + environment: Optional[KeyValuePairTypeDef] = None, + watch_until: List[Status] = [], + after_success: bool = False, +) -> None: + """Submits a batch job and optionally waits for it to reach one of the given states. + + Args: + job_definition: the ARN for the AWS batch job definition, or the name of the job definition + to get the latest revision + name: the name of the job, otherwise one will be automatically generated + region_name: the AWS region + print_logs: true to print CloudWatch logs, false otherwise + queue: the name of the AWS batch queue + cpus: the number of CPUs to request + mem_mb: the amount of memory to request (in megabytes) + command: the command(s) to use + parameters: the (JSON) dictionary of parameters to use + environment: the (JSON) dictionary of environment variables to use + watch_until: watch until any of the given statuses are reached. If the job reaches a + status past all statuses, then an exception is thrown. For example, `Running` will + fail if `Succeeded` is reached, while `Succeeded` will fail if `Failed` is reached. To + wait for the job to complete regardless of status, use both `Succeeded` and `Failed`. + See the `--after-success` option to control this behavior. + after_success: true to treat states after the `watch_until` states as success, otherwise + failure. + """ + logger = logging.getLogger(__name__) + + batch_client: batch.Client = boto3.client( + service_name="batch", region_name=region_name # type: ignore + ) + + job = BatchJob( + client=batch_client, + queue=queue, + job_definition=job_definition, + name=name, + cpus=cpus, + mem_mb=mem_mb, + command=command, + environment=None if environment is None else [environment], + parameters=parameters, + logger=logger, + ) + + # Submit the job + logger.info("Submitting job...") + job.submit() + logger.info(f"Job submitted with name '{job.name}' and id '{job.job_id}'") + + # Optionally wait on it to complete + # Note: watch_until should be type Optional[List[Status]], but see: + # - https://github.com/anntzer/defopt/issues/83 + if len(watch_until) > 0: + if print_logs: + _log_it(region_name=region_name, job=job, logger=logger) + + # Wait for the job to reach on of the statuses + job.wait_on( + status_to_state=dict((status, True) for status in watch_until), + after_success=after_success, + ) + logger.info( + f"Job name '{job.name}' and id '{job.job_id}' reached status '{job.get_status()}'" + ) + + +def _watch_logs( + region_name: str, + job: BatchJob, + logger: logging.Logger, + polling_interval: int = DEFAULT_LOGS_POLLING_INTERVAL, +) -> None: + """A method to watch logs indefinitely. + + Args: + region_name: the AWS region + job: the AWS batch job + logger: the logger to which logs should be printed + polling_interval: the default time to wait for new CloudWatch logs after no more logs are + returned + """ + # wait until it's running to get the CloudWatch logs + job.wait_on_running() + + client: Optional[logs.Client] = boto3.client( + service_name="logs", region_name=region_name # type: ignore + ) + log: Log = Log(client=client, group="/aws/batch/job", stream=job.stream) + + while True: + for line in log: + logger.info(line) + time.sleep(polling_interval) + log.reset() diff --git a/pyfgaws/core/__init__.py b/pyfgaws/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyfgaws/core/logging.py b/pyfgaws/core/logging.py new file mode 100644 index 0000000..5a9e7d5 --- /dev/null +++ b/pyfgaws/core/logging.py @@ -0,0 +1,42 @@ +""" +Methods for setting up logging for tools. +----------------------------------------- +""" + +import logging +import socket +from threading import RLock + + +# Global that is set to True once logging initialization is run to prevent running > once. +__PYFGAWS_LOGGING_SETUP: bool = False + +# A lock used to make sure initialization is performed only once +__LOCK = RLock() + + +def setup_logging(level: str = "INFO") -> None: + """Globally configure logging for all modules under pyfgaws. + + Configures logging to run at a specific level and output messages to stderr with + useful information preceding the actual log message. + """ + global __PYFGAWS_LOGGING_SETUP + + with __LOCK: + if not __PYFGAWS_LOGGING_SETUP: + format = ( + f"%(asctime)s {socket.gethostname()} %(name)s:%(funcName)s:%(lineno)s " + + "[%(levelname)s]: %(message)s" + ) + handler = logging.StreamHandler() + handler.setLevel(level) + handler.setFormatter(logging.Formatter(format)) + + logger = logging.getLogger("pyfgaws") + logger.setLevel(level) + logger.addHandler(handler) + else: + logging.getLogger(__name__).warn("Logging already initialized.") + + __PYFGAWS_LOGGING_SETUP = True diff --git a/pyfgaws/logs/__init__.py b/pyfgaws/logs/__init__.py new file mode 100644 index 0000000..e029158 --- /dev/null +++ b/pyfgaws/logs/__init__.py @@ -0,0 +1,3 @@ +from pyfgaws.logs.api import DEFAULT_POLLING_INTERVAL, Log + +__all__ = ("DEFAULT_POLLING_INTERVAL", "Log") diff --git a/pyfgaws/logs/api.py b/pyfgaws/logs/api.py new file mode 100644 index 0000000..2814537 --- /dev/null +++ b/pyfgaws/logs/api.py @@ -0,0 +1,71 @@ +""" +Utility methods for interacting with AWS CloudWatch Logs +-------------------------------------------------------- +""" + +import queue +from typing import Iterator +from typing import Optional + +from mypy_boto3 import logs + +# The number of seconds to wait to poll for a job's status +DEFAULT_POLLING_INTERVAL: int = 5 + + +class Log(Iterator[str]): + """Log to iterate through CloudWatch logs. + + Iterating over this class returns all available events. + + The `reset()` method allows iterating across any newly available events since the last + iteration. + + Attributes: + client: the logs client + group: the log group name + stream: the log stream name + next_token: the token for the next set of items to return, or None if the oldest events. + """ + + def __init__( + self, client: logs.Client, group: str, stream: str, next_token: Optional[str] = None + ) -> None: + self.client: logs.Client = client + self.group: str = group + self.stream: str = stream + self.next_token: Optional[str] = next_token + self._events: queue.Queue = queue.Queue() + self._is_done: bool = False + + def __next__(self) -> str: + if not self._events.empty(): + return self._events.get() + elif self._is_done: + raise StopIteration + + # add more events + if self.next_token is None: + response = self.client.get_log_events( + logGroupName=self.group, logStreamName=self.stream, startFromHead=True, + ) + else: + response = self.client.get_log_events( + logGroupName=self.group, + logStreamName=self.stream, + nextToken=self.next_token, + startFromHead=True, + ) + for event in response["events"]: + string: str = f"""{event["timestamp"]} {event["message"]}""" + self._events.put(item=string) + old_next_token = self.next_token + self.next_token = response["nextForwardToken"] + self._is_done = self.next_token == old_next_token + + return self.__next__() + + def reset(self) -> "Log": + """Reset the iterator such that new events may be returned.""" + self._is_done = False + return self diff --git a/pyfgaws/logs/tests/__init__.py b/pyfgaws/logs/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyfgaws/logs/tests/test_api.py b/pyfgaws/logs/tests/test_api.py new file mode 100644 index 0000000..67592ba --- /dev/null +++ b/pyfgaws/logs/tests/test_api.py @@ -0,0 +1,81 @@ +"""Tests for :module:`~pyfgaws.logs.api`""" + +from typing import List + +import pytest +from mypy_boto3.logs import Client +from mypy_boto3.logs.type_defs import GetLogEventsResponseTypeDef # noqa + +from pyfgaws.logs import Log +from pyfgaws.tests import stubbed_client + + +def stubbed_client_get_log_events(service_responses: List[GetLogEventsResponseTypeDef]) -> Client: + return stubbed_client( + service_name="logs", method="get_log_events", service_responses=service_responses + ) + + +def valid_empty_service_response() -> GetLogEventsResponseTypeDef: + return {"events": [], "nextForwardToken": "token-2", "nextBackwardToken": "token-2"} + + +def valid_single_event_service_response() -> GetLogEventsResponseTypeDef: + return { + "events": [{"timestamp": 123, "message": "message", "ingestionTime": 123}], + "nextForwardToken": "token-2", + "nextBackwardToken": "token-1", + } + + +def valid_multiple_event_service_response() -> GetLogEventsResponseTypeDef: + return { + "events": [ + {"timestamp": 1, "message": "message-1", "ingestionTime": 11}, + {"timestamp": 2, "message": "message-2", "ingestionTime": 12}, + ], + "nextForwardToken": "token-2", + "nextBackwardToken": "token-1", + } + + +def valid_multiple_service_responses() -> List[GetLogEventsResponseTypeDef]: + return [ + { + "events": [{"timestamp": 123, "message": "message", "ingestionTime": 123}], + "nextForwardToken": "token-2", + "nextBackwardToken": "token-1", + }, + { + "events": [ + {"timestamp": 1, "message": "message-1", "ingestionTime": 11}, + {"timestamp": 2, "message": "message-2", "ingestionTime": 12}, + ], + "nextForwardToken": "token-3", + "nextBackwardToken": "token-2", + }, + {"events": [], "nextForwardToken": "token-3", "nextBackwardToken": "token-3"}, + ] + + +@pytest.mark.parametrize( + "service_responses", + [ + [valid_empty_service_response(), valid_empty_service_response()], + [valid_single_event_service_response(), valid_empty_service_response()], + [valid_multiple_event_service_response(), valid_empty_service_response()], + valid_multiple_service_responses(), + ], +) +def test_get_log_events(service_responses: List[GetLogEventsResponseTypeDef]) -> None: + client = stubbed_client_get_log_events(service_responses=service_responses) + + events = list(Log(client=client, group="name", stream="name")) + + expected = [ + f"""{item["timestamp"]} {item["message"]}""" + for service_response in service_responses + for item in service_response["events"] + ] + + assert events == expected diff --git a/pyfgaws/logs/tools.py b/pyfgaws/logs/tools.py new file mode 100644 index 0000000..57886f9 --- /dev/null +++ b/pyfgaws/logs/tools.py @@ -0,0 +1,51 @@ +""" +Command-line tools for interacting with AWS CloudWatch Logs +----------------------------------------------------------- +""" + +import logging +import time +from typing import Optional + +import boto3 + +from pyfgaws.logs import DEFAULT_POLLING_INTERVAL +from pyfgaws.logs import Log + + +def watch_logs( + *, + group: str, + stream: str, + region_name: Optional[str] = None, + polling_interval: int = DEFAULT_POLLING_INTERVAL, + raw_output: bool = False, +) -> None: + """Watches a cloud watch log stream + + Args: + group: the name of the log group + stream: the name of the log stream + region_name: the AWS region + polling_interval: the time to wait before polling for new logs + raw_output: output the logs directly to standard output, otherwise uses the internal + logger. + """ + logger = logging.getLogger(__name__) + + client = boto3.client( + service_name="logs", region_name=region_name # type: ignore + ) + + logger.info(f"Polling log group '{group}' and stream '{stream}'") + + log: Log = Log(client=client, group=group, stream=stream) + + while True: + for line in log: + if raw_output: + print(line) + else: + logger.info(line) + time.sleep(polling_interval) + log.reset() diff --git a/pyfgaws/tests/__init__.py b/pyfgaws/tests/__init__.py new file mode 100644 index 0000000..e3e8cd8 --- /dev/null +++ b/pyfgaws/tests/__init__.py @@ -0,0 +1,43 @@ +"""Testing utilities for :module:`~pyfgaws`""" + +from typing import Any +from typing import List + +import botocore.session +import pytest +from botocore.stub import Stubber + + +def _to_name(tool) -> str: # type: ignore + """Gives the tool name for a function by taking the function name and replacing + underscores with hyphens.""" + return tool.__name__.replace("_", "-") + + +def test_tool_funcs(tool, main) -> None: # type: ignore + name = _to_name(tool) + argv = [name, "-h"] + with pytest.raises(SystemExit) as e: + main(argv=argv) + assert e.type == SystemExit + assert e.value.code == 0 # code should be 0 for help + + +def stubbed_client(service_name: str, method: str, service_responses: List[Any]) -> Any: + """Creates a stubbed client. + + Args: + service_name: the name of the AWS service (ex. logs, batch) to stub + method: The name of the client method to stub + service_responses: one or more service responses to add + + Returns: + an activated stubbed client with the given responses added + """ + + client = botocore.session.get_session().create_client(service_name, region_name="us-east-1") + stubber = Stubber(client) + for service_response in service_responses: + stubber.add_response(method=method, service_response=service_response) + stubber.activate() + return client diff --git a/pyfgaws/tests/test_main.py b/pyfgaws/tests/test_main.py new file mode 100644 index 0000000..4209fc7 --- /dev/null +++ b/pyfgaws/tests/test_main.py @@ -0,0 +1,73 @@ +"""Tests for :module:`~pyfgws.tools.__main__` + +Motivation +~~~~~~~~~~ + +The idea is to run help on the main tools method (`-h`) as well as on +each tool (ex. `tool-name -h`). This should force :module:`~defopt` +to parse the docstring for the method. Since :module:`~defopt` uses +:module:`~argparse` underneath, `SystemExit`s are raised, which are +different than regular `Exceptions`. The exit code returned by help +(the usage) is 0. A improperly formatted docstring throws a +:class:`NotImplementedException`. We add a few test below to show +this. +""" + +import defopt +import pytest + +from pyfgaws.__main__ import TOOLS +from pyfgaws.__main__ import main +from pyfgaws.tests import test_tool_funcs as _test_tool_funcs + + +def invalid_docs_func(*, a: str) -> None: + """This is a dummy method with incorrectly formatted docs + + * this is bad + + Args: + a: some string + """ + print(a) + + +def valid_docs_func(*, a: int) -> None: + """Some func + + * this is good + + Args: + a: some int + """ + print(a) + + +def test_incorrect_docs() -> None: + """A little test to demonstrate that when the error when a tools docs are + incorrectly formatted, a TypeError is thrown""" + with pytest.raises(TypeError): + defopt.run(invalid_docs_func, argv=["-a", "str"]) + + +def test_incorrect_param_type() -> None: + """A little test to demonstrate that when the input arg has the wrong type, + a SystemExit is thrown with exit code 2""" + with pytest.raises(SystemExit) as e: + defopt.run(valid_docs_func, argv=["-a", "str"]) + assert e.type == SystemExit + assert e.value.code == 2 # code should be 2 for parse error + + +def test_tools_help() -> None: + """ Tests that running dts.tools with -h exits OK""" + argv = ["-h"] + with pytest.raises(SystemExit) as e: + main(argv=argv) + assert e.type == SystemExit + assert e.value.code == 0 # code should be 0 for help + + +@pytest.mark.parametrize("tool", TOOLS) +def test_tool_funcs(tool) -> None: # type: ignore + _test_tool_funcs(tool, main) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f9f2ca2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,49 @@ +[tool.poetry] +name = "pyfgaws" +version = "0.0.1" +description = "Tools and python libraries for working with AWS." +authors = ["Nils Homer", "Tim Fennell"] +license = "MIT" +readme = "README.md" +homepage = "https://github.com/fulcrumgenomics/pyfgaws" +repository = "https://github.com/fulcrumgenomics/pyfgaws" +keywords = ["aws", "bioinformatics"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: Software Development :: Documentation", + "Topic :: Software Development :: Libraries :: Python Modules", +] +include = [ + "LICENSE", +] + + +[tool.poetry.dependencies] +python = "^3.8" +attrs = "^19.3.0" +defopt = "^6.0" +boto3 = "^1.13.18" +botocore = "^1.13.18" +boto3-stubs = {extras = ["batch", "logs"], version = "^1.13.19"} +namegenerator = "^1.0.6" + +[tool.poetry.dev-dependencies] +pytest = "^5.4.2" +mypy = "^0.770" +flake8 = "^3.8.1" +black = "^19.10b0" +pytest-cov = ">=2.8.1" + +[tool.poetry.scripts] +fgaws-tools = "pyfgaws.__main__:main" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api"