From 49f1136faeba68b939c498a12fcf94cf4fd97479 Mon Sep 17 00:00:00 2001 From: pcstout Date: Wed, 20 Nov 2019 08:10:47 -0700 Subject: [PATCH] Convert to Package. Add Tests. Refactor. --- .appveyor.yml | 20 + .coveragerc | 6 + .gitignore | 27 +- .travis.yml | 32 ++ CHANGELOG.md | 5 + MANIFEST.in | 3 + Makefile | 47 ++ NOTICE | 13 + Pipfile | 22 + Pipfile.lock | 665 +++++++++++++++++++++++ README.md | 66 ++- requirements.txt | 1 - setup.cfg | 2 + setup.py | 33 ++ src/synapse_uploader/__init__.py | 0 src/synapse_uploader/cli.py | 110 ++++ src/synapse_uploader/synapse_uploader.py | 308 +++++++++++ src/synapse_uploader/utils.py | 42 ++ synapse_uploader.py | 321 ----------- tests/__init__.py | 0 tests/conftest.py | 83 +++ tests/synapse_test_helper.py | 113 ++++ tests/templates/private.test.env.json | 8 + tests/test_cli.py | 21 + tests/test_synapse_test_helper.py | 88 +++ tests/test_synapse_uploader.py | 405 ++++++++++++++ 26 files changed, 2093 insertions(+), 348 deletions(-) create mode 100644 .appveyor.yml create mode 100644 .coveragerc create mode 100644 .travis.yml create mode 100644 CHANGELOG.md create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 NOTICE create mode 100644 Pipfile create mode 100644 Pipfile.lock delete mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 src/synapse_uploader/__init__.py create mode 100644 src/synapse_uploader/cli.py create mode 100755 src/synapse_uploader/synapse_uploader.py create mode 100644 src/synapse_uploader/utils.py delete mode 100755 synapse_uploader.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/synapse_test_helper.py create mode 100644 tests/templates/private.test.env.json create mode 100644 tests/test_cli.py create mode 100644 tests/test_synapse_test_helper.py create mode 100644 tests/test_synapse_uploader.py diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..0ab0a66 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,20 @@ +environment: + PYTHONIOENCODING: UTF-8 + LOG_LEVEL: DEBUG + matrix: + - PYTHON: "C:\\Python36-x64" + - PYTHON: "C:\\Python37-x64" + +install: + - set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% + - pip install pipenv + - pipenv --python=%PYTHON%\\python.exe + - pipenv lock -r >> requirements.txt + - pipenv lock -r --dev >> requirements.txt + - pip install -r requirements.txt + +build: off + +test_script: + - python --version + - pytest -v -s --cov diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f6994d2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = + src +omit = + */tests/* diff --git a/.gitignore b/.gitignore index 71bf35f..e058c11 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,26 @@ -.vscode +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +*.pyc + +.idea .venv -log.txt \ No newline at end of file +.vscode +.coverage +htmlcov +tests/private.* +*.log diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..578e8ec --- /dev/null +++ b/.travis.yml @@ -0,0 +1,32 @@ +sudo: false +language: python +matrix: + include: + - os: linux + python: 3.6 + - os: linux + python: 3.7 + - os: osx + language: generic + env: PYENV_VERSION=3.6.8 + - os: osx + language: generic + env: PYENV_VERSION=3.7.4 +install: + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew outdated pyenv || brew upgrade pyenv ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then pyenv install $PYENV_VERSION ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then ~/.pyenv/versions/$PYENV_VERSION/bin/python -m venv .venv ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then source .venv/bin/activate ; fi + - python --version + - python -m pip install -U pip + - python -m pip install -U pipenv + - pipenv install --dev + - pip install coveralls +before_script: + - python -m coverage erase +script: + - python --version + - python -m pytest -v -s --cov +after_success: + - python -m coveralls diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e4e0a43 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +## Version 0.0.1 (2019-11-20) +### Added +- Initial release. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f178f26 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE +include NOTICE +include README.md diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8380bae --- /dev/null +++ b/Makefile @@ -0,0 +1,47 @@ +.PHONY: pip_install +pip_install: + pipenv install --dev + + +.PHONY: test +test: + pytest -v --cov --cov-report=term --cov-report=html + + +.PHONY: build +build: clean + python setup.py sdist + python setup.py bdist_wheel + twine check dist/* + + +.PHONY: clean +clean: + rm -rf ./build/* + rm -rf ./dist/* + rm -rf ./htmlcov + + +.PHONY: install_local +install_local: + pip install -e . + + +.PHONY: install_test +install_test: + pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple synapse-uploader + + +.PHONY: publish_test +publish_test: build + twine upload --repository-url https://test.pypi.org/legacy/ dist/* + + +.PHONY: publish +publish: build + twine upload dist/* + + +.PHONY: uninstall +uninstall: + pip uninstall -y synapse-uploader diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..fd5d52d --- /dev/null +++ b/NOTICE @@ -0,0 +1,13 @@ +Copyright 2019-present, Bill & Melinda Gates Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..d70308e --- /dev/null +++ b/Pipfile @@ -0,0 +1,22 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pytest = "*" +pylint = "*" +pytest-mock = "*" +pytest-pylint = "*" +pytest-cov = "*" +coverage = "*" +coveralls = "*" +twine = "*" +wheel = "*" +autopep8 = "*" + +[packages] +synapseclient = "*" + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..7331a4b --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,665 @@ +{ + "_meta": { + "hash": { + "sha256": "32ce3db4fcb722a7e476dc939f89d103b3ca36a82b48c3585d191213c440eaf0" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "backports.csv": { + "hashes": [ + "sha256:1277dfff73130b2e106bf3dd347adb3c5f6c4340882289d88f31240da92cbd6d", + "sha256:21f6e09bab589e6c1f877edbc40277b65e626262a86e69a70137db714eaac5ce" + ], + "version": "==1.0.7" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "cffi": { + "hashes": [ + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" + }, + "deprecated": { + "hashes": [ + "sha256:8bfeba6e630abf42b5d111b68a05f7fe3d6de7004391b3cd614947594f87a4ff", + "sha256:b784e0ca85a8c1e694d77e545c10827bd99772392e79d5f5442e761515a1246e" + ], + "version": "==1.2.4" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "version": "==0.18.2" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "keyring": { + "hashes": [ + "sha256:26edbe23dd0d79682b9c2cf825e58376fa87412ba4c94018edbbcabb7dcedfcf", + "sha256:445d9521b4fcf900e51c075112e25ddcf8af1db7d1d717380b64eda2cda84abc" + ], + "version": "==12.0.2" + }, + "keyrings.alt": { + "hashes": [ + "sha256:6a00fa799baf1385cf9620bd01bcc815aa56e6970342a567bcfea0c4d21abe5f", + "sha256:b59c86b67b9027a86e841a49efc41025bcc3b1b0308629617b66b7011e52db5a" + ], + "version": "==3.1" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "secretstorage": { + "hashes": [ + "sha256:3af65c87765323e6f64c83575b05393f9e003431959c9395d1791d51497f29b6" + ], + "markers": "sys_platform == 'linux2' or sys_platform == 'linux'", + "version": "==2.3.1" + }, + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "version": "==1.13.0" + }, + "synapseclient": { + "hashes": [ + "sha256:d739a872b8135a82c79bef88312b2b6acf874fa1e1cb37e2d4d26bcda298cd2d" + ], + "index": "pypi", + "version": "==1.9.4" + }, + "urllib3": { + "hashes": [ + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + ], + "version": "==1.25.7" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + } + }, + "develop": { + "astroid": { + "hashes": [ + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + ], + "version": "==2.3.3" + }, + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "autopep8": { + "hashes": [ + "sha256:4d8eec30cc81bc5617dbf1218201d770dc35629363547f17577c61683ccfb3ee" + ], + "index": "pypi", + "version": "==1.4.4" + }, + "bleach": { + "hashes": [ + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" + ], + "version": "==3.1.0" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "cffi": { + "hashes": [ + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "coverage": { + "hashes": [ + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + ], + "index": "pypi", + "version": "==4.5.4" + }, + "coveralls": { + "hashes": [ + "sha256:9bc5a1f92682eef59f688a8f280207190d9a6afb84cef8f567fa47631a784060", + "sha256:fb51cddef4bc458de347274116df15d641a735d3f0a580a9472174e2e62f408c" + ], + "index": "pypi", + "version": "==1.8.2" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "importlib-metadata": { + "hashes": [ + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + ], + "markers": "python_version < '3.8'", + "version": "==0.23" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "jeepney": { + "hashes": [ + "sha256:13806f91a96e9b2623fd2a81b950d763ee471454aafd9eb6d75dbe7afce428fb", + "sha256:f6a3f93464a0cf052f4e87da3c8b3ed1e27696758fb9739c63d3a74d9a1b6774" + ], + "version": "==0.4.1" + }, + "keyring": { + "hashes": [ + "sha256:26edbe23dd0d79682b9c2cf825e58376fa87412ba4c94018edbbcabb7dcedfcf", + "sha256:445d9521b4fcf900e51c075112e25ddcf8af1db7d1d717380b64eda2cda84abc" + ], + "version": "==12.0.2" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "more-itertools": { + "hashes": [ + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + ], + "version": "==7.2.0" + }, + "packaging": { + "hashes": [ + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + ], + "version": "==19.2" + }, + "pkginfo": { + "hashes": [ + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" + ], + "version": "==1.5.0.1" + }, + "pluggy": { + "hashes": [ + "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", + "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + ], + "version": "==0.13.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "pygments": { + "hashes": [ + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + ], + "version": "==2.4.2" + }, + "pylint": { + "hashes": [ + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + ], + "index": "pypi", + "version": "==2.4.4" + }, + "pyparsing": { + "hashes": [ + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + ], + "version": "==2.4.5" + }, + "pytest": { + "hashes": [ + "sha256:8e256fe71eb74e14a4d20a5987bb5e1488f0511ee800680aaedc62b9358714e8", + "sha256:ff0090819f669aaa0284d0f4aad1a6d9d67a6efdc6dd4eb4ac56b704f890a0d6" + ], + "index": "pypi", + "version": "==5.2.4" + }, + "pytest-cov": { + "hashes": [ + "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b", + "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626" + ], + "index": "pypi", + "version": "==2.8.1" + }, + "pytest-mock": { + "hashes": [ + "sha256:ba8c0c38fdccee77d4666373e95cc115954863ede741b8505088fa1fc1a87c6c", + "sha256:fff6cbd15a05f104062aa778e1ded35927d3d29bb1164218b140678fb368e32b" + ], + "index": "pypi", + "version": "==1.12.0" + }, + "pytest-pylint": { + "hashes": [ + "sha256:8c38ea779e540e27ec4378b0820d906006e09f4ac834defbd886abbf57c7d2ec", + "sha256:a4f5e5007f88c2095dcac799e9f7eed3d7e7a2e657596e26093814980ff5fa20", + "sha256:a574c246535308f8f6ceac10fa82f8fffffa837071f7985b22515895185700c1" + ], + "index": "pypi", + "version": "==0.14.1" + }, + "readme-renderer": { + "hashes": [ + "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", + "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" + ], + "version": "==24.0" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "secretstorage": { + "hashes": [ + "sha256:3af65c87765323e6f64c83575b05393f9e003431959c9395d1791d51497f29b6" + ], + "markers": "sys_platform == 'linux2' or sys_platform == 'linux'", + "version": "==2.3.1" + }, + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "version": "==1.13.0" + }, + "tqdm": { + "hashes": [ + "sha256:9de4722323451eb7818deb0161d9d5523465353a6707a9f500d97ee42919b902", + "sha256:c1d677f3a85fa291b34bdf8f770f877119b9754b32673699653556f85e2c2f13" + ], + "version": "==4.38.0" + }, + "twine": { + "hashes": [ + "sha256:8d85e75338c97ea7ed04330b1dce1d948ce83cec333fb9a0e26a11ffdc4a40dd", + "sha256:af3a83c627bd609d3ffe0d48f420e28584c448764ceeb203bb8eafdc8eabb250" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "typed-ast": { + "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", + "version": "==1.4.0" + }, + "urllib3": { + "hashes": [ + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + ], + "version": "==1.25.7" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, + "wheel": { + "hashes": [ + "sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646", + "sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28" + ], + "index": "pypi", + "version": "==0.33.6" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + }, + "zipp": { + "hashes": [ + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" + ], + "version": "==0.6.0" + } + } +} diff --git a/README.md b/README.md index 3e23b97..f298c0c 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,38 @@ -# Synapse File Uploader +# Synapse Uploader -A utility to upload a directory and all its contents to a [Synapse](https://www.synapse.org/) Project. +Utility to upload a directory and files to [Synapse](https://www.synapse.org/). ## Dependencies -- [Python](https://www.python.org/) +- [Python3](https://www.python.org/) - A [Synapse](https://www.synapse.org/) account with a username/password. Authentication through a 3rd party (.e.g., Google) will not work, you must have a Synapse user/pass for the [API to authenticate](http://docs.synapse.org/python/#connecting-to-synapse). -- synapseclient - Follow install instructions [here](http://docs.synapse.org/python/) or `(sudo) pip3 install (--upgrade) synapseclient[pandas,pysftp]` ## Install -Copy the Python file to your local system or clone the GIT repository. - ```bash -$ git clone git@github.com:pcstout/synapse_uploader.git -$ cd synapse_uploader -$ pip install -r requirements.txt -$ chmod u+x *.py +pip install synapse-uploader ``` -Add environment variables for Synapse credentials. -Specify these variables on the command line when executing the script or add them to your environment. +## Configuration + +Your Synapse credentials can be provided on the command line (`--username`, `--password`) or via environment variables. + ```bash -SYNAPSE_USER=your-synapse-username +SYNAPSE_USERNAME=your-synapse-username SYNAPSE_PASSWORD=your-synapse-password ``` ## Usage ```text -usage: synapse_uploader.py [-h] [-r REMOTE_FOLDER_PATH] [-d DEPTH] - [-u USERNAME] [-p PASSWORD] [-l LOG_LEVEL] - project-id local-folder-path +usage: synapse-uploader [-h] [-r REMOTE_FOLDER_PATH] [-d DEPTH] [-t THREADS] + [-u USERNAME] [-p PASSWORD] [-ll LOG_LEVEL] + [-ld LOG_DIR] + entity-id local-path positional arguments: - project-id Synapse Project ID to upload to (e.g., syn123456789). - local-folder-path Path of the folder to upload. + entity-id Synapse entity ID to upload to (e.g., syn123456789). + local-path Path of the directory or file to upload. optional arguments: -h, --help show this help message and exit @@ -44,24 +41,45 @@ optional arguments: -d DEPTH, --depth DEPTH The maximum number of child folders or files under a Synapse Project/Folder. + -t THREADS, --threads THREADS + The maximum number of threads to use. -u USERNAME, --username USERNAME Synapse username. -p PASSWORD, --password PASSWORD Synapse password. - -l LOG_LEVEL, --log-level LOG_LEVEL + -ll LOG_LEVEL, --log-level LOG_LEVEL Set the logging level. + -ld LOG_DIR, --log-dir LOG_DIR + Set the directory where the log file will be written. ``` ## Examples Upload all the folders and files in `~/my_study` to your Project ID `syn123456`: -- Linux: `./synapse_uploader.py syn123456 ~/my_study` -- Windows: `synapse_uploader.py syn123456 %USERPROFILE%\my_study` +- Linux: `synapse_uploader syn123456 ~/my_study` +- Windows: `synapse_uploader syn123456 %USERPROFILE%\my_study` Upload all the folders and files in `~/my_study` to your Project ID `syn123456` in the `drafts/my_study` folder: -- Linux: `./synapse_uploader.py syn123456 ~/my_study -r drafts/my_study` -- Windows: `synapse_uploader.py syn123456 %USERPROFILE%\my_study -r drafts\my_study` +- Linux: `synapse_uploader syn123456 ~/my_study -r drafts/my_study` +- Windows: `synapse_uploader syn123456 %USERPROFILE%\my_study -r drafts\my_study` + +> Note: The correct path separator (`\` for Windows and `/` for Linux) must be used in both the `local-folder-path` and the `remote-folder-path`. + +## Development Setup + +```bash +pipenv --three +pipenv shell +make pip_install +make build +make install_local +``` +See [Makefile](Makefile) for all commands. + +### Testing -> Note: The correct path separator (`\` for Windows and `/` for Linux) must be used in both the `local-folder-path` and the `remote-folder-path`. \ No newline at end of file +- Create and activate a virtual environment: +- Copy [private.test.env.json](tests/templates/private.test.env.json) to the [tests](tests) directory and set each of the variables. +- Run the tests: `make test` diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5174cbe..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -synapseclient==1.9.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3480374 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..407247a --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="synapse-uploader", + version="0.0.1", + author="Patrick Stout", + author_email="pstout@prevagroup.com", + license="Apache2", + description="Utility to upload a directory and files to Synapse.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/ki-tools/synapse_uploader", + package_dir={"": "src"}, + packages=setuptools.find_packages(where="src"), + classifiers=( + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ), + entry_points={ + 'console_scripts': [ + "synapse-uploader = synapse_uploader.cli:main" + ] + }, + install_requires=[ + "synapseclient>=1.9.4,<2.0.0" + ] +) diff --git a/src/synapse_uploader/__init__.py b/src/synapse_uploader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/synapse_uploader/cli.py b/src/synapse_uploader/cli.py new file mode 100644 index 0000000..02822b6 --- /dev/null +++ b/src/synapse_uploader/cli.py @@ -0,0 +1,110 @@ +import os +import logging +import argparse +from datetime import datetime +from .synapse_uploader import SynapseUploader +from .utils import Utils + + +class LogFilter(logging.Filter): + FILTERS = [ + '##################################################', + 'Uploading file to Synapse storage', + 'Connection pool is full, discarding connection:' + ] + + def filter(self, record): + for filter in self.FILTERS: + if filter in record.msg: + return False + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('entity_id', + metavar='entity-id', + help='Synapse entity ID to upload to (e.g., syn123456789).') + + parser.add_argument('local_path', + metavar='local-path', + help='Path of the directory or file to upload.') + + parser.add_argument('-r', '--remote-folder-path', + help='Folder to upload to in Synapse.', + default=None) + + parser.add_argument('-d', '--depth', + help='The maximum number of child folders or files under a Synapse Project/Folder.', + type=int, + default=SynapseUploader.MAX_SYNAPSE_DEPTH) + + parser.add_argument('-t', '--threads', + help='The maximum number of threads to use.', + type=int, + default=None) + + parser.add_argument('-u', '--username', + help='Synapse username.', + default=None) + + parser.add_argument('-p', '--password', + help='Synapse password.', + default=None) + + parser.add_argument('-ll', '--log-level', + help='Set the logging level.', + default='INFO') + + parser.add_argument('-ld', '--log-dir', + help='Set the directory where the log file will be written.') + + args = parser.parse_args() + + log_level = getattr(logging, args.log_level.upper()) + + timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f") + log_filename = '{0}.log'.format(timestamp) + + if args.log_dir: + log_filename = os.path.join(Utils.expand_path(args.log_dir), log_filename) + else: + log_filename = os.path.join(Utils.app_log_dir(), log_filename) + + Utils.ensure_dirs(os.path.dirname(log_filename)) + + logging.basicConfig( + filename=log_filename, + filemode='w', + format='%(asctime)s %(levelname)s: %(message)s', + level=log_level + ) + + # Add console logging. + console = logging.StreamHandler() + console.setLevel(log_level) + console.setFormatter(logging.Formatter('%(message)s')) + logging.getLogger().addHandler(console) + + # Filter logs + log_filter = LogFilter() + for logger in [logging.getLogger(name) for name in logging.root.manager.loggerDict]: + logger.addFilter(log_filter) + + print('Logging output to: {0}'.format(log_filename)) + + SynapseUploader( + args.entity_id, + args.local_path, + remote_path=args.remote_folder_path, + max_depth=args.depth, + max_threads=args.threads, + username=args.username, + password=args.password + ).execute() + + print('Output logged to: {0}'.format(log_filename)) + + +if __name__ == "__main__": + main() diff --git a/src/synapse_uploader/synapse_uploader.py b/src/synapse_uploader/synapse_uploader.py new file mode 100755 index 0000000..e5b0713 --- /dev/null +++ b/src/synapse_uploader/synapse_uploader.py @@ -0,0 +1,308 @@ +import os +import getpass +import time +import random +import concurrent.futures +import threading +import logging +from datetime import datetime +import synapseclient as syn +from .utils import Utils + + +class SynapseUploader: + # Maximum number of files per Project/Folder in Synapse. + MAX_SYNAPSE_DEPTH = 10000 + + # Minimum depth for Projects/Folders in Synapse. + MIN_SYNAPSE_DEPTH = 2 + + def __init__(self, + synapse_entity_id, + local_path, + remote_path=None, + max_depth=MAX_SYNAPSE_DEPTH, + max_threads=None, + username=None, + password=None, + synapse_client=None): + + self._synapse_entity_id = synapse_entity_id + self._local_path = Utils.expand_path(local_path) + self._remote_path = remote_path + self._max_depth = max_depth + self._max_threads = max_threads + self._username = username + self._password = password + self._synapse_client = synapse_client + + self.start_time = None + self.end_time = None + + self._thread_lock = threading.Lock() + self._synapse_parents = {} + self.has_errors = False + + if max_depth > self.MAX_SYNAPSE_DEPTH: + raise Exception('Maximum depth must be less than or equal to {0}.'.format(self.MAX_SYNAPSE_DEPTH)) + + if max_depth < self.MIN_SYNAPSE_DEPTH: + raise Exception('Maximum depth must be greater than or equal to {0}.'.format(self.MIN_SYNAPSE_DEPTH)) + + if remote_path: + self._remote_path = remote_path.replace(' ', '').lstrip(os.sep).rstrip(os.sep) + if len(self._remote_path) == 0: + self._remote_path = None + + def execute(self): + self.start_time = datetime.now() + + if not self._synapse_login(): + self.has_errors = True + logging.error('Could not log into Synapse. Aborting.') + return + + remote_entity = self._synapse_client.get(self._synapse_entity_id, downloadFile=False) + remote_entity_is_file = False + + if isinstance(remote_entity, syn.Project): + remote_type = 'Project' + self._set_synapse_parent(remote_entity) + elif isinstance(remote_entity, syn.Folder): + remote_type = 'Folder' + self._set_synapse_parent(remote_entity) + elif isinstance(remote_entity, syn.File): + remote_type = 'File' + remote_entity_is_file = True + else: + raise Exception('Remote entity must be a project, folder, or file. Found {0}'.format(type(remote_entity))) + + local_entity_is_file = False + + if os.path.isfile(self._local_path): + local_type = 'File' + local_entity_is_file = True + elif os.path.isdir(self._local_path): + local_type = 'Directory' + else: + raise Exception('Local entity must be a directory or file: {0}'.format(self._local_path)) + + if remote_entity_is_file and not local_entity_is_file: + raise Exception('Local entity must be a file when remote entity is a file: {0}'.format(self._local_path)) + + if remote_entity_is_file and self._remote_path: + raise Exception( + 'Cannot specify a remote path when remote entity is a file: {0}'.format(self._local_path)) + + logging.info('Uploading to {0}: {1} ({2})'.format(remote_type, remote_entity.name, remote_entity.id)) + logging.info('Uploading {0}: {1}'.format(local_type, self._local_path)) + + if remote_entity_is_file: + remote_file_name = remote_entity['_file_handle']['fileName'] + local_file_name = os.path.basename(self._local_path) + if local_file_name != remote_file_name: + raise Exception('Local filename: {0} does not match remote file name: {1}'.format(local_file_name, + remote_file_name)) + + remote_parent = self._synapse_client.get(remote_entity.get('parentId')) + self._set_synapse_parent(remote_parent) + self._upload_file_to_synapse(self._local_path, remote_parent) + else: + if self._remote_path: + logging.info('Uploading to: {0}'.format(self._remote_path)) + + remote_parent = remote_entity + + # Create the remote_path if specified. + if self._remote_path: + full_path = '' + for folder in filter(None, self._remote_path.split(os.sep)): + full_path = os.path.join(full_path, folder) + remote_parent = self._create_folder_in_synapse(full_path, remote_parent) + + with concurrent.futures.ThreadPoolExecutor(max_workers=self._max_threads) as executor: + self._upload_folder(executor, self._local_path, remote_parent) + + self.end_time = datetime.now() + logging.info('') + logging.info('Run time: {0}'.format(self.end_time - self.start_time)) + + if self.has_errors: + logging.error('Finished with errors. Please see log file.') + else: + logging.info('Finished successfully.') + + def _synapse_login(self): + if self._synapse_client and self._synapse_client.credentials: + logging.info('Already logged into Synapse.') + else: + self._username = self._username or os.getenv('SYNAPSE_USERNAME') + self._password = self._password or os.getenv('SYNAPSE_PASSWORD') + + if not self._username: + self._username = input('Synapse username: ') + + if not self._password: + self._password = getpass.getpass(prompt='Synapse password: ') + + logging.info('Logging into Synapse as: {0}'.format(self._username)) + try: + self._synapse_client = syn.Synapse(skip_checks=True) + self._synapse_client.login(self._username, self._password, silent=True) + except Exception as ex: + self._synapse_client = None + self.has_errors = True + logging.error('Synapse login failed: {0}'.format(str(ex))) + + return self._synapse_client is not None + + def _upload_folder(self, executor, local_path, synapse_parent): + if not synapse_parent: + self.has_errors = True + logging.error('Parent not found, cannot execute folder: {0}'.format(local_path)) + return + + parent = synapse_parent + + dirs, files = self._get_dirs_and_files(local_path) + + child_count = 0 + + # Upload the files + for file_entry in files: + if (child_count + 1) >= self._max_depth: + parent = self._create_folder_in_synapse('more', parent) + child_count = 0 + + executor.submit(self._upload_file_to_synapse, file_entry.path, parent) + child_count += 1 + + # Upload the directories. + for dir_entry in dirs: + if (child_count + 1) >= self._max_depth: + parent = self._create_folder_in_synapse('more', parent) + child_count = 0 + + syn_dir = self._create_folder_in_synapse(dir_entry.path, parent) + self._upload_folder(executor, dir_entry.path, syn_dir) + child_count += 1 + + def _create_folder_in_synapse(self, path, synapse_parent): + synapse_folder = None + + if not synapse_parent: + self.has_errors = True + logging.error('Parent not found, cannot create folder: {0}'.format(path)) + return synapse_folder + + folder_name = os.path.basename(path) + full_synapse_path = self._get_synapse_path(folder_name, synapse_parent) + + max_attempts = 5 + attempt_number = 0 + exception = None + + while attempt_number < max_attempts and not synapse_folder: + try: + attempt_number += 1 + exception = None + synapse_folder = self._synapse_client.store(syn.Folder(name=folder_name, parent=synapse_parent), + forceVersion=False) + except Exception as ex: + exception = ex + logging.error('[Folder ERROR] {0} -> {1} : {2}'.format(path, full_synapse_path, str(ex))) + if attempt_number < max_attempts: + sleep_time = random.randint(1, 5) + logging.info('[Folder RETRY in {0}s] {1} -> {2}'.format(sleep_time, path, full_synapse_path)) + time.sleep(sleep_time) + + if exception: + self.has_errors = True + logging.error('[Folder FAILED] {0} -> {1} : {2}'.format(path, full_synapse_path, str(exception))) + else: + logging.info('[Folder] {0} -> {1}'.format(path, full_synapse_path)) + self._set_synapse_parent(synapse_folder) + + return synapse_folder + + def _upload_file_to_synapse(self, local_file, synapse_parent): + synapse_file = None + + if not synapse_parent: + self.has_errors = True + logging.error('Parent not found, cannot execute file: {0}'.format(local_file)) + return synapse_file + + # Skip empty files since these will error when uploading via the synapseclient. + if os.path.getsize(local_file) < 1: + logging.info('Skipping empty file: {0}'.format(local_file)) + return synapse_file + + file_name = os.path.basename(local_file) + full_synapse_path = self._get_synapse_path(file_name, synapse_parent) + + max_attempts = 5 + attempt_number = 0 + exception = None + + while attempt_number < max_attempts and not synapse_file: + try: + attempt_number += 1 + exception = None + synapse_file = self._synapse_client.store( + syn.File(path=local_file, name=file_name, parent=synapse_parent), + forceVersion=False) + except Exception as ex: + exception = ex + logging.error('[File ERROR] {0} -> {1} : {2}'.format(local_file, full_synapse_path, str(ex))) + if attempt_number < max_attempts: + sleep_time = random.randint(1, 5) + logging.info('[File RETRY in {0}s] {1} -> {2}'.format(sleep_time, local_file, full_synapse_path)) + time.sleep(sleep_time) + + if exception: + self.has_errors = True + logging.error('[File FAILED] {0} -> {1} : {2}'.format(local_file, full_synapse_path, str(exception))) + else: + logging.info('[File] {0} -> {1}'.format(local_file, full_synapse_path)) + + return synapse_file + + def _set_synapse_parent(self, parent): + with self._thread_lock: + self._synapse_parents[parent.id] = parent + + def _get_synapse_parent(self, parent_id): + with self._thread_lock: + return self._synapse_parents.get(parent_id, None) + + def _get_synapse_path(self, folder_or_file_name, parent): + segments = [] + + if isinstance(parent, syn.Project): + segments.insert(0, parent.name) + else: + next_parent = parent + while next_parent: + segments.insert(0, next_parent.name) + next_parent = self._get_synapse_parent(next_parent.parentId) + + segments.append(folder_or_file_name) + + return os.path.join(*segments) + + def _get_dirs_and_files(self, local_path): + dirs = [] + files = [] + + with os.scandir(local_path) as iter: + for entry in iter: + if entry.is_dir(follow_symlinks=False): + dirs.append(entry) + else: + files.append(entry) + + dirs.sort(key=lambda f: f.name) + files.sort(key=lambda f: f.name) + + return dirs, files diff --git a/src/synapse_uploader/utils.py b/src/synapse_uploader/utils.py new file mode 100644 index 0000000..a18a5d9 --- /dev/null +++ b/src/synapse_uploader/utils.py @@ -0,0 +1,42 @@ +import os +import pathlib + + +class Utils: + + @staticmethod + def app_dir(): + """Gets the application's primary directory for the current user. + + Returns: + Absolute path to the directory. + """ + return os.path.join(pathlib.Path.home(), '.syntools') + + @staticmethod + def app_log_dir(): + """Gets the applications primary log directory for the current user. + + Returns: + Absolute path to the directory. + """ + return os.path.join(Utils.app_dir(), 'logs') + + @staticmethod + def expand_path(local_path): + var_path = os.path.expandvars(local_path) + expanded_path = os.path.expanduser(var_path) + return os.path.abspath(expanded_path) + + @staticmethod + def ensure_dirs(local_path): + """Ensures the directories in local_path exist. + + Args: + local_path: The local path to ensure. + + Returns: + None + """ + if not os.path.isdir(local_path): + os.makedirs(local_path) diff --git a/synapse_uploader.py b/synapse_uploader.py deleted file mode 100755 index 243ee53..0000000 --- a/synapse_uploader.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017-present, Bill & Melinda Gates Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os -import argparse -import getpass -import time -import random -import concurrent.futures -import threading -import logging -import synapseclient -from synapseclient import Project, Folder, File - - -class SynapseUploader: - - # Maximum number of files per Project/Folder in Synapse. - MAX_SYNAPSE_DEPTH = 10000 - - # Minimum depth for Projects/Folders in Synapse. - MIN_SYNAPSE_DEPTH = 2 - - def __init__(self, synapse_project, local_path, remote_path=None, max_depth=MAX_SYNAPSE_DEPTH, username=None, password=None): - self._synapse_project = synapse_project - self._local_path = local_path.rstrip(os.sep) - self._remote_path = None - self._max_depth = max_depth - self._synapse_parents = {} - self._username = username - self._password = password - self._hasErrors = False - self._thread_lock = threading.Lock() - - if max_depth < self.MIN_SYNAPSE_DEPTH: - raise Exception('Maximum depth must be greater than {0}.'.format( - self.MIN_SYNAPSE_DEPTH - 1)) - - if remote_path != None and len(remote_path.strip()) > 0: - self._remote_path = remote_path.strip().lstrip(os.sep).rstrip(os.sep) - if len(self._remote_path) == 0: - self._remote_path = None - - def start(self): - logging.info('Uploading to Project: {0}'.format(self._synapse_project)) - logging.info('Uploading Directory: {0}'.format(self._local_path)) - - if self._remote_path != None: - logging.info('Uploading To: {0}'.format(self._remote_path)) - - self.login() - - project = self._synapse_client.get(Project(id=self._synapse_project)) - self.set_synapse_parent(project) - - parent = project - - # Create the remote_path if specified. - if self._remote_path: - full_path = '' - for folder in filter(None, self._remote_path.split(os.sep)): - full_path = os.path.join(full_path, folder) - parent = self.create_directory_in_synapse(full_path, parent) - - self.upload_folder(self._local_path, parent) - - completion_status = 'With Errors' if self._hasErrors else 'Successfully' - - logging.info('Upload Completed {0}'.format(completion_status)) - - def login(self): - logging.info('Logging into Synapse...') - syn_user = self._username or os.getenv('SYNAPSE_USER') - syn_pass = self._password or os.getenv('SYNAPSE_PASSWORD') - - if syn_user == None: - syn_user = input('Synapse username: ') - - if syn_pass == None: - syn_pass = getpass.getpass(prompt='Synapse password: ') - - self._synapse_client = synapseclient.Synapse() - self._synapse_client.login(syn_user, syn_pass, silent=True) - - def set_synapse_parent(self, parent): - with self._thread_lock: - self._synapse_parents[parent.id] = parent - - def get_synapse_parent(self, parent_id): - with self._thread_lock: - return self._synapse_parents.get(parent_id, None) - - def get_synapse_path(self, folder_or_file_name, parent): - segments = [] - - if isinstance(parent, Project): - segments.insert(0, parent.name) - else: - next_parent = parent - while next_parent: - if next_parent: - segments.insert(0, next_parent.name) - next_parent = self.get_synapse_parent(next_parent.parentId) - - segments.append(folder_or_file_name) - - return os.path.join(*segments) - - def get_dirs_and_files(self, local_path): - dirs = [] - files = [] - - with os.scandir(local_path) as iter: - for entry in iter: - if entry.is_dir(follow_symlinks=False): - dirs.append(entry) - elif entry.is_file(follow_symlinks=False): - files.append(entry) - - dirs.sort(key=lambda f: f.name) - files.sort(key=lambda f: f.name) - - return dirs, files - - def upload_folder(self, local_path, synapse_parent): - parent = synapse_parent - - dirs, files = self.get_dirs_and_files(local_path) - - child_count = 0 - - with concurrent.futures.ThreadPoolExecutor() as executor: - # Upload the directories. - for dir_entry in dirs: - if (child_count + 1) >= self._max_depth: - parent = self.create_directory_in_synapse('more', parent) - child_count = 0 - - syn_dir = self.create_directory_in_synapse( - dir_entry.path, parent) - executor.submit(self.upload_folder, dir_entry.path, syn_dir) - child_count += 1 - - # Upload the files - for file_entry in files: - if (child_count + 1) >= self._max_depth: - parent = self.create_directory_in_synapse('more', parent) - child_count = 0 - - executor.submit(self.upload_file_to_synapse, - file_entry.path, parent) - child_count += 1 - - def create_directory_in_synapse(self, path, synapse_parent): - if not synapse_parent: - self._hasErrors = True - logging.error( - ' -! Parent not found, cannot create folder: {0}'.format(path)) - return - - folder_name = os.path.basename(path) - full_synapse_path = self.get_synapse_path(folder_name, synapse_parent) - - logging.info( - 'Processing Folder: {0}{1} -> {2}'.format(path, os.linesep, full_synapse_path)) - - synapse_folder = Folder(name=folder_name, parent=synapse_parent) - - max_attempts = 5 - attempt_number = 0 - - while attempt_number < max_attempts and not synapse_folder.get('id', None): - try: - attempt_number += 1 - synapse_folder = self._synapse_client.store( - synapse_folder, forceVersion=False) - except Exception as ex: - logging.error( - ' -! Error creating folder: {0}'.format(str(ex))) - if attempt_number < max_attempts: - sleep_time = random.randint(1, 5) - logging.info( - ' -! Retrying in {0} seconds'.format(sleep_time)) - time.sleep(sleep_time) - - if not synapse_folder.get('id', None): - self._hasErrors = True - logging.error(' -! Failed to create folder: {0}'.format(path)) - else: - if attempt_number > 1: - logging.info(' -> Folder created') - self.set_synapse_parent(synapse_folder) - - return synapse_folder - - def upload_file_to_synapse(self, local_file, synapse_parent): - if not synapse_parent: - self._hasErrors = True - logging.error( - ' -! Parent not found, cannot upload file: {0}'.format(local_file)) - return None - - # Skip empty files since these will error when uploading via the synapseclient. - if (os.path.getsize(local_file) < 1): - logging.info('Skipping Empty File: {0}'.format(local_file)) - return None - - file_name = os.path.basename(local_file) - full_synapse_path = self.get_synapse_path(file_name, synapse_parent) - - logging.info( - 'Processing File: {0}{1} -> {2}'.format(local_file, os.linesep, full_synapse_path)) - - synapse_file = File(path=local_file, parent=synapse_parent) - - max_attempts = 5 - attempt_number = 0 - - while attempt_number < max_attempts and not synapse_file.get('id', None): - try: - attempt_number += 1 - synapse_file = self._synapse_client.store( - synapse_file, forceVersion=False) - except Exception as ex: - logging.error(' -! Error uploading file: {0}'.format(str(ex))) - if attempt_number < max_attempts: - sleep_time = random.randint(1, 5) - logging.info( - ' -! Retrying in {0} seconds'.format(sleep_time)) - time.sleep(sleep_time) - - if not synapse_file.get('id', None): - self._hasErrors = True - logging.error(' -! Failed to upload file: {0}'.format(local_file)) - else: - if attempt_number > 1: - logging.info(' -> File uploaded') - - return synapse_file - - -class LogFilter(logging.Filter): - FILTERS = [ - '##################################################', - 'Uploading file to Synapse storage', - 'Connection pool is full, discarding connection:' - ] - - def filter(self, record): - for filter in self.FILTERS: - if filter in record.msg: - return False - return True - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('project_id', metavar='project-id', - help='Synapse Project ID to upload to (e.g., syn123456789).') - parser.add_argument('local_folder_path', metavar='local-folder-path', - help='Path of the folder to upload.') - parser.add_argument('-r', '--remote-folder-path', - help='Folder to upload to in Synapse.', default=None) - parser.add_argument('-d', '--depth', help='The maximum number of child folders or files under a Synapse Project/Folder.', - type=int, default=SynapseUploader.MAX_SYNAPSE_DEPTH) - parser.add_argument('-u', '--username', - help='Synapse username.', default=None) - parser.add_argument('-p', '--password', - help='Synapse password.', default=None) - parser.add_argument('-l', '--log-level', - help='Set the logging level.', default='INFO') - - args = parser.parse_args() - - log_level = getattr(logging, args.log_level.upper()) - log_file_name = 'log.txt' - - logging.basicConfig( - filename=log_file_name, - filemode='w', - format='%(asctime)s %(levelname)s: %(message)s', - level=log_level - ) - - # Add console logging. - console = logging.StreamHandler() - console.setLevel(log_level) - console.setFormatter(logging.Formatter('%(message)s')) - logging.getLogger().addHandler(console) - - # Filter logs - filter = LogFilter() - for logger in [logging.getLogger(name) for name in logging.root.manager.loggerDict]: - logger.addFilter(filter) - - SynapseUploader( - args.project_id, - args.local_folder_path, - remote_path=args.remote_folder_path, - max_depth=args.depth, - username=args.username, - password=args.password - ).start() - - -if __name__ == "__main__": - main() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9fb2a25 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,83 @@ +import pytest +import tempfile +import os +import json +import shutil +from tests.synapse_test_helper import SynapseTestHelper + +# Load Environment variables. +module_dir = os.path.dirname(os.path.abspath(__file__)) + +test_env_file = os.path.join(module_dir, 'private.test.env.json') + +if os.path.isfile(test_env_file): + with open(test_env_file) as f: + config = json.load(f).get('test') + + # Validate required properties are present + for prop in ['SYNAPSE_USERNAME', 'SYNAPSE_PASSWORD']: + if not prop in config or not config[prop]: + raise Exception( + 'Property: "{0}" is missing in {1}'.format(prop, test_env_file)) + + for key, value in config.items(): + os.environ[key] = value +else: + print('WARNING: Test environment file not found at: {0}'.format(test_env_file)) + + +@pytest.fixture(scope='session') +def syn_client(syn_test_helper): + return syn_test_helper.client() + + +@pytest.fixture(scope='session') +def syn_test_helper(): + """ + Provides the SynapseTestHelper as a fixture per session. + """ + helper = SynapseTestHelper() + yield helper + helper.dispose() + + +@pytest.fixture(scope='session') +def syn_project(syn_test_helper): + return syn_test_helper.create_project() + + +@pytest.fixture() +def new_syn_test_helper(): + """ + Provides the SynapseTestHelper as a fixture per function. + """ + helper = SynapseTestHelper() + yield helper + helper.dispose() + + +@pytest.fixture() +def new_syn_project(new_syn_test_helper): + return new_syn_test_helper.create_project() + + +@pytest.fixture() +def new_temp_dir(): + path = tempfile.mkdtemp() + yield path + if os.path.isdir(path): + shutil.rmtree(path) + + +@pytest.fixture() +def new_temp_file(syn_test_helper): + """ + Generates a temp file containing the SynapseTestHelper.uniq_name. + """ + fd, tmp_filename = tempfile.mkstemp() + with os.fdopen(fd, 'w') as tmp: + tmp.write(syn_test_helper.uniq_name()) + yield tmp_filename + + if os.path.isfile(tmp_filename): + os.remove(tmp_filename) diff --git a/tests/synapse_test_helper.py b/tests/synapse_test_helper.py new file mode 100644 index 0000000..449595c --- /dev/null +++ b/tests/synapse_test_helper.py @@ -0,0 +1,113 @@ +import os +import uuid +import synapseclient +from synapseclient import Project, Folder, File + + +class SynapseTestHelper: + """ + Test helper for working with Synapse. + """ + _test_id = uuid.uuid4().hex + _trash = [] + _synapse_client = None + + def client(self): + if not self._synapse_client: + syn_user = os.getenv('SYNAPSE_USERNAME') + syn_pass = os.getenv('SYNAPSE_PASSWORD') + + self._synapse_client = synapseclient.Synapse() + self._synapse_client.login(syn_user, syn_pass, silent=True) + + return self._synapse_client + + def test_id(self): + """ + Gets a unique value to use as a test identifier. + This string can be used to help identify the test instance that created the object. + """ + return self._test_id + + def uniq_name(self, prefix='', postfix=''): + return "{0}{1}_{2}{3}".format(prefix, self.test_id(), uuid.uuid4().hex, postfix) + + def dispose_of(self, *syn_objects): + """ + Adds a Synapse object to the list of objects to be deleted. + """ + for syn_object in syn_objects: + if syn_object in self._trash: + continue + self._trash.append(syn_object) + + def dispose(self): + """ + Cleans up any Synapse objects that were created during testing. + This method needs to be manually called after each or all tests are done. + """ + projects = [] + folders = [] + files = [] + others = [] + + for obj in self._trash: + if isinstance(obj, Project): + projects.append(obj) + elif isinstance(obj, Folder): + folders.append(obj) + elif isinstance(obj, File): + files.append(obj) + else: + others.append(obj) + + for syn_obj in files: + try: + self.client().delete(syn_obj) + except: + pass + self._trash.remove(syn_obj) + + for syn_obj in folders: + try: + self.client().delete(syn_obj) + except: + pass + self._trash.remove(syn_obj) + + for syn_obj in projects: + try: + self.client().delete(syn_obj) + except: + pass + self._trash.remove(syn_obj) + + for obj in others: + print('WARNING: Non-Supported object found: {0}'.format(obj)) + self._trash.remove(obj) + + def create_project(self, **kwargs): + """ + Creates a new Project and adds it to the trash queue. + """ + if not 'name' in kwargs: + kwargs['name'] = self.uniq_name(prefix=kwargs.get('prefix', '')) + + kwargs.pop('prefix', None) + + project = self.client().store(Project(**kwargs)) + self.dispose_of(project) + return project + + def create_file(self, **kwargs): + """ + Creates a new File and adds it to the trash queue. + """ + if not 'name' in kwargs: + kwargs['name'] = self.uniq_name(prefix=kwargs.get('prefix', '')) + + kwargs.pop('prefix', None) + + file = self.client().store(File(**kwargs)) + self.dispose_of(file) + return file diff --git a/tests/templates/private.test.env.json b/tests/templates/private.test.env.json new file mode 100644 index 0000000..ed50458 --- /dev/null +++ b/tests/templates/private.test.env.json @@ -0,0 +1,8 @@ +{ + "description": "This file contains environment variables that are used during tests.", + "test": { + "SYNAPSE_USERNAME": "...set to a Synapse username that is suitable for testing...", + "SYNAPSE_PASSWORD": "...set to Synapse password...", + "LOG_LEVEL": "DEBUG" + } +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..463223c --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,21 @@ +import src.synapse_uploader.cli as cli +from src.synapse_uploader.synapse_uploader import SynapseUploader + + +def test_cli(mocker): + args = ['', 'syn123', '/tmp', '-r', '10', '-d', '20', '-t', '30', '-u', '40', '-p', '50', '-ll', 'debug'] + mocker.patch('sys.argv', args) + mocker.patch('src.synapse_uploader.synapse_uploader.SynapseUploader.execute', return_value=None) + mock_init = mocker.patch.object(SynapseUploader, '__init__', return_value=None) + + cli.main() + + mock_init.assert_called_once_with( + 'syn123', + '/tmp', + remote_path='10', + max_depth=20, + max_threads=30, + username='40', + password='50' + ) diff --git a/tests/test_synapse_test_helper.py b/tests/test_synapse_test_helper.py new file mode 100644 index 0000000..25c6453 --- /dev/null +++ b/tests/test_synapse_test_helper.py @@ -0,0 +1,88 @@ +import os +import pytest +import synapseclient +from synapseclient import Project, Folder, File + + +def test_test_id(syn_test_helper): + assert syn_test_helper.test_id() == syn_test_helper._test_id + + +def test_uniq_name(syn_test_helper): + assert syn_test_helper.test_id() in syn_test_helper.uniq_name() + + last_name = None + for i in list(range(3)): + uniq_name = syn_test_helper.uniq_name(prefix='aaa-', postfix='-zzz') + assert uniq_name != last_name + assert uniq_name.startswith( + 'aaa-{0}'.format(syn_test_helper.test_id())) + assert uniq_name.endswith('-zzz') + last_name = uniq_name + + +def test_dispose_of(syn_test_helper): + # Add a single object + for obj in [object(), object()]: + syn_test_helper.dispose_of(obj) + assert obj in syn_test_helper._trash + + # Add a list of objects + obj1 = object() + obj2 = object() + syn_test_helper.dispose_of(obj1, obj2) + assert obj1 in syn_test_helper._trash + assert obj2 in syn_test_helper._trash + + # Does not add duplicates + syn_test_helper.dispose_of(obj1, obj2) + assert len(syn_test_helper._trash) == 4 + + +def test_dispose(syn_client, syn_test_helper, new_temp_file): + project = syn_client.store(Project(name=syn_test_helper.uniq_name())) + + folder = syn_client.store( + Folder(name=syn_test_helper.uniq_name(prefix='Folder '), parent=project)) + + file = syn_client.store(File(name=syn_test_helper.uniq_name(prefix='File '), path=new_temp_file, parent=folder)) + + syn_objects = [project, folder, file] + + for syn_obj in syn_objects: + syn_test_helper.dispose_of(syn_obj) + assert syn_obj in syn_test_helper._trash + + syn_test_helper.dispose() + assert len(syn_test_helper._trash) == 0 + + for syn_obj in syn_objects: + with pytest.raises(synapseclient.exceptions.SynapseHTTPError) as ex: + syn_client.get(syn_obj, downloadFile=False) + + err_str = str(ex.value) + assert "Not Found" in err_str or "cannot be found" in err_str or "is in trash can" in err_str or "does not exist" in err_str + + try: + os.remove(new_temp_file) + except: + pass + + +def test_create_project(syn_test_helper): + # Uses the name arg + name = syn_test_helper.uniq_name() + project = syn_test_helper.create_project(name=name) + assert project.name == name + assert project in syn_test_helper._trash + syn_test_helper.dispose() + assert project not in syn_test_helper._trash + + # Uses the prefix arg + prefix = '-z-z-z-' + project = syn_test_helper.create_project(prefix=prefix) + assert project.name.startswith(prefix) + + # Generates a name + project = syn_test_helper.create_project() + assert syn_test_helper.test_id() in project.name diff --git a/tests/test_synapse_uploader.py b/tests/test_synapse_uploader.py new file mode 100644 index 0000000..5371f75 --- /dev/null +++ b/tests/test_synapse_uploader.py @@ -0,0 +1,405 @@ +import os +import uuid +import getpass +import pytest +import synapseclient as syn +from src.synapse_uploader.synapse_uploader import SynapseUploader + + +def mkdir(*path_segments): + path = os.path.join(*path_segments) + if not os.path.isdir(path): + os.mkdir(path) + return path + + +def mkfile(*path_segments, content=str(uuid.uuid4())): + path = os.path.join(*path_segments) + with open(path, 'w') as file: + file.write(content) + return path + + +def get_syn_folders(syn_client, syn_parent): + syn_folders = list(syn_client.getChildren(syn_parent, includeTypes=['folder'])) + syn_folder_names = [s['name'] for s in syn_folders] + return syn_folders, syn_folder_names + + +def get_syn_files(syn_client, syn_parent): + syn_files = list(syn_client.getChildren(syn_parent, includeTypes=['file'])) + syn_file_names = [s['name'] for s in syn_files] + return syn_files, syn_file_names + + +def find_by_name(list, name): + return next((x for x in list if x['name'] == name), None) + + +def test_synapse_project_value(): + syn_id = 'syn123' + syn_uploader = SynapseUploader(syn_id, 'None') + assert syn_uploader._synapse_entity_id == syn_id + + +def test_local_path_value(): + local_path = os.getcwd() + syn_uploader = SynapseUploader('None', local_path) + assert syn_uploader._local_path == local_path + + +def test_remote_path_value(): + path_segments = ['one', 'two', 'three'] + remote_path = os.sep.join(path_segments) + + syn_uploader = SynapseUploader('None', 'None', remote_path=remote_path) + assert syn_uploader._remote_path == remote_path + + # Strips spaces and separators + syn_uploader = SynapseUploader('None', 'None', remote_path='{0} {0}'.format(os.sep)) + assert syn_uploader._remote_path is None + + syn_uploader = SynapseUploader('None', 'None', remote_path='{0} one {0}'.format(os.sep)) + assert syn_uploader._remote_path == 'one' + + syn_uploader = SynapseUploader('None', 'None', remote_path='{0} one {0} two {0} three {0}'.format(os.sep)) + assert syn_uploader._remote_path == remote_path + + +def test_max_depth_value(): + max_depth = 10 + syn_uploader = SynapseUploader('None', 'None', max_depth=max_depth) + assert syn_uploader._max_depth == max_depth + + with pytest.raises(Exception) as ex: + SynapseUploader('None', 'None', max_depth=(SynapseUploader.MAX_SYNAPSE_DEPTH + 1)) + assert str(ex.value) == 'Maximum depth must be less than or equal to 10000.' + + +def test_min_depth_value(): + with pytest.raises(Exception) as ex: + SynapseUploader('None', 'None', max_depth=(SynapseUploader.MIN_SYNAPSE_DEPTH - 1)) + assert str(ex.value) == 'Maximum depth must be greater than or equal to 2.' + + +def test_username_value(): + username = 'test_user' + syn_uploader = SynapseUploader('None', 'None', username=username) + assert syn_uploader._username == username + + +def test_password_value(): + password = 'test_password' + syn_uploader = SynapseUploader('None', 'None', password=password) + assert syn_uploader._password == password + + +def test_synapse_client_value(): + client = object() + syn_uploader = SynapseUploader('None', 'None', synapse_client=client) + assert syn_uploader._synapse_client == client + + +def test_login(syn_client, monkeypatch, mocker): + # Uses ENV + syn_uploader = SynapseUploader('None', 'None') + syn_uploader._synapse_login() is True + assert syn_uploader._synapse_client is not None + + # Uses the passed in params + syn_uploader = SynapseUploader('None', 'None', + username=os.environ['SYNAPSE_USERNAME'], password=os.environ['SYNAPSE_PASSWORD']) + assert syn_uploader._synapse_login() is True + assert syn_uploader._synapse_client is not None + + # Uses the passed in client + syn_uploader = SynapseUploader('None', 'None', synapse_client=syn_client) + syn_uploader._synapse_login() is True + assert syn_uploader._synapse_client == syn_client + + # Fails to _synapse_login + syn_uploader = SynapseUploader('None', 'None', username=uuid.uuid4(), password=uuid.uuid4()) + assert syn_uploader._synapse_login() is False + assert syn_uploader._synapse_client is None + + # Prompts for the username and password + with monkeypatch.context() as mp: + mp.delenv('SYNAPSE_USERNAME') + mp.delenv('SYNAPSE_PASSWORD') + + mock_username = uuid.uuid4() + mock_password = uuid.uuid4() + + mocker.patch('builtins.input', return_value=mock_username) + mocker.patch('getpass.getpass', return_value=mock_password) + syn_uploader = SynapseUploader('None', 'None') + syn_uploader._synapse_login() + assert syn_uploader._username == mock_username + assert syn_uploader._password == mock_password + input.assert_called_once() + getpass.getpass.assert_called_once() + + +def test_upload_bad_credentials(): + syn_uploader = SynapseUploader('None', 'None', username=uuid.uuid4(), password=uuid.uuid4()) + syn_uploader.execute() + assert syn_uploader._synapse_client is None + + +def test_upload_remote_path(syn_client, new_syn_project, new_temp_dir): + """ + Tests this scenario: + + Remote Path: one/two/three + + file1 + folder1/ + file2 + folder2/ + file3 + """ + path_segments = ['one', 'two', 'three'] + remote_path = os.path.join(*path_segments) + + mkfile(new_temp_dir, 'file1') + folder1 = mkdir(new_temp_dir, 'folder1') + mkfile(folder1, 'file2') + folder2 = mkdir(folder1, 'folder2') + mkfile(folder2, 'file3') + + SynapseUploader(new_syn_project.id, new_temp_dir, remote_path=remote_path, synapse_client=syn_client).execute() + + parent = new_syn_project + for segment in path_segments: + syn_files, syn_file_names = get_syn_files(syn_client, parent) + syn_folders, syn_folder_names = get_syn_folders(syn_client, parent) + assert len(syn_files) == 0 + assert len(syn_folders) == 1 + folder = find_by_name(syn_folders, segment) + assert folder + parent = folder + + syn_files, syn_file_names = get_syn_files(syn_client, parent) + syn_folders, syn_folder_names = get_syn_folders(syn_client, parent) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_folder = find_by_name(syn_folders, 'folder1') + assert syn_folder + assert syn_file_names == ['file1'] + assert syn_folder_names == ['folder1'] + + syn_files, syn_file_names = get_syn_files(syn_client, syn_folder) + syn_folders, syn_folder_names = get_syn_folders(syn_client, syn_folder) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_folder = find_by_name(syn_folders, 'folder2') + assert syn_folder + assert syn_file_names == ['file2'] + assert syn_folder_names == ['folder2'] + + syn_files, syn_file_names = get_syn_files(syn_client, syn_folder) + syn_folders, _ = get_syn_folders(syn_client, syn_folder) + assert len(syn_files) == 1 + assert len(syn_folders) == 0 + assert syn_file_names == ['file3'] + + +def test_upload(syn_client, syn_test_helper, new_temp_dir): + """ + Tests this scenario: + + file1 + file2 + file3 + folder1/ + file4 + folder2/ + file5 + folder3/ + file6 + """ + for i in range(1, 4): + mkfile(new_temp_dir, 'file{0}'.format(i)) + + folder1 = mkdir(new_temp_dir, 'folder1') + mkfile(folder1, 'file4') + folder2 = mkdir(folder1, 'folder2') + mkfile(folder2, 'file5') + folder3 = mkdir(folder2, 'folder3') + mkfile(folder3, 'file6') + mkfile(folder3, 'file7', content='') # Empty files should NOT get uploaded. + + project1 = syn_test_helper.create_project() + project2 = syn_test_helper.create_project() + + # Test uploading to a Project and Folder + upload_targets = [project1, + syn_client.store(syn.Folder(name=syn_test_helper.uniq_name(), parent=project2))] + + for upload_target in upload_targets: + SynapseUploader(upload_target.id, new_temp_dir, synapse_client=syn_client).execute() + + syn_files, syn_file_names = get_syn_files(syn_client, upload_target) + syn_folders, _ = get_syn_folders(syn_client, upload_target) + assert len(syn_files) == 3 + assert len(syn_folders) == 1 + syn_folder = find_by_name(syn_folders, 'folder1') + assert syn_folder + assert syn_file_names == ['file1', 'file2', 'file3'] + + syn_files, _ = get_syn_files(syn_client, syn_folders[-1]) + syn_folders, _ = get_syn_folders(syn_client, syn_folders[-1]) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_file = find_by_name(syn_files, 'file4') + syn_folder = find_by_name(syn_folders, 'folder2') + assert syn_file + assert syn_folder + + syn_files, _ = get_syn_files(syn_client, syn_folders[-1]) + syn_folders, _ = get_syn_folders(syn_client, syn_folders[-1]) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_file = find_by_name(syn_files, 'file5') + syn_folder = find_by_name(syn_folders, 'folder3') + assert syn_file + assert syn_folder + + syn_files, _ = get_syn_files(syn_client, syn_folders[-1]) + syn_folders, _ = get_syn_folders(syn_client, syn_folders[-1]) + assert len(syn_files) == 1 + assert len(syn_folders) == 0 + syn_file = find_by_name(syn_files, 'file6') + assert syn_file + + +def test_upload_file(syn_client, syn_test_helper, new_syn_project, new_temp_file, new_temp_dir): + file_name = os.path.basename(new_temp_file) + syn_file = syn_test_helper.create_file(name=file_name, path=new_temp_file, parent=new_syn_project) + + SynapseUploader(syn_file.id, new_temp_file, synapse_client=syn_client).execute() + + syn_files, syn_file_names = get_syn_files(syn_client, new_syn_project) + syn_folders, _ = get_syn_folders(syn_client, new_syn_project) + assert len(syn_files) == 1 + assert len(syn_folders) == 0 + assert file_name in syn_file_names + + # Test exceptions + with pytest.raises(Exception) as ex: + SynapseUploader(syn_file.id, new_temp_dir, synapse_client=syn_client).execute() + assert 'Local entity must be a file when remote entity is a file:' in str(ex.value) + + with pytest.raises(Exception) as ex: + SynapseUploader(syn_file.id, new_temp_file, remote_path='/test', synapse_client=syn_client).execute() + assert 'Cannot specify a remote path when remote entity is a file:' in str(ex.value) + + # Local filename: {0} does not match remote file name: + other_temp_file = mkfile(new_temp_dir, syn_test_helper.uniq_name()) + other_temp_file_name = os.path.basename(other_temp_file) + other_syn_file = syn_test_helper.create_file(name=other_temp_file_name, + path=other_temp_file, + parent=new_syn_project) + with pytest.raises(Exception) as ex: + SynapseUploader(syn_file.id, other_temp_file, synapse_client=syn_client).execute() + assert 'Local filename: {0} does not match remote file name:'.format(other_temp_file_name) in str(ex.value) + + +def test_upload_max_depth(syn_client, new_syn_project, new_temp_dir): + """ + Tests this scenario: + + file1 + file2 + file3 + file4 + file5 + folder1/ + file1-1 + file1-2 + folder2 + folder3 + folder4 + folder5 + + TO: + + file1 + file2 + more/ + file3 + file4 + more/ + file5 + folder1/ + file1-1 + file1-2 + more/ + folder2 + folder3 + more/ + folder4 + folder5 + """ + for i in range(1, 6): + mkfile(new_temp_dir, 'file{0}'.format(i)) + folder_path = mkdir(new_temp_dir, 'folder{0}'.format(i)) + if i == 1: + mkfile(folder_path, 'file1-1'.format(i)) + mkfile(folder_path, 'file1-2'.format(i)) + + SynapseUploader(new_syn_project.id, new_temp_dir, max_depth=3, synapse_client=syn_client).execute() + + syn_files, syn_file_names = get_syn_files(syn_client, new_syn_project) + assert len(syn_files) == 2 + assert syn_file_names == ['file1', 'file2'] + syn_folders, syn_folder_names = get_syn_folders(syn_client, new_syn_project) + assert len(syn_folders) == 1 + assert syn_folder_names == ['more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_files, syn_file_names = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 2 + assert syn_file_names == ['file3', 'file4'] + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 1 + assert syn_folder_names == ['more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_files, syn_file_names = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 1 + assert syn_file_names == ['file5'] + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 2 + assert syn_folder_names == ['folder1', 'more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_folder1 = find_by_name(syn_folders, 'folder1') + + child_syn_files, child_syn_file_names = get_syn_files(syn_client, syn_folder1) + assert len(child_syn_files) == 2 + assert child_syn_file_names == ['file1-1', 'file1-2'] + child_syn_folders, _ = get_syn_folders(syn_client, syn_folder1) + assert len(child_syn_folders) == 0 + + syn_files, _ = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 0 + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 3 + assert syn_folder_names == ['folder2', 'folder3', 'more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_files, _ = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 0 + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 2 + assert syn_folder_names == ['folder4', 'folder5'] + + +def test_upload_failures(): + # TODO: add tests. + pass