diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..0ab0a66 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,20 @@ +environment: + PYTHONIOENCODING: UTF-8 + LOG_LEVEL: DEBUG + matrix: + - PYTHON: "C:\\Python36-x64" + - PYTHON: "C:\\Python37-x64" + +install: + - set PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH% + - pip install pipenv + - pipenv --python=%PYTHON%\\python.exe + - pipenv lock -r >> requirements.txt + - pipenv lock -r --dev >> requirements.txt + - pip install -r requirements.txt + +build: off + +test_script: + - python --version + - pytest -v -s --cov diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..f6994d2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = + src +omit = + */tests/* diff --git a/.gitignore b/.gitignore index 71bf35f..e058c11 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,26 @@ -.vscode +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +*.pyc + +.idea .venv -log.txt \ No newline at end of file +.vscode +.coverage +htmlcov +tests/private.* +*.log diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..578e8ec --- /dev/null +++ b/.travis.yml @@ -0,0 +1,32 @@ +sudo: false +language: python +matrix: + include: + - os: linux + python: 3.6 + - os: linux + python: 3.7 + - os: osx + language: generic + env: PYENV_VERSION=3.6.8 + - os: osx + language: generic + env: PYENV_VERSION=3.7.4 +install: + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew update ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then brew outdated pyenv || brew upgrade pyenv ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then pyenv install $PYENV_VERSION ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then ~/.pyenv/versions/$PYENV_VERSION/bin/python -m venv .venv ; fi + - if [ "$TRAVIS_OS_NAME" == "osx" ] ; then source .venv/bin/activate ; fi + - python --version + - python -m pip install -U pip + - python -m pip install -U pipenv + - pipenv install --dev + - pip install coveralls +before_script: + - python -m coverage erase +script: + - python --version + - python -m pytest -v -s --cov +after_success: + - python -m coveralls diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e4e0a43 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +## Version 0.0.1 (2019-11-20) +### Added +- Initial release. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f178f26 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE +include NOTICE +include README.md diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8380bae --- /dev/null +++ b/Makefile @@ -0,0 +1,47 @@ +.PHONY: pip_install +pip_install: + pipenv install --dev + + +.PHONY: test +test: + pytest -v --cov --cov-report=term --cov-report=html + + +.PHONY: build +build: clean + python setup.py sdist + python setup.py bdist_wheel + twine check dist/* + + +.PHONY: clean +clean: + rm -rf ./build/* + rm -rf ./dist/* + rm -rf ./htmlcov + + +.PHONY: install_local +install_local: + pip install -e . + + +.PHONY: install_test +install_test: + pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple synapse-uploader + + +.PHONY: publish_test +publish_test: build + twine upload --repository-url https://test.pypi.org/legacy/ dist/* + + +.PHONY: publish +publish: build + twine upload dist/* + + +.PHONY: uninstall +uninstall: + pip uninstall -y synapse-uploader diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..fd5d52d --- /dev/null +++ b/NOTICE @@ -0,0 +1,13 @@ +Copyright 2019-present, Bill & Melinda Gates Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..d70308e --- /dev/null +++ b/Pipfile @@ -0,0 +1,22 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pytest = "*" +pylint = "*" +pytest-mock = "*" +pytest-pylint = "*" +pytest-cov = "*" +coverage = "*" +coveralls = "*" +twine = "*" +wheel = "*" +autopep8 = "*" + +[packages] +synapseclient = "*" + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..7331a4b --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,665 @@ +{ + "_meta": { + "hash": { + "sha256": "32ce3db4fcb722a7e476dc939f89d103b3ca36a82b48c3585d191213c440eaf0" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "backports.csv": { + "hashes": [ + "sha256:1277dfff73130b2e106bf3dd347adb3c5f6c4340882289d88f31240da92cbd6d", + "sha256:21f6e09bab589e6c1f877edbc40277b65e626262a86e69a70137db714eaac5ce" + ], + "version": "==1.0.7" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "cffi": { + "hashes": [ + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" + }, + "deprecated": { + "hashes": [ + "sha256:8bfeba6e630abf42b5d111b68a05f7fe3d6de7004391b3cd614947594f87a4ff", + "sha256:b784e0ca85a8c1e694d77e545c10827bd99772392e79d5f5442e761515a1246e" + ], + "version": "==1.2.4" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "version": "==0.18.2" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "keyring": { + "hashes": [ + "sha256:26edbe23dd0d79682b9c2cf825e58376fa87412ba4c94018edbbcabb7dcedfcf", + "sha256:445d9521b4fcf900e51c075112e25ddcf8af1db7d1d717380b64eda2cda84abc" + ], + "version": "==12.0.2" + }, + "keyrings.alt": { + "hashes": [ + "sha256:6a00fa799baf1385cf9620bd01bcc815aa56e6970342a567bcfea0c4d21abe5f", + "sha256:b59c86b67b9027a86e841a49efc41025bcc3b1b0308629617b66b7011e52db5a" + ], + "version": "==3.1" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "secretstorage": { + "hashes": [ + "sha256:3af65c87765323e6f64c83575b05393f9e003431959c9395d1791d51497f29b6" + ], + "markers": "sys_platform == 'linux2' or sys_platform == 'linux'", + "version": "==2.3.1" + }, + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "version": "==1.13.0" + }, + "synapseclient": { + "hashes": [ + "sha256:d739a872b8135a82c79bef88312b2b6acf874fa1e1cb37e2d4d26bcda298cd2d" + ], + "index": "pypi", + "version": "==1.9.4" + }, + "urllib3": { + "hashes": [ + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + ], + "version": "==1.25.7" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + } + }, + "develop": { + "astroid": { + "hashes": [ + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + ], + "version": "==2.3.3" + }, + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "version": "==19.3.0" + }, + "autopep8": { + "hashes": [ + "sha256:4d8eec30cc81bc5617dbf1218201d770dc35629363547f17577c61683ccfb3ee" + ], + "index": "pypi", + "version": "==1.4.4" + }, + "bleach": { + "hashes": [ + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" + ], + "version": "==3.1.0" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "cffi": { + "hashes": [ + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" + ], + "version": "==1.13.2" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "coverage": { + "hashes": [ + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + ], + "index": "pypi", + "version": "==4.5.4" + }, + "coveralls": { + "hashes": [ + "sha256:9bc5a1f92682eef59f688a8f280207190d9a6afb84cef8f567fa47631a784060", + "sha256:fb51cddef4bc458de347274116df15d641a735d3f0a580a9472174e2e62f408c" + ], + "index": "pypi", + "version": "==1.8.2" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "importlib-metadata": { + "hashes": [ + "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", + "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" + ], + "markers": "python_version < '3.8'", + "version": "==0.23" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "jeepney": { + "hashes": [ + "sha256:13806f91a96e9b2623fd2a81b950d763ee471454aafd9eb6d75dbe7afce428fb", + "sha256:f6a3f93464a0cf052f4e87da3c8b3ed1e27696758fb9739c63d3a74d9a1b6774" + ], + "version": "==0.4.1" + }, + "keyring": { + "hashes": [ + "sha256:26edbe23dd0d79682b9c2cf825e58376fa87412ba4c94018edbbcabb7dcedfcf", + "sha256:445d9521b4fcf900e51c075112e25ddcf8af1db7d1d717380b64eda2cda84abc" + ], + "version": "==12.0.2" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "more-itertools": { + "hashes": [ + "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", + "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" + ], + "version": "==7.2.0" + }, + "packaging": { + "hashes": [ + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + ], + "version": "==19.2" + }, + "pkginfo": { + "hashes": [ + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" + ], + "version": "==1.5.0.1" + }, + "pluggy": { + "hashes": [ + "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", + "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" + ], + "version": "==0.13.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "pygments": { + "hashes": [ + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + ], + "version": "==2.4.2" + }, + "pylint": { + "hashes": [ + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + ], + "index": "pypi", + "version": "==2.4.4" + }, + "pyparsing": { + "hashes": [ + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + ], + "version": "==2.4.5" + }, + "pytest": { + "hashes": [ + "sha256:8e256fe71eb74e14a4d20a5987bb5e1488f0511ee800680aaedc62b9358714e8", + "sha256:ff0090819f669aaa0284d0f4aad1a6d9d67a6efdc6dd4eb4ac56b704f890a0d6" + ], + "index": "pypi", + "version": "==5.2.4" + }, + "pytest-cov": { + "hashes": [ + "sha256:cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b", + "sha256:cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626" + ], + "index": "pypi", + "version": "==2.8.1" + }, + "pytest-mock": { + "hashes": [ + "sha256:ba8c0c38fdccee77d4666373e95cc115954863ede741b8505088fa1fc1a87c6c", + "sha256:fff6cbd15a05f104062aa778e1ded35927d3d29bb1164218b140678fb368e32b" + ], + "index": "pypi", + "version": "==1.12.0" + }, + "pytest-pylint": { + "hashes": [ + "sha256:8c38ea779e540e27ec4378b0820d906006e09f4ac834defbd886abbf57c7d2ec", + "sha256:a4f5e5007f88c2095dcac799e9f7eed3d7e7a2e657596e26093814980ff5fa20", + "sha256:a574c246535308f8f6ceac10fa82f8fffffa837071f7985b22515895185700c1" + ], + "index": "pypi", + "version": "==0.14.1" + }, + "readme-renderer": { + "hashes": [ + "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", + "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" + ], + "version": "==24.0" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "version": "==2.22.0" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "secretstorage": { + "hashes": [ + "sha256:3af65c87765323e6f64c83575b05393f9e003431959c9395d1791d51497f29b6" + ], + "markers": "sys_platform == 'linux2' or sys_platform == 'linux'", + "version": "==2.3.1" + }, + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "version": "==1.13.0" + }, + "tqdm": { + "hashes": [ + "sha256:9de4722323451eb7818deb0161d9d5523465353a6707a9f500d97ee42919b902", + "sha256:c1d677f3a85fa291b34bdf8f770f877119b9754b32673699653556f85e2c2f13" + ], + "version": "==4.38.0" + }, + "twine": { + "hashes": [ + "sha256:8d85e75338c97ea7ed04330b1dce1d948ce83cec333fb9a0e26a11ffdc4a40dd", + "sha256:af3a83c627bd609d3ffe0d48f420e28584c448764ceeb203bb8eafdc8eabb250" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "typed-ast": { + "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", + "version": "==1.4.0" + }, + "urllib3": { + "hashes": [ + "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293", + "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745" + ], + "version": "==1.25.7" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, + "wheel": { + "hashes": [ + "sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646", + "sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28" + ], + "index": "pypi", + "version": "==0.33.6" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + }, + "zipp": { + "hashes": [ + "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", + "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" + ], + "version": "==0.6.0" + } + } +} diff --git a/README.md b/README.md index 3e23b97..f298c0c 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,38 @@ -# Synapse File Uploader +# Synapse Uploader -A utility to upload a directory and all its contents to a [Synapse](https://www.synapse.org/) Project. +Utility to upload a directory and files to [Synapse](https://www.synapse.org/). ## Dependencies -- [Python](https://www.python.org/) +- [Python3](https://www.python.org/) - A [Synapse](https://www.synapse.org/) account with a username/password. Authentication through a 3rd party (.e.g., Google) will not work, you must have a Synapse user/pass for the [API to authenticate](http://docs.synapse.org/python/#connecting-to-synapse). -- synapseclient - Follow install instructions [here](http://docs.synapse.org/python/) or `(sudo) pip3 install (--upgrade) synapseclient[pandas,pysftp]` ## Install -Copy the Python file to your local system or clone the GIT repository. - ```bash -$ git clone git@github.com:pcstout/synapse_uploader.git -$ cd synapse_uploader -$ pip install -r requirements.txt -$ chmod u+x *.py +pip install synapse-uploader ``` -Add environment variables for Synapse credentials. -Specify these variables on the command line when executing the script or add them to your environment. +## Configuration + +Your Synapse credentials can be provided on the command line (`--username`, `--password`) or via environment variables. + ```bash -SYNAPSE_USER=your-synapse-username +SYNAPSE_USERNAME=your-synapse-username SYNAPSE_PASSWORD=your-synapse-password ``` ## Usage ```text -usage: synapse_uploader.py [-h] [-r REMOTE_FOLDER_PATH] [-d DEPTH] - [-u USERNAME] [-p PASSWORD] [-l LOG_LEVEL] - project-id local-folder-path +usage: synapse-uploader [-h] [-r REMOTE_FOLDER_PATH] [-d DEPTH] [-t THREADS] + [-u USERNAME] [-p PASSWORD] [-ll LOG_LEVEL] + [-ld LOG_DIR] + entity-id local-path positional arguments: - project-id Synapse Project ID to upload to (e.g., syn123456789). - local-folder-path Path of the folder to upload. + entity-id Synapse entity ID to upload to (e.g., syn123456789). + local-path Path of the directory or file to upload. optional arguments: -h, --help show this help message and exit @@ -44,24 +41,45 @@ optional arguments: -d DEPTH, --depth DEPTH The maximum number of child folders or files under a Synapse Project/Folder. + -t THREADS, --threads THREADS + The maximum number of threads to use. -u USERNAME, --username USERNAME Synapse username. -p PASSWORD, --password PASSWORD Synapse password. - -l LOG_LEVEL, --log-level LOG_LEVEL + -ll LOG_LEVEL, --log-level LOG_LEVEL Set the logging level. + -ld LOG_DIR, --log-dir LOG_DIR + Set the directory where the log file will be written. ``` ## Examples Upload all the folders and files in `~/my_study` to your Project ID `syn123456`: -- Linux: `./synapse_uploader.py syn123456 ~/my_study` -- Windows: `synapse_uploader.py syn123456 %USERPROFILE%\my_study` +- Linux: `synapse_uploader syn123456 ~/my_study` +- Windows: `synapse_uploader syn123456 %USERPROFILE%\my_study` Upload all the folders and files in `~/my_study` to your Project ID `syn123456` in the `drafts/my_study` folder: -- Linux: `./synapse_uploader.py syn123456 ~/my_study -r drafts/my_study` -- Windows: `synapse_uploader.py syn123456 %USERPROFILE%\my_study -r drafts\my_study` +- Linux: `synapse_uploader syn123456 ~/my_study -r drafts/my_study` +- Windows: `synapse_uploader syn123456 %USERPROFILE%\my_study -r drafts\my_study` + +> Note: The correct path separator (`\` for Windows and `/` for Linux) must be used in both the `local-folder-path` and the `remote-folder-path`. + +## Development Setup + +```bash +pipenv --three +pipenv shell +make pip_install +make build +make install_local +``` +See [Makefile](Makefile) for all commands. + +### Testing -> Note: The correct path separator (`\` for Windows and `/` for Linux) must be used in both the `local-folder-path` and the `remote-folder-path`. \ No newline at end of file +- Create and activate a virtual environment: +- Copy [private.test.env.json](tests/templates/private.test.env.json) to the [tests](tests) directory and set each of the variables. +- Run the tests: `make test` diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5174cbe..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -synapseclient==1.9.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3480374 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..407247a --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="synapse-uploader", + version="0.0.1", + author="Patrick Stout", + author_email="pstout@prevagroup.com", + license="Apache2", + description="Utility to upload a directory and files to Synapse.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/ki-tools/synapse_uploader", + package_dir={"": "src"}, + packages=setuptools.find_packages(where="src"), + classifiers=( + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ), + entry_points={ + 'console_scripts': [ + "synapse-uploader = synapse_uploader.cli:main" + ] + }, + install_requires=[ + "synapseclient>=1.9.4,<2.0.0" + ] +) diff --git a/src/synapse_uploader/__init__.py b/src/synapse_uploader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/synapse_uploader/cli.py b/src/synapse_uploader/cli.py new file mode 100644 index 0000000..02822b6 --- /dev/null +++ b/src/synapse_uploader/cli.py @@ -0,0 +1,110 @@ +import os +import logging +import argparse +from datetime import datetime +from .synapse_uploader import SynapseUploader +from .utils import Utils + + +class LogFilter(logging.Filter): + FILTERS = [ + '##################################################', + 'Uploading file to Synapse storage', + 'Connection pool is full, discarding connection:' + ] + + def filter(self, record): + for filter in self.FILTERS: + if filter in record.msg: + return False + return True + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('entity_id', + metavar='entity-id', + help='Synapse entity ID to upload to (e.g., syn123456789).') + + parser.add_argument('local_path', + metavar='local-path', + help='Path of the directory or file to upload.') + + parser.add_argument('-r', '--remote-folder-path', + help='Folder to upload to in Synapse.', + default=None) + + parser.add_argument('-d', '--depth', + help='The maximum number of child folders or files under a Synapse Project/Folder.', + type=int, + default=SynapseUploader.MAX_SYNAPSE_DEPTH) + + parser.add_argument('-t', '--threads', + help='The maximum number of threads to use.', + type=int, + default=None) + + parser.add_argument('-u', '--username', + help='Synapse username.', + default=None) + + parser.add_argument('-p', '--password', + help='Synapse password.', + default=None) + + parser.add_argument('-ll', '--log-level', + help='Set the logging level.', + default='INFO') + + parser.add_argument('-ld', '--log-dir', + help='Set the directory where the log file will be written.') + + args = parser.parse_args() + + log_level = getattr(logging, args.log_level.upper()) + + timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f") + log_filename = '{0}.log'.format(timestamp) + + if args.log_dir: + log_filename = os.path.join(Utils.expand_path(args.log_dir), log_filename) + else: + log_filename = os.path.join(Utils.app_log_dir(), log_filename) + + Utils.ensure_dirs(os.path.dirname(log_filename)) + + logging.basicConfig( + filename=log_filename, + filemode='w', + format='%(asctime)s %(levelname)s: %(message)s', + level=log_level + ) + + # Add console logging. + console = logging.StreamHandler() + console.setLevel(log_level) + console.setFormatter(logging.Formatter('%(message)s')) + logging.getLogger().addHandler(console) + + # Filter logs + log_filter = LogFilter() + for logger in [logging.getLogger(name) for name in logging.root.manager.loggerDict]: + logger.addFilter(log_filter) + + print('Logging output to: {0}'.format(log_filename)) + + SynapseUploader( + args.entity_id, + args.local_path, + remote_path=args.remote_folder_path, + max_depth=args.depth, + max_threads=args.threads, + username=args.username, + password=args.password + ).execute() + + print('Output logged to: {0}'.format(log_filename)) + + +if __name__ == "__main__": + main() diff --git a/src/synapse_uploader/synapse_uploader.py b/src/synapse_uploader/synapse_uploader.py new file mode 100755 index 0000000..e5b0713 --- /dev/null +++ b/src/synapse_uploader/synapse_uploader.py @@ -0,0 +1,308 @@ +import os +import getpass +import time +import random +import concurrent.futures +import threading +import logging +from datetime import datetime +import synapseclient as syn +from .utils import Utils + + +class SynapseUploader: + # Maximum number of files per Project/Folder in Synapse. + MAX_SYNAPSE_DEPTH = 10000 + + # Minimum depth for Projects/Folders in Synapse. + MIN_SYNAPSE_DEPTH = 2 + + def __init__(self, + synapse_entity_id, + local_path, + remote_path=None, + max_depth=MAX_SYNAPSE_DEPTH, + max_threads=None, + username=None, + password=None, + synapse_client=None): + + self._synapse_entity_id = synapse_entity_id + self._local_path = Utils.expand_path(local_path) + self._remote_path = remote_path + self._max_depth = max_depth + self._max_threads = max_threads + self._username = username + self._password = password + self._synapse_client = synapse_client + + self.start_time = None + self.end_time = None + + self._thread_lock = threading.Lock() + self._synapse_parents = {} + self.has_errors = False + + if max_depth > self.MAX_SYNAPSE_DEPTH: + raise Exception('Maximum depth must be less than or equal to {0}.'.format(self.MAX_SYNAPSE_DEPTH)) + + if max_depth < self.MIN_SYNAPSE_DEPTH: + raise Exception('Maximum depth must be greater than or equal to {0}.'.format(self.MIN_SYNAPSE_DEPTH)) + + if remote_path: + self._remote_path = remote_path.replace(' ', '').lstrip(os.sep).rstrip(os.sep) + if len(self._remote_path) == 0: + self._remote_path = None + + def execute(self): + self.start_time = datetime.now() + + if not self._synapse_login(): + self.has_errors = True + logging.error('Could not log into Synapse. Aborting.') + return + + remote_entity = self._synapse_client.get(self._synapse_entity_id, downloadFile=False) + remote_entity_is_file = False + + if isinstance(remote_entity, syn.Project): + remote_type = 'Project' + self._set_synapse_parent(remote_entity) + elif isinstance(remote_entity, syn.Folder): + remote_type = 'Folder' + self._set_synapse_parent(remote_entity) + elif isinstance(remote_entity, syn.File): + remote_type = 'File' + remote_entity_is_file = True + else: + raise Exception('Remote entity must be a project, folder, or file. Found {0}'.format(type(remote_entity))) + + local_entity_is_file = False + + if os.path.isfile(self._local_path): + local_type = 'File' + local_entity_is_file = True + elif os.path.isdir(self._local_path): + local_type = 'Directory' + else: + raise Exception('Local entity must be a directory or file: {0}'.format(self._local_path)) + + if remote_entity_is_file and not local_entity_is_file: + raise Exception('Local entity must be a file when remote entity is a file: {0}'.format(self._local_path)) + + if remote_entity_is_file and self._remote_path: + raise Exception( + 'Cannot specify a remote path when remote entity is a file: {0}'.format(self._local_path)) + + logging.info('Uploading to {0}: {1} ({2})'.format(remote_type, remote_entity.name, remote_entity.id)) + logging.info('Uploading {0}: {1}'.format(local_type, self._local_path)) + + if remote_entity_is_file: + remote_file_name = remote_entity['_file_handle']['fileName'] + local_file_name = os.path.basename(self._local_path) + if local_file_name != remote_file_name: + raise Exception('Local filename: {0} does not match remote file name: {1}'.format(local_file_name, + remote_file_name)) + + remote_parent = self._synapse_client.get(remote_entity.get('parentId')) + self._set_synapse_parent(remote_parent) + self._upload_file_to_synapse(self._local_path, remote_parent) + else: + if self._remote_path: + logging.info('Uploading to: {0}'.format(self._remote_path)) + + remote_parent = remote_entity + + # Create the remote_path if specified. + if self._remote_path: + full_path = '' + for folder in filter(None, self._remote_path.split(os.sep)): + full_path = os.path.join(full_path, folder) + remote_parent = self._create_folder_in_synapse(full_path, remote_parent) + + with concurrent.futures.ThreadPoolExecutor(max_workers=self._max_threads) as executor: + self._upload_folder(executor, self._local_path, remote_parent) + + self.end_time = datetime.now() + logging.info('') + logging.info('Run time: {0}'.format(self.end_time - self.start_time)) + + if self.has_errors: + logging.error('Finished with errors. Please see log file.') + else: + logging.info('Finished successfully.') + + def _synapse_login(self): + if self._synapse_client and self._synapse_client.credentials: + logging.info('Already logged into Synapse.') + else: + self._username = self._username or os.getenv('SYNAPSE_USERNAME') + self._password = self._password or os.getenv('SYNAPSE_PASSWORD') + + if not self._username: + self._username = input('Synapse username: ') + + if not self._password: + self._password = getpass.getpass(prompt='Synapse password: ') + + logging.info('Logging into Synapse as: {0}'.format(self._username)) + try: + self._synapse_client = syn.Synapse(skip_checks=True) + self._synapse_client.login(self._username, self._password, silent=True) + except Exception as ex: + self._synapse_client = None + self.has_errors = True + logging.error('Synapse login failed: {0}'.format(str(ex))) + + return self._synapse_client is not None + + def _upload_folder(self, executor, local_path, synapse_parent): + if not synapse_parent: + self.has_errors = True + logging.error('Parent not found, cannot execute folder: {0}'.format(local_path)) + return + + parent = synapse_parent + + dirs, files = self._get_dirs_and_files(local_path) + + child_count = 0 + + # Upload the files + for file_entry in files: + if (child_count + 1) >= self._max_depth: + parent = self._create_folder_in_synapse('more', parent) + child_count = 0 + + executor.submit(self._upload_file_to_synapse, file_entry.path, parent) + child_count += 1 + + # Upload the directories. + for dir_entry in dirs: + if (child_count + 1) >= self._max_depth: + parent = self._create_folder_in_synapse('more', parent) + child_count = 0 + + syn_dir = self._create_folder_in_synapse(dir_entry.path, parent) + self._upload_folder(executor, dir_entry.path, syn_dir) + child_count += 1 + + def _create_folder_in_synapse(self, path, synapse_parent): + synapse_folder = None + + if not synapse_parent: + self.has_errors = True + logging.error('Parent not found, cannot create folder: {0}'.format(path)) + return synapse_folder + + folder_name = os.path.basename(path) + full_synapse_path = self._get_synapse_path(folder_name, synapse_parent) + + max_attempts = 5 + attempt_number = 0 + exception = None + + while attempt_number < max_attempts and not synapse_folder: + try: + attempt_number += 1 + exception = None + synapse_folder = self._synapse_client.store(syn.Folder(name=folder_name, parent=synapse_parent), + forceVersion=False) + except Exception as ex: + exception = ex + logging.error('[Folder ERROR] {0} -> {1} : {2}'.format(path, full_synapse_path, str(ex))) + if attempt_number < max_attempts: + sleep_time = random.randint(1, 5) + logging.info('[Folder RETRY in {0}s] {1} -> {2}'.format(sleep_time, path, full_synapse_path)) + time.sleep(sleep_time) + + if exception: + self.has_errors = True + logging.error('[Folder FAILED] {0} -> {1} : {2}'.format(path, full_synapse_path, str(exception))) + else: + logging.info('[Folder] {0} -> {1}'.format(path, full_synapse_path)) + self._set_synapse_parent(synapse_folder) + + return synapse_folder + + def _upload_file_to_synapse(self, local_file, synapse_parent): + synapse_file = None + + if not synapse_parent: + self.has_errors = True + logging.error('Parent not found, cannot execute file: {0}'.format(local_file)) + return synapse_file + + # Skip empty files since these will error when uploading via the synapseclient. + if os.path.getsize(local_file) < 1: + logging.info('Skipping empty file: {0}'.format(local_file)) + return synapse_file + + file_name = os.path.basename(local_file) + full_synapse_path = self._get_synapse_path(file_name, synapse_parent) + + max_attempts = 5 + attempt_number = 0 + exception = None + + while attempt_number < max_attempts and not synapse_file: + try: + attempt_number += 1 + exception = None + synapse_file = self._synapse_client.store( + syn.File(path=local_file, name=file_name, parent=synapse_parent), + forceVersion=False) + except Exception as ex: + exception = ex + logging.error('[File ERROR] {0} -> {1} : {2}'.format(local_file, full_synapse_path, str(ex))) + if attempt_number < max_attempts: + sleep_time = random.randint(1, 5) + logging.info('[File RETRY in {0}s] {1} -> {2}'.format(sleep_time, local_file, full_synapse_path)) + time.sleep(sleep_time) + + if exception: + self.has_errors = True + logging.error('[File FAILED] {0} -> {1} : {2}'.format(local_file, full_synapse_path, str(exception))) + else: + logging.info('[File] {0} -> {1}'.format(local_file, full_synapse_path)) + + return synapse_file + + def _set_synapse_parent(self, parent): + with self._thread_lock: + self._synapse_parents[parent.id] = parent + + def _get_synapse_parent(self, parent_id): + with self._thread_lock: + return self._synapse_parents.get(parent_id, None) + + def _get_synapse_path(self, folder_or_file_name, parent): + segments = [] + + if isinstance(parent, syn.Project): + segments.insert(0, parent.name) + else: + next_parent = parent + while next_parent: + segments.insert(0, next_parent.name) + next_parent = self._get_synapse_parent(next_parent.parentId) + + segments.append(folder_or_file_name) + + return os.path.join(*segments) + + def _get_dirs_and_files(self, local_path): + dirs = [] + files = [] + + with os.scandir(local_path) as iter: + for entry in iter: + if entry.is_dir(follow_symlinks=False): + dirs.append(entry) + else: + files.append(entry) + + dirs.sort(key=lambda f: f.name) + files.sort(key=lambda f: f.name) + + return dirs, files diff --git a/src/synapse_uploader/utils.py b/src/synapse_uploader/utils.py new file mode 100644 index 0000000..a18a5d9 --- /dev/null +++ b/src/synapse_uploader/utils.py @@ -0,0 +1,42 @@ +import os +import pathlib + + +class Utils: + + @staticmethod + def app_dir(): + """Gets the application's primary directory for the current user. + + Returns: + Absolute path to the directory. + """ + return os.path.join(pathlib.Path.home(), '.syntools') + + @staticmethod + def app_log_dir(): + """Gets the applications primary log directory for the current user. + + Returns: + Absolute path to the directory. + """ + return os.path.join(Utils.app_dir(), 'logs') + + @staticmethod + def expand_path(local_path): + var_path = os.path.expandvars(local_path) + expanded_path = os.path.expanduser(var_path) + return os.path.abspath(expanded_path) + + @staticmethod + def ensure_dirs(local_path): + """Ensures the directories in local_path exist. + + Args: + local_path: The local path to ensure. + + Returns: + None + """ + if not os.path.isdir(local_path): + os.makedirs(local_path) diff --git a/synapse_uploader.py b/synapse_uploader.py deleted file mode 100755 index 243ee53..0000000 --- a/synapse_uploader.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2017-present, Bill & Melinda Gates Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os -import argparse -import getpass -import time -import random -import concurrent.futures -import threading -import logging -import synapseclient -from synapseclient import Project, Folder, File - - -class SynapseUploader: - - # Maximum number of files per Project/Folder in Synapse. - MAX_SYNAPSE_DEPTH = 10000 - - # Minimum depth for Projects/Folders in Synapse. - MIN_SYNAPSE_DEPTH = 2 - - def __init__(self, synapse_project, local_path, remote_path=None, max_depth=MAX_SYNAPSE_DEPTH, username=None, password=None): - self._synapse_project = synapse_project - self._local_path = local_path.rstrip(os.sep) - self._remote_path = None - self._max_depth = max_depth - self._synapse_parents = {} - self._username = username - self._password = password - self._hasErrors = False - self._thread_lock = threading.Lock() - - if max_depth < self.MIN_SYNAPSE_DEPTH: - raise Exception('Maximum depth must be greater than {0}.'.format( - self.MIN_SYNAPSE_DEPTH - 1)) - - if remote_path != None and len(remote_path.strip()) > 0: - self._remote_path = remote_path.strip().lstrip(os.sep).rstrip(os.sep) - if len(self._remote_path) == 0: - self._remote_path = None - - def start(self): - logging.info('Uploading to Project: {0}'.format(self._synapse_project)) - logging.info('Uploading Directory: {0}'.format(self._local_path)) - - if self._remote_path != None: - logging.info('Uploading To: {0}'.format(self._remote_path)) - - self.login() - - project = self._synapse_client.get(Project(id=self._synapse_project)) - self.set_synapse_parent(project) - - parent = project - - # Create the remote_path if specified. - if self._remote_path: - full_path = '' - for folder in filter(None, self._remote_path.split(os.sep)): - full_path = os.path.join(full_path, folder) - parent = self.create_directory_in_synapse(full_path, parent) - - self.upload_folder(self._local_path, parent) - - completion_status = 'With Errors' if self._hasErrors else 'Successfully' - - logging.info('Upload Completed {0}'.format(completion_status)) - - def login(self): - logging.info('Logging into Synapse...') - syn_user = self._username or os.getenv('SYNAPSE_USER') - syn_pass = self._password or os.getenv('SYNAPSE_PASSWORD') - - if syn_user == None: - syn_user = input('Synapse username: ') - - if syn_pass == None: - syn_pass = getpass.getpass(prompt='Synapse password: ') - - self._synapse_client = synapseclient.Synapse() - self._synapse_client.login(syn_user, syn_pass, silent=True) - - def set_synapse_parent(self, parent): - with self._thread_lock: - self._synapse_parents[parent.id] = parent - - def get_synapse_parent(self, parent_id): - with self._thread_lock: - return self._synapse_parents.get(parent_id, None) - - def get_synapse_path(self, folder_or_file_name, parent): - segments = [] - - if isinstance(parent, Project): - segments.insert(0, parent.name) - else: - next_parent = parent - while next_parent: - if next_parent: - segments.insert(0, next_parent.name) - next_parent = self.get_synapse_parent(next_parent.parentId) - - segments.append(folder_or_file_name) - - return os.path.join(*segments) - - def get_dirs_and_files(self, local_path): - dirs = [] - files = [] - - with os.scandir(local_path) as iter: - for entry in iter: - if entry.is_dir(follow_symlinks=False): - dirs.append(entry) - elif entry.is_file(follow_symlinks=False): - files.append(entry) - - dirs.sort(key=lambda f: f.name) - files.sort(key=lambda f: f.name) - - return dirs, files - - def upload_folder(self, local_path, synapse_parent): - parent = synapse_parent - - dirs, files = self.get_dirs_and_files(local_path) - - child_count = 0 - - with concurrent.futures.ThreadPoolExecutor() as executor: - # Upload the directories. - for dir_entry in dirs: - if (child_count + 1) >= self._max_depth: - parent = self.create_directory_in_synapse('more', parent) - child_count = 0 - - syn_dir = self.create_directory_in_synapse( - dir_entry.path, parent) - executor.submit(self.upload_folder, dir_entry.path, syn_dir) - child_count += 1 - - # Upload the files - for file_entry in files: - if (child_count + 1) >= self._max_depth: - parent = self.create_directory_in_synapse('more', parent) - child_count = 0 - - executor.submit(self.upload_file_to_synapse, - file_entry.path, parent) - child_count += 1 - - def create_directory_in_synapse(self, path, synapse_parent): - if not synapse_parent: - self._hasErrors = True - logging.error( - ' -! Parent not found, cannot create folder: {0}'.format(path)) - return - - folder_name = os.path.basename(path) - full_synapse_path = self.get_synapse_path(folder_name, synapse_parent) - - logging.info( - 'Processing Folder: {0}{1} -> {2}'.format(path, os.linesep, full_synapse_path)) - - synapse_folder = Folder(name=folder_name, parent=synapse_parent) - - max_attempts = 5 - attempt_number = 0 - - while attempt_number < max_attempts and not synapse_folder.get('id', None): - try: - attempt_number += 1 - synapse_folder = self._synapse_client.store( - synapse_folder, forceVersion=False) - except Exception as ex: - logging.error( - ' -! Error creating folder: {0}'.format(str(ex))) - if attempt_number < max_attempts: - sleep_time = random.randint(1, 5) - logging.info( - ' -! Retrying in {0} seconds'.format(sleep_time)) - time.sleep(sleep_time) - - if not synapse_folder.get('id', None): - self._hasErrors = True - logging.error(' -! Failed to create folder: {0}'.format(path)) - else: - if attempt_number > 1: - logging.info(' -> Folder created') - self.set_synapse_parent(synapse_folder) - - return synapse_folder - - def upload_file_to_synapse(self, local_file, synapse_parent): - if not synapse_parent: - self._hasErrors = True - logging.error( - ' -! Parent not found, cannot upload file: {0}'.format(local_file)) - return None - - # Skip empty files since these will error when uploading via the synapseclient. - if (os.path.getsize(local_file) < 1): - logging.info('Skipping Empty File: {0}'.format(local_file)) - return None - - file_name = os.path.basename(local_file) - full_synapse_path = self.get_synapse_path(file_name, synapse_parent) - - logging.info( - 'Processing File: {0}{1} -> {2}'.format(local_file, os.linesep, full_synapse_path)) - - synapse_file = File(path=local_file, parent=synapse_parent) - - max_attempts = 5 - attempt_number = 0 - - while attempt_number < max_attempts and not synapse_file.get('id', None): - try: - attempt_number += 1 - synapse_file = self._synapse_client.store( - synapse_file, forceVersion=False) - except Exception as ex: - logging.error(' -! Error uploading file: {0}'.format(str(ex))) - if attempt_number < max_attempts: - sleep_time = random.randint(1, 5) - logging.info( - ' -! Retrying in {0} seconds'.format(sleep_time)) - time.sleep(sleep_time) - - if not synapse_file.get('id', None): - self._hasErrors = True - logging.error(' -! Failed to upload file: {0}'.format(local_file)) - else: - if attempt_number > 1: - logging.info(' -> File uploaded') - - return synapse_file - - -class LogFilter(logging.Filter): - FILTERS = [ - '##################################################', - 'Uploading file to Synapse storage', - 'Connection pool is full, discarding connection:' - ] - - def filter(self, record): - for filter in self.FILTERS: - if filter in record.msg: - return False - return True - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('project_id', metavar='project-id', - help='Synapse Project ID to upload to (e.g., syn123456789).') - parser.add_argument('local_folder_path', metavar='local-folder-path', - help='Path of the folder to upload.') - parser.add_argument('-r', '--remote-folder-path', - help='Folder to upload to in Synapse.', default=None) - parser.add_argument('-d', '--depth', help='The maximum number of child folders or files under a Synapse Project/Folder.', - type=int, default=SynapseUploader.MAX_SYNAPSE_DEPTH) - parser.add_argument('-u', '--username', - help='Synapse username.', default=None) - parser.add_argument('-p', '--password', - help='Synapse password.', default=None) - parser.add_argument('-l', '--log-level', - help='Set the logging level.', default='INFO') - - args = parser.parse_args() - - log_level = getattr(logging, args.log_level.upper()) - log_file_name = 'log.txt' - - logging.basicConfig( - filename=log_file_name, - filemode='w', - format='%(asctime)s %(levelname)s: %(message)s', - level=log_level - ) - - # Add console logging. - console = logging.StreamHandler() - console.setLevel(log_level) - console.setFormatter(logging.Formatter('%(message)s')) - logging.getLogger().addHandler(console) - - # Filter logs - filter = LogFilter() - for logger in [logging.getLogger(name) for name in logging.root.manager.loggerDict]: - logger.addFilter(filter) - - SynapseUploader( - args.project_id, - args.local_folder_path, - remote_path=args.remote_folder_path, - max_depth=args.depth, - username=args.username, - password=args.password - ).start() - - -if __name__ == "__main__": - main() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9fb2a25 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,83 @@ +import pytest +import tempfile +import os +import json +import shutil +from tests.synapse_test_helper import SynapseTestHelper + +# Load Environment variables. +module_dir = os.path.dirname(os.path.abspath(__file__)) + +test_env_file = os.path.join(module_dir, 'private.test.env.json') + +if os.path.isfile(test_env_file): + with open(test_env_file) as f: + config = json.load(f).get('test') + + # Validate required properties are present + for prop in ['SYNAPSE_USERNAME', 'SYNAPSE_PASSWORD']: + if not prop in config or not config[prop]: + raise Exception( + 'Property: "{0}" is missing in {1}'.format(prop, test_env_file)) + + for key, value in config.items(): + os.environ[key] = value +else: + print('WARNING: Test environment file not found at: {0}'.format(test_env_file)) + + +@pytest.fixture(scope='session') +def syn_client(syn_test_helper): + return syn_test_helper.client() + + +@pytest.fixture(scope='session') +def syn_test_helper(): + """ + Provides the SynapseTestHelper as a fixture per session. + """ + helper = SynapseTestHelper() + yield helper + helper.dispose() + + +@pytest.fixture(scope='session') +def syn_project(syn_test_helper): + return syn_test_helper.create_project() + + +@pytest.fixture() +def new_syn_test_helper(): + """ + Provides the SynapseTestHelper as a fixture per function. + """ + helper = SynapseTestHelper() + yield helper + helper.dispose() + + +@pytest.fixture() +def new_syn_project(new_syn_test_helper): + return new_syn_test_helper.create_project() + + +@pytest.fixture() +def new_temp_dir(): + path = tempfile.mkdtemp() + yield path + if os.path.isdir(path): + shutil.rmtree(path) + + +@pytest.fixture() +def new_temp_file(syn_test_helper): + """ + Generates a temp file containing the SynapseTestHelper.uniq_name. + """ + fd, tmp_filename = tempfile.mkstemp() + with os.fdopen(fd, 'w') as tmp: + tmp.write(syn_test_helper.uniq_name()) + yield tmp_filename + + if os.path.isfile(tmp_filename): + os.remove(tmp_filename) diff --git a/tests/synapse_test_helper.py b/tests/synapse_test_helper.py new file mode 100644 index 0000000..449595c --- /dev/null +++ b/tests/synapse_test_helper.py @@ -0,0 +1,113 @@ +import os +import uuid +import synapseclient +from synapseclient import Project, Folder, File + + +class SynapseTestHelper: + """ + Test helper for working with Synapse. + """ + _test_id = uuid.uuid4().hex + _trash = [] + _synapse_client = None + + def client(self): + if not self._synapse_client: + syn_user = os.getenv('SYNAPSE_USERNAME') + syn_pass = os.getenv('SYNAPSE_PASSWORD') + + self._synapse_client = synapseclient.Synapse() + self._synapse_client.login(syn_user, syn_pass, silent=True) + + return self._synapse_client + + def test_id(self): + """ + Gets a unique value to use as a test identifier. + This string can be used to help identify the test instance that created the object. + """ + return self._test_id + + def uniq_name(self, prefix='', postfix=''): + return "{0}{1}_{2}{3}".format(prefix, self.test_id(), uuid.uuid4().hex, postfix) + + def dispose_of(self, *syn_objects): + """ + Adds a Synapse object to the list of objects to be deleted. + """ + for syn_object in syn_objects: + if syn_object in self._trash: + continue + self._trash.append(syn_object) + + def dispose(self): + """ + Cleans up any Synapse objects that were created during testing. + This method needs to be manually called after each or all tests are done. + """ + projects = [] + folders = [] + files = [] + others = [] + + for obj in self._trash: + if isinstance(obj, Project): + projects.append(obj) + elif isinstance(obj, Folder): + folders.append(obj) + elif isinstance(obj, File): + files.append(obj) + else: + others.append(obj) + + for syn_obj in files: + try: + self.client().delete(syn_obj) + except: + pass + self._trash.remove(syn_obj) + + for syn_obj in folders: + try: + self.client().delete(syn_obj) + except: + pass + self._trash.remove(syn_obj) + + for syn_obj in projects: + try: + self.client().delete(syn_obj) + except: + pass + self._trash.remove(syn_obj) + + for obj in others: + print('WARNING: Non-Supported object found: {0}'.format(obj)) + self._trash.remove(obj) + + def create_project(self, **kwargs): + """ + Creates a new Project and adds it to the trash queue. + """ + if not 'name' in kwargs: + kwargs['name'] = self.uniq_name(prefix=kwargs.get('prefix', '')) + + kwargs.pop('prefix', None) + + project = self.client().store(Project(**kwargs)) + self.dispose_of(project) + return project + + def create_file(self, **kwargs): + """ + Creates a new File and adds it to the trash queue. + """ + if not 'name' in kwargs: + kwargs['name'] = self.uniq_name(prefix=kwargs.get('prefix', '')) + + kwargs.pop('prefix', None) + + file = self.client().store(File(**kwargs)) + self.dispose_of(file) + return file diff --git a/tests/templates/private.test.env.json b/tests/templates/private.test.env.json new file mode 100644 index 0000000..ed50458 --- /dev/null +++ b/tests/templates/private.test.env.json @@ -0,0 +1,8 @@ +{ + "description": "This file contains environment variables that are used during tests.", + "test": { + "SYNAPSE_USERNAME": "...set to a Synapse username that is suitable for testing...", + "SYNAPSE_PASSWORD": "...set to Synapse password...", + "LOG_LEVEL": "DEBUG" + } +} \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..463223c --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,21 @@ +import src.synapse_uploader.cli as cli +from src.synapse_uploader.synapse_uploader import SynapseUploader + + +def test_cli(mocker): + args = ['', 'syn123', '/tmp', '-r', '10', '-d', '20', '-t', '30', '-u', '40', '-p', '50', '-ll', 'debug'] + mocker.patch('sys.argv', args) + mocker.patch('src.synapse_uploader.synapse_uploader.SynapseUploader.execute', return_value=None) + mock_init = mocker.patch.object(SynapseUploader, '__init__', return_value=None) + + cli.main() + + mock_init.assert_called_once_with( + 'syn123', + '/tmp', + remote_path='10', + max_depth=20, + max_threads=30, + username='40', + password='50' + ) diff --git a/tests/test_synapse_test_helper.py b/tests/test_synapse_test_helper.py new file mode 100644 index 0000000..25c6453 --- /dev/null +++ b/tests/test_synapse_test_helper.py @@ -0,0 +1,88 @@ +import os +import pytest +import synapseclient +from synapseclient import Project, Folder, File + + +def test_test_id(syn_test_helper): + assert syn_test_helper.test_id() == syn_test_helper._test_id + + +def test_uniq_name(syn_test_helper): + assert syn_test_helper.test_id() in syn_test_helper.uniq_name() + + last_name = None + for i in list(range(3)): + uniq_name = syn_test_helper.uniq_name(prefix='aaa-', postfix='-zzz') + assert uniq_name != last_name + assert uniq_name.startswith( + 'aaa-{0}'.format(syn_test_helper.test_id())) + assert uniq_name.endswith('-zzz') + last_name = uniq_name + + +def test_dispose_of(syn_test_helper): + # Add a single object + for obj in [object(), object()]: + syn_test_helper.dispose_of(obj) + assert obj in syn_test_helper._trash + + # Add a list of objects + obj1 = object() + obj2 = object() + syn_test_helper.dispose_of(obj1, obj2) + assert obj1 in syn_test_helper._trash + assert obj2 in syn_test_helper._trash + + # Does not add duplicates + syn_test_helper.dispose_of(obj1, obj2) + assert len(syn_test_helper._trash) == 4 + + +def test_dispose(syn_client, syn_test_helper, new_temp_file): + project = syn_client.store(Project(name=syn_test_helper.uniq_name())) + + folder = syn_client.store( + Folder(name=syn_test_helper.uniq_name(prefix='Folder '), parent=project)) + + file = syn_client.store(File(name=syn_test_helper.uniq_name(prefix='File '), path=new_temp_file, parent=folder)) + + syn_objects = [project, folder, file] + + for syn_obj in syn_objects: + syn_test_helper.dispose_of(syn_obj) + assert syn_obj in syn_test_helper._trash + + syn_test_helper.dispose() + assert len(syn_test_helper._trash) == 0 + + for syn_obj in syn_objects: + with pytest.raises(synapseclient.exceptions.SynapseHTTPError) as ex: + syn_client.get(syn_obj, downloadFile=False) + + err_str = str(ex.value) + assert "Not Found" in err_str or "cannot be found" in err_str or "is in trash can" in err_str or "does not exist" in err_str + + try: + os.remove(new_temp_file) + except: + pass + + +def test_create_project(syn_test_helper): + # Uses the name arg + name = syn_test_helper.uniq_name() + project = syn_test_helper.create_project(name=name) + assert project.name == name + assert project in syn_test_helper._trash + syn_test_helper.dispose() + assert project not in syn_test_helper._trash + + # Uses the prefix arg + prefix = '-z-z-z-' + project = syn_test_helper.create_project(prefix=prefix) + assert project.name.startswith(prefix) + + # Generates a name + project = syn_test_helper.create_project() + assert syn_test_helper.test_id() in project.name diff --git a/tests/test_synapse_uploader.py b/tests/test_synapse_uploader.py new file mode 100644 index 0000000..5371f75 --- /dev/null +++ b/tests/test_synapse_uploader.py @@ -0,0 +1,405 @@ +import os +import uuid +import getpass +import pytest +import synapseclient as syn +from src.synapse_uploader.synapse_uploader import SynapseUploader + + +def mkdir(*path_segments): + path = os.path.join(*path_segments) + if not os.path.isdir(path): + os.mkdir(path) + return path + + +def mkfile(*path_segments, content=str(uuid.uuid4())): + path = os.path.join(*path_segments) + with open(path, 'w') as file: + file.write(content) + return path + + +def get_syn_folders(syn_client, syn_parent): + syn_folders = list(syn_client.getChildren(syn_parent, includeTypes=['folder'])) + syn_folder_names = [s['name'] for s in syn_folders] + return syn_folders, syn_folder_names + + +def get_syn_files(syn_client, syn_parent): + syn_files = list(syn_client.getChildren(syn_parent, includeTypes=['file'])) + syn_file_names = [s['name'] for s in syn_files] + return syn_files, syn_file_names + + +def find_by_name(list, name): + return next((x for x in list if x['name'] == name), None) + + +def test_synapse_project_value(): + syn_id = 'syn123' + syn_uploader = SynapseUploader(syn_id, 'None') + assert syn_uploader._synapse_entity_id == syn_id + + +def test_local_path_value(): + local_path = os.getcwd() + syn_uploader = SynapseUploader('None', local_path) + assert syn_uploader._local_path == local_path + + +def test_remote_path_value(): + path_segments = ['one', 'two', 'three'] + remote_path = os.sep.join(path_segments) + + syn_uploader = SynapseUploader('None', 'None', remote_path=remote_path) + assert syn_uploader._remote_path == remote_path + + # Strips spaces and separators + syn_uploader = SynapseUploader('None', 'None', remote_path='{0} {0}'.format(os.sep)) + assert syn_uploader._remote_path is None + + syn_uploader = SynapseUploader('None', 'None', remote_path='{0} one {0}'.format(os.sep)) + assert syn_uploader._remote_path == 'one' + + syn_uploader = SynapseUploader('None', 'None', remote_path='{0} one {0} two {0} three {0}'.format(os.sep)) + assert syn_uploader._remote_path == remote_path + + +def test_max_depth_value(): + max_depth = 10 + syn_uploader = SynapseUploader('None', 'None', max_depth=max_depth) + assert syn_uploader._max_depth == max_depth + + with pytest.raises(Exception) as ex: + SynapseUploader('None', 'None', max_depth=(SynapseUploader.MAX_SYNAPSE_DEPTH + 1)) + assert str(ex.value) == 'Maximum depth must be less than or equal to 10000.' + + +def test_min_depth_value(): + with pytest.raises(Exception) as ex: + SynapseUploader('None', 'None', max_depth=(SynapseUploader.MIN_SYNAPSE_DEPTH - 1)) + assert str(ex.value) == 'Maximum depth must be greater than or equal to 2.' + + +def test_username_value(): + username = 'test_user' + syn_uploader = SynapseUploader('None', 'None', username=username) + assert syn_uploader._username == username + + +def test_password_value(): + password = 'test_password' + syn_uploader = SynapseUploader('None', 'None', password=password) + assert syn_uploader._password == password + + +def test_synapse_client_value(): + client = object() + syn_uploader = SynapseUploader('None', 'None', synapse_client=client) + assert syn_uploader._synapse_client == client + + +def test_login(syn_client, monkeypatch, mocker): + # Uses ENV + syn_uploader = SynapseUploader('None', 'None') + syn_uploader._synapse_login() is True + assert syn_uploader._synapse_client is not None + + # Uses the passed in params + syn_uploader = SynapseUploader('None', 'None', + username=os.environ['SYNAPSE_USERNAME'], password=os.environ['SYNAPSE_PASSWORD']) + assert syn_uploader._synapse_login() is True + assert syn_uploader._synapse_client is not None + + # Uses the passed in client + syn_uploader = SynapseUploader('None', 'None', synapse_client=syn_client) + syn_uploader._synapse_login() is True + assert syn_uploader._synapse_client == syn_client + + # Fails to _synapse_login + syn_uploader = SynapseUploader('None', 'None', username=uuid.uuid4(), password=uuid.uuid4()) + assert syn_uploader._synapse_login() is False + assert syn_uploader._synapse_client is None + + # Prompts for the username and password + with monkeypatch.context() as mp: + mp.delenv('SYNAPSE_USERNAME') + mp.delenv('SYNAPSE_PASSWORD') + + mock_username = uuid.uuid4() + mock_password = uuid.uuid4() + + mocker.patch('builtins.input', return_value=mock_username) + mocker.patch('getpass.getpass', return_value=mock_password) + syn_uploader = SynapseUploader('None', 'None') + syn_uploader._synapse_login() + assert syn_uploader._username == mock_username + assert syn_uploader._password == mock_password + input.assert_called_once() + getpass.getpass.assert_called_once() + + +def test_upload_bad_credentials(): + syn_uploader = SynapseUploader('None', 'None', username=uuid.uuid4(), password=uuid.uuid4()) + syn_uploader.execute() + assert syn_uploader._synapse_client is None + + +def test_upload_remote_path(syn_client, new_syn_project, new_temp_dir): + """ + Tests this scenario: + + Remote Path: one/two/three + + file1 + folder1/ + file2 + folder2/ + file3 + """ + path_segments = ['one', 'two', 'three'] + remote_path = os.path.join(*path_segments) + + mkfile(new_temp_dir, 'file1') + folder1 = mkdir(new_temp_dir, 'folder1') + mkfile(folder1, 'file2') + folder2 = mkdir(folder1, 'folder2') + mkfile(folder2, 'file3') + + SynapseUploader(new_syn_project.id, new_temp_dir, remote_path=remote_path, synapse_client=syn_client).execute() + + parent = new_syn_project + for segment in path_segments: + syn_files, syn_file_names = get_syn_files(syn_client, parent) + syn_folders, syn_folder_names = get_syn_folders(syn_client, parent) + assert len(syn_files) == 0 + assert len(syn_folders) == 1 + folder = find_by_name(syn_folders, segment) + assert folder + parent = folder + + syn_files, syn_file_names = get_syn_files(syn_client, parent) + syn_folders, syn_folder_names = get_syn_folders(syn_client, parent) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_folder = find_by_name(syn_folders, 'folder1') + assert syn_folder + assert syn_file_names == ['file1'] + assert syn_folder_names == ['folder1'] + + syn_files, syn_file_names = get_syn_files(syn_client, syn_folder) + syn_folders, syn_folder_names = get_syn_folders(syn_client, syn_folder) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_folder = find_by_name(syn_folders, 'folder2') + assert syn_folder + assert syn_file_names == ['file2'] + assert syn_folder_names == ['folder2'] + + syn_files, syn_file_names = get_syn_files(syn_client, syn_folder) + syn_folders, _ = get_syn_folders(syn_client, syn_folder) + assert len(syn_files) == 1 + assert len(syn_folders) == 0 + assert syn_file_names == ['file3'] + + +def test_upload(syn_client, syn_test_helper, new_temp_dir): + """ + Tests this scenario: + + file1 + file2 + file3 + folder1/ + file4 + folder2/ + file5 + folder3/ + file6 + """ + for i in range(1, 4): + mkfile(new_temp_dir, 'file{0}'.format(i)) + + folder1 = mkdir(new_temp_dir, 'folder1') + mkfile(folder1, 'file4') + folder2 = mkdir(folder1, 'folder2') + mkfile(folder2, 'file5') + folder3 = mkdir(folder2, 'folder3') + mkfile(folder3, 'file6') + mkfile(folder3, 'file7', content='') # Empty files should NOT get uploaded. + + project1 = syn_test_helper.create_project() + project2 = syn_test_helper.create_project() + + # Test uploading to a Project and Folder + upload_targets = [project1, + syn_client.store(syn.Folder(name=syn_test_helper.uniq_name(), parent=project2))] + + for upload_target in upload_targets: + SynapseUploader(upload_target.id, new_temp_dir, synapse_client=syn_client).execute() + + syn_files, syn_file_names = get_syn_files(syn_client, upload_target) + syn_folders, _ = get_syn_folders(syn_client, upload_target) + assert len(syn_files) == 3 + assert len(syn_folders) == 1 + syn_folder = find_by_name(syn_folders, 'folder1') + assert syn_folder + assert syn_file_names == ['file1', 'file2', 'file3'] + + syn_files, _ = get_syn_files(syn_client, syn_folders[-1]) + syn_folders, _ = get_syn_folders(syn_client, syn_folders[-1]) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_file = find_by_name(syn_files, 'file4') + syn_folder = find_by_name(syn_folders, 'folder2') + assert syn_file + assert syn_folder + + syn_files, _ = get_syn_files(syn_client, syn_folders[-1]) + syn_folders, _ = get_syn_folders(syn_client, syn_folders[-1]) + assert len(syn_files) == 1 + assert len(syn_folders) == 1 + syn_file = find_by_name(syn_files, 'file5') + syn_folder = find_by_name(syn_folders, 'folder3') + assert syn_file + assert syn_folder + + syn_files, _ = get_syn_files(syn_client, syn_folders[-1]) + syn_folders, _ = get_syn_folders(syn_client, syn_folders[-1]) + assert len(syn_files) == 1 + assert len(syn_folders) == 0 + syn_file = find_by_name(syn_files, 'file6') + assert syn_file + + +def test_upload_file(syn_client, syn_test_helper, new_syn_project, new_temp_file, new_temp_dir): + file_name = os.path.basename(new_temp_file) + syn_file = syn_test_helper.create_file(name=file_name, path=new_temp_file, parent=new_syn_project) + + SynapseUploader(syn_file.id, new_temp_file, synapse_client=syn_client).execute() + + syn_files, syn_file_names = get_syn_files(syn_client, new_syn_project) + syn_folders, _ = get_syn_folders(syn_client, new_syn_project) + assert len(syn_files) == 1 + assert len(syn_folders) == 0 + assert file_name in syn_file_names + + # Test exceptions + with pytest.raises(Exception) as ex: + SynapseUploader(syn_file.id, new_temp_dir, synapse_client=syn_client).execute() + assert 'Local entity must be a file when remote entity is a file:' in str(ex.value) + + with pytest.raises(Exception) as ex: + SynapseUploader(syn_file.id, new_temp_file, remote_path='/test', synapse_client=syn_client).execute() + assert 'Cannot specify a remote path when remote entity is a file:' in str(ex.value) + + # Local filename: {0} does not match remote file name: + other_temp_file = mkfile(new_temp_dir, syn_test_helper.uniq_name()) + other_temp_file_name = os.path.basename(other_temp_file) + other_syn_file = syn_test_helper.create_file(name=other_temp_file_name, + path=other_temp_file, + parent=new_syn_project) + with pytest.raises(Exception) as ex: + SynapseUploader(syn_file.id, other_temp_file, synapse_client=syn_client).execute() + assert 'Local filename: {0} does not match remote file name:'.format(other_temp_file_name) in str(ex.value) + + +def test_upload_max_depth(syn_client, new_syn_project, new_temp_dir): + """ + Tests this scenario: + + file1 + file2 + file3 + file4 + file5 + folder1/ + file1-1 + file1-2 + folder2 + folder3 + folder4 + folder5 + + TO: + + file1 + file2 + more/ + file3 + file4 + more/ + file5 + folder1/ + file1-1 + file1-2 + more/ + folder2 + folder3 + more/ + folder4 + folder5 + """ + for i in range(1, 6): + mkfile(new_temp_dir, 'file{0}'.format(i)) + folder_path = mkdir(new_temp_dir, 'folder{0}'.format(i)) + if i == 1: + mkfile(folder_path, 'file1-1'.format(i)) + mkfile(folder_path, 'file1-2'.format(i)) + + SynapseUploader(new_syn_project.id, new_temp_dir, max_depth=3, synapse_client=syn_client).execute() + + syn_files, syn_file_names = get_syn_files(syn_client, new_syn_project) + assert len(syn_files) == 2 + assert syn_file_names == ['file1', 'file2'] + syn_folders, syn_folder_names = get_syn_folders(syn_client, new_syn_project) + assert len(syn_folders) == 1 + assert syn_folder_names == ['more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_files, syn_file_names = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 2 + assert syn_file_names == ['file3', 'file4'] + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 1 + assert syn_folder_names == ['more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_files, syn_file_names = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 1 + assert syn_file_names == ['file5'] + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 2 + assert syn_folder_names == ['folder1', 'more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_folder1 = find_by_name(syn_folders, 'folder1') + + child_syn_files, child_syn_file_names = get_syn_files(syn_client, syn_folder1) + assert len(child_syn_files) == 2 + assert child_syn_file_names == ['file1-1', 'file1-2'] + child_syn_folders, _ = get_syn_folders(syn_client, syn_folder1) + assert len(child_syn_folders) == 0 + + syn_files, _ = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 0 + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 3 + assert syn_folder_names == ['folder2', 'folder3', 'more'] + + more_folder = find_by_name(syn_folders, 'more') + + syn_files, _ = get_syn_files(syn_client, more_folder) + assert len(syn_files) == 0 + syn_folders, syn_folder_names = get_syn_folders(syn_client, more_folder) + assert len(syn_folders) == 2 + assert syn_folder_names == ['folder4', 'folder5'] + + +def test_upload_failures(): + # TODO: add tests. + pass