Version 0.0.6

.env-template

@@ -0,0 +1,2 @@
+SYNAPSE_AUTH_TOKEN=
+LOG_LEVEL=DEBUG

.gitignore

@@ -19,8 +19,10 @@ var/
 
 .idea
 .venv
+.env
 .vscode
 .coverage
+.tox
 htmlcov
 tests/private.*
 *.log

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## Version 0.0.6 (2023-10-11)
+
+### Changes
+
+- Only support Python 3.10+.
+- Added Synapsis.
+
 ## Version 0.0.5 (2021-07-28)
 
 ### Added

Pipfile

@@ -5,7 +5,7 @@ verify_ssl = true
 
 [dev-packages]
 pytest = "*"
-pylint = "*"
+pylint = "<3.0.0"
 pytest-mock = "*"
 pytest-pylint = "*"
 pytest-cov = "*"
@@ -14,9 +14,12 @@ coveralls = "*"
 twine = "*"
 wheel = "*"
 autopep8 = "*"
+synapse_test_helper = ">=0.0.3"
+python-dotenv = "*"
 
 [packages]
-synapseclient = "*"
+synapseclient = ">=2.3.1,<3.0.0"
+synapsis = ">=0.0.6"
 
 [requires]
-python_version = "3.7"
+python_version = "3.11"

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+pythonpath = src
+testpaths =
+    tests
+

setup.py

@@ -18,8 +18,8 @@
     packages=setuptools.find_packages(where="src"),
     classifiers=(
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",
     ),
@@ -29,6 +29,7 @@
         ]
     },
     install_requires=[
-        "synapseclient>=2.1.0,<3.0.0"
+        "synapseclient>=2.3.1,<3.0.0",
+        "synapsis>=0.0.6"
     ]
 )

src/synapse_uploader/_version.py

@@ -1 +1 @@
-__version__ = '0.0.5'
+__version__ = '0.0.6'

src/synapse_uploader/cli.py

@@ -6,6 +6,7 @@
 from ._version import __version__
 from .synapse_uploader import SynapseUploader
 from .utils import Utils
+from synapsis import cli as synapsis_cli
 
 
 class LogFilter(logging.Filter):
@@ -24,6 +25,7 @@ def filter(self, record):
 
 def main():
     parser = argparse.ArgumentParser()
+    synapsis_cli.inject(parser)
     parser.add_argument('--version', action='version', version='%(prog)s {0}'.format(__version__))
     parser.add_argument('entity_id',
                         metavar='entity-id',
@@ -47,14 +49,6 @@ def main():
                         type=int,
                         default=None)
 
-    parser.add_argument('-u', '--username',
-                        help='Synapse username.',
-                        default=None)
-
-    parser.add_argument('-p', '--password',
-                        help='Synapse password.',
-                        default=None)
-
     parser.add_argument('-ll', '--log-level',
                         help='Set the logging level.',
                         default='INFO')
@@ -104,16 +98,18 @@ def main():
     print('Logging output to: {0}'.format(log_filename))
 
     try:
+        cache_dir = args.cache_dir
+        if cache_dir:
+            cache_dir = os.path.join(cache_dir, '.synapseCache')
+
+        synapsis_cli.configure(args, synapse_args={'cache_root_dir': cache_dir, 'multi_threaded': False}, login=True)
         cmd = SynapseUploader(
             args.entity_id,
             args.local_path,
             remote_path=args.remote_folder_path,
             max_depth=args.depth,
             max_threads=args.threads,
-            username=args.username,
-            password=args.password,
-            force_upload=args.force_upload,
-            cache_dir=args.cache_dir
+            force_upload=args.force_upload
         )
         cmd.execute()
         if cmd.errors:

src/synapse_uploader/synapse_uploader.py

@@ -1,5 +1,4 @@
 import os
-import getpass
 import time
 import random
 import concurrent.futures
@@ -9,6 +8,7 @@
 from datetime import datetime
 import synapseclient as syn
 from .utils import Utils
+from synapsis import Synapsis
 
 
 class SynapseUploader:
@@ -24,22 +24,14 @@ def __init__(self,
                  remote_path=None,
                  max_depth=MAX_SYNAPSE_DEPTH,
                  max_threads=None,
-                 username=None,
-                 password=None,
-                 synapse_client=None,
-                 force_upload=False,
-                 cache_dir=None):
+                 force_upload=False):
 
         self._synapse_entity_id = synapse_entity_id
         self._local_path = Utils.expand_path(local_path)
         self._remote_path = remote_path
         self._max_depth = max_depth
         self._max_threads = max_threads
-        self._username = username
-        self._password = password
-        self._synapse_client = synapse_client
         self._force_upload = force_upload
-        self._cache_dir = cache_dir
 
         self.start_time = None
         self.end_time = None
@@ -64,29 +56,18 @@ def execute(self):
             self._show_error('Maximum depth must be greater than or equal to {0}.'.format(self.MIN_SYNAPSE_DEPTH))
             return self
 
-        if not self._synapse_login():
-            self._show_error('Could not log into Synapse. Aborting.')
-            return self
-
         if self._force_upload:
             logging.info('Forcing upload. Entity versions will be incremented.')
 
-        remote_entity = self._synapse_client.get(self._synapse_entity_id, downloadFile=False)
-        remote_entity_is_file = False
-
-        if isinstance(remote_entity, syn.Project):
-            remote_type = 'Project'
-            self._set_synapse_parent(remote_entity)
-        elif isinstance(remote_entity, syn.Folder):
-            remote_type = 'Folder'
-            self._set_synapse_parent(remote_entity)
-        elif isinstance(remote_entity, syn.File):
-            remote_type = 'File'
-            remote_entity_is_file = True
-        else:
+        remote_entity = Synapsis.get(self._synapse_entity_id, downloadFile=False)
+        remote_entity_type = Synapsis.ConcreteTypes.get(remote_entity)
+        if not (remote_entity_type.is_project or remote_entity_type.is_folder or remote_entity_type.is_file):
             self._show_error('Remote entity must be a project, folder, or file. Found {0}'.format(type(remote_entity)))
             return self
 
+        if remote_entity_type.is_project or remote_entity_type.is_folder:
+            self._set_synapse_parent(remote_entity)
+
         local_entity_is_file = False
 
         if os.path.isfile(self._local_path):
@@ -98,26 +79,27 @@ def execute(self):
             self._show_error('Local entity must be a directory or file: {0}'.format(self._local_path))
             return self
 
-        if remote_entity_is_file and not local_entity_is_file:
+        if remote_entity_type.is_file and not local_entity_is_file:
             self._show_error('Local entity must be a file when remote entity is a file: {0}'.format(self._local_path))
             return self
 
-        if remote_entity_is_file and self._remote_path:
+        if remote_entity_type.is_file and self._remote_path:
             self._show_error('Cannot specify a remote path when remote entity is a file: {0}'.format(self._local_path))
             return self
 
-        logging.info('Uploading to {0}: {1} ({2})'.format(remote_type, remote_entity.name, remote_entity.id))
+        logging.info(
+            'Uploading to {0}: {1} ({2})'.format(remote_entity_type.name, remote_entity.name, remote_entity.id))
         logging.info('Uploading {0}: {1}'.format(local_type, self._local_path))
 
-        if remote_entity_is_file:
+        if remote_entity_type.is_file:
             remote_file_name = remote_entity['_file_handle']['fileName']
             local_file_name = os.path.basename(self._local_path)
             if local_file_name != remote_file_name:
                 self._show_error('Local filename: {0} does not match remote file name: {1}'.format(local_file_name,
                                                                                                    remote_file_name))
                 return self
 
-            remote_parent = self._synapse_client.get(remote_entity.get('parentId'))
+            remote_parent = Synapsis.get(remote_entity.get('parentId'))
             self._set_synapse_parent(remote_parent)
             self._upload_file_to_synapse(self._local_path, remote_parent)
         else:
@@ -141,31 +123,6 @@ def execute(self):
         logging.info('Run time: {0}'.format(self.end_time - self.start_time))
         return self
 
-    def _synapse_login(self):
-        if self._synapse_client and self._synapse_client.credentials:
-            logging.info('Already logged into Synapse.')
-        else:
-            self._username = self._username or os.getenv('SYNAPSE_USERNAME')
-            self._password = self._password or os.getenv('SYNAPSE_PASSWORD')
-
-            if not self._username:
-                self._username = input('Synapse username: ')
-
-            if not self._password:
-                self._password = getpass.getpass(prompt='Synapse password: ')
-
-            logging.info('Logging into Synapse as: {0}'.format(self._username))
-            try:
-                self._synapse_client = syn.Synapse(skip_checks=True)
-                if self._cache_dir:
-                    self._synapse_client.cache.cache_root_dir = os.path.join(self._cache_dir, '.synapseCache')
-                self._synapse_client.login(self._username, self._password, silent=True)
-            except Exception as ex:
-                self._synapse_client = None
-                self._show_error('Synapse login failed: {0}'.format(str(ex)))
-
-        return self._synapse_client is not None
-
     def _upload_folder(self, executor, local_path, synapse_parent):
         if not synapse_parent:
             self._show_error('Parent not found, cannot execute folder: {0}'.format(local_path))
@@ -214,8 +171,8 @@ def _create_folder_in_synapse(self, path, synapse_parent):
             try:
                 attempt_number += 1
                 exception = None
-                synapse_folder = self._synapse_client.store(syn.Folder(name=folder_name, parent=synapse_parent),
-                                                            forceVersion=self._force_upload)
+                synapse_folder = Synapsis.store(syn.Folder(name=folder_name, parent=synapse_parent),
+                                                forceVersion=self._force_upload)
             except Exception as ex:
                 exception = ex
                 logging.error('[Folder ERROR] {0} -> {1} : {2}'.format(path, full_synapse_path, str(ex)))
@@ -263,7 +220,7 @@ def _upload_file_to_synapse(self, local_file, synapse_parent):
                 if file_obj:
                     file_obj.path = local_file
                     if self._force_upload:
-                        self._synapse_client.cache.remove(file_obj)
+                        Synapsis.cache.remove(file_obj)
                     else:
                         if file_obj['_file_handle']['contentSize'] == local_file_size and \
                                 file_obj['_file_handle']['contentMd5'] == Utils.get_md5(local_file):
@@ -273,7 +230,7 @@ def _upload_file_to_synapse(self, local_file, synapse_parent):
                     file_obj = syn.File(path=local_file, name=file_name, parent=synapse_parent)
 
                 if needs_upload or self._force_upload:
-                    synapse_file = self._synapse_client.store(file_obj, forceVersion=self._force_upload)
+                    synapse_file = Synapsis.store(file_obj, forceVersion=self._force_upload)
             except Exception as ex:
                 exception = ex
                 logging.error('[File ERROR] {0} -> {1} : {2}'.format(local_file, full_synapse_path, str(ex)))
@@ -295,14 +252,14 @@ def _find_synapse_file(self, synapse_parent_id, local_file_path):
 
         for child in children:
             if child['name'] == os.path.basename(local_file_path):
-                syn_file = self._synapse_client.get(child['id'], downloadFile=False)
+                syn_file = Synapsis.get(child['id'], downloadFile=False)
                 # Synapse can store a file with two names: 1) The entity name 2) the actual filename.
                 # Check that the actual filename matches the local file name to ensure we have the same file.
                 if syn_file['_file_handle']['fileName'] != os.path.basename(local_file_path):
                     for find_child in children:
                         if find_child == child:
                             continue
-                        syn_child = self._synapse_client.get(find_child['id'], downloadFile=False)
+                        syn_child = Synapsis.get(find_child['id'], downloadFile=False)
                         if syn_child['_file_handle']['fileName'] == os.path.basename(local_file_path):
                             return syn_child
                 return syn_file
@@ -314,7 +271,7 @@ def _find_synapse_file(self, synapse_parent_id, local_file_path):
     @functools.lru_cache(maxsize=LRU_MAXSIZE, typed=True)
     def _get_synapse_children(self, synapse_parent_id):
         """Gets the child files metadata for a parent Synapse container."""
-        return list(self._synapse_client.getChildren(synapse_parent_id, includeTypes=["file"]))
+        return list(Synapsis.getChildren(synapse_parent_id, includeTypes=["file"]))
 
     def _set_synapse_parent(self, parent):
         with self._thread_lock: