feat: add middleauth+https paths indicate CAVE interface (#106)

* feat: add middleauth+https paths indicate CAVE interface

* fix: missing parameter

* docs: describe what the heck CAVE is

* fix: don't call replace when proto is None

automated_test.py

@@ -564,6 +564,15 @@ def okgoogle(url):
         'a/username2/b/c/d', None, None
       ))
 
+def test_middleauth_path_extraction():
+  import cloudfiles.paths
+  path = cloudfiles.paths.extract('middleauth+https://example.com/wow/cool/')
+  assert path.format == 'precomputed'
+  assert path.protocol == 'middleauth+https'
+  assert path.bucket is None
+  assert path.path == 'wow/cool/'
+  assert path.host == "https://example.com"
+
 @pytest.mark.parametrize("protocol", ('mem', 'file', 's3'))
 def test_access_non_cannonical_minimal_path(s3, protocol):
   from cloudfiles import CloudFiles, exceptions

cloudfiles/cloudfiles.py

@@ -44,7 +44,7 @@
 from .interfaces import (
   FileInterface, HttpInterface, 
   S3Interface, GoogleCloudStorageInterface,
-  MemoryInterface
+  MemoryInterface, CaveInterface,
 )
 
 INTERFACES = {
@@ -54,6 +54,7 @@
   'http': HttpInterface,
   'https': HttpInterface,
   'mem': MemoryInterface,
+  'middleauth+https': CaveInterface,
 }
 for alias in ALIASES:
   INTERFACES[alias] = S3Interface

cloudfiles/interfaces.py

@@ -24,7 +24,12 @@
 from .connectionpools import S3ConnectionPool, GCloudBucketPool, MemoryPool, MEMORY_DATA
 from .exceptions import MD5IntegrityError, CompressionError
 from .lib import mkdir, sip, md5, validate_s3_multipart_etag
-from .secrets import http_credentials, CLOUD_FILES_DIR, CLOUD_FILES_LOCK_DIR
+from .secrets import (
+  http_credentials,
+  cave_credentials,
+  CLOUD_FILES_DIR, 
+  CLOUD_FILES_LOCK_DIR,
+)
 
 COMPRESSION_EXTENSIONS = ('.gz', '.br', '.zstd','.bz2','.xz')
 GZIP_TYPES = (True, 'gzip', 1)
@@ -731,6 +736,9 @@ def __init__(self, path, secrets=None, request_payer=None, **kwargs):
     if secrets and 'user' in secrets and 'password' in secrets:
       self.session.auth = (secrets['user'], secrets['password'])
 
+  def default_headers(self):
+    return {}
+
   def get_path_to_file(self, file_path):
     return posixpath.join(self._path.host, self._path.path, file_path)
 
@@ -749,7 +757,8 @@ def put_file(self, file_path, content, content_type,
   @retry
   def head(self, file_path):
     key = self.get_path_to_file(file_path)
-    with self.session.head(key) as resp:
+    headers = self.default_headers()
+    with self.session.head(key, headers=headers) as resp:
       resp.raise_for_status()
       return resp.headers
 
@@ -761,13 +770,14 @@ def size(self, file_path):
   def get_file(self, file_path, start=None, end=None, part_size=None):
     key = self.get_path_to_file(file_path)
 
+    headers = self.default_headers()
     if start is not None or end is not None:
       start = int(start) if start is not None else 0
       end = int(end - 1) if end is not None else ''
-      headers = { "Range": "bytes={}-{}".format(start, end) }
-      resp = self.session.get(key, headers=headers)
-    else:
-      resp = self.session.get(key)
+      headers["Range"] = f"bytes={start}-{end}"
+
+    resp = self.session.get(key, headers=headers)
+
     if resp.status_code in (404, 403):
       return (None, None, None, None)
     resp.close()
@@ -788,7 +798,8 @@ def get_file(self, file_path, start=None, end=None, part_size=None):
   @retry
   def exists(self, file_path):
     key = self.get_path_to_file(file_path)
-    with self.session.get(key, stream=True) as resp:
+    headers = self.default_headers()
+    with self.session.get(key, stream=True, headers=headers) as resp:
       return resp.ok
 
   def files_exist(self, file_paths):
@@ -805,11 +816,15 @@ def _list_files_google(self, prefix, flat):
     if prefix and prefix[-1] != '/':
       prefix += '/'
 
+    headers = self.default_headers()
+
     @retry
     def request(token):
+      nonlocal headers
       results = self.session.get(
         f"https://storage.googleapis.com/storage/v1/b/{bucket}/o",
         params={ "prefix": prefix, "pageToken": token },
+        headers=headers,
       )
       results.raise_for_status()
       results.close()
@@ -832,12 +847,13 @@ def _list_files_apache(self, prefix, flat):
     baseurl = posixpath.join(self._path.host, self._path.path)
 
     directories = ['']
+    headers = self.default_headers()
 
     while directories:
       directory = directories.pop()
       url = posixpath.join(baseurl, directory)
 
-      resp = requests.get(url)
+      resp = requests.get(url, headers=headers)
       resp.raise_for_status()
 
       if 'text/html' not in resp.headers["Content-Type"]:
@@ -1200,3 +1216,16 @@ def release_connection(self):
     with S3_BUCKET_POOL_LOCK:
       pool = S3_POOL[S3ConnectionPoolParams(service, self._path.bucket, self._request_payer)]
     pool.release_connection(self._conn)
+
+class CaveInterface(HttpInterface):
+  """
+  CAVE is an internal system that powers proofreading 
+  systems in Seung Lab. If you have no idea what this
+  is, don't worry about it.
+  see: https://github.com/CAVEconnectome
+  """
+  def default_headers(self):
+    cred = cave_credentials()
+    return {
+      "Authorization": f"Bearer {cred['token']}",
+    }

cloudfiles/paths.py

@@ -26,7 +26,8 @@
 ALIASES = {}
 BASE_ALLOWED_PROTOCOLS = [ 
   'gs', 'file', 's3', 
-  'http', 'https', 'mem' 
+  'http', 'https', 'mem',
+  'middleauth+https', 'ngauth+https',
 ]
 ALLOWED_PROTOCOLS = list(BASE_ALLOWED_PROTOCOLS)
 ALLOWED_FORMATS = [ 
@@ -69,7 +70,13 @@ def cloudpath_error(cloudpath):
 def mkregexp():
   fmt_capture = r'|'.join(ALLOWED_FORMATS)
   fmt_capture = "(?:(?P<fmt>{})://)".format(fmt_capture)
-  proto_capture = r'|'.join(ALLOWED_PROTOCOLS)
+
+  allowed_protos = [
+    p.replace('+', r'\+')
+    for p in ALLOWED_PROTOCOLS
+  ]
+
+  proto_capture = r'|'.join(allowed_protos)
   proto_capture = "(?:(?P<proto>{})://)".format(proto_capture)
   regexp = "{}?{}?".format(fmt_capture, proto_capture)
   return regexp
@@ -292,8 +299,12 @@ def extract_format_protocol(cloudpath:str, allow_defaults=True) -> tuple:
   proto = m.group('proto')
   endpoint = None
 
-  if proto in ('http', 'https'):
-    cloudpath = proto + "://" + cloudpath
+  tmp_proto = None
+  if proto is not None:
+    tmp_proto = proto.replace("middleauth+", "").replace("ngauth+", "")
+
+  if tmp_proto in ('http', 'https'):
+    cloudpath = tmp_proto + "://" + cloudpath
     parse = urllib.parse.urlparse(cloudpath)
     endpoint = parse.scheme + "://" + parse.netloc
     cloudpath = cloudpath.replace(endpoint, '', 1)

cloudfiles/secrets.py

@@ -137,6 +137,24 @@ def aws_credentials(bucket = '', service = 'aws', skip_files=False):
   AWS_CREDENTIALS_CACHE[service][bucket] = aws_credentials
   return aws_credentials
 
+CAVE_CREDENTIALS = None
+def cave_credentials():
+  global CAVE_CREDENTIALS
+  default_file_path = 'cave-secret.json'
+  path = secretpath(default_file_path)
+
+  if CAVE_CREDENTIALS:
+    return CAVE_CREDENTIALS
+
+  if os.path.exists(path):
+    with open(path, 'rt') as f:
+      CAVE_CREDENTIALS = json.loads(f.read())
+  else:
+    CAVE_CREDENTIALS = None
+
+  return CAVE_CREDENTIALS
+
+
 HTTP_CREDENTIALS = None
 def http_credentials():
   global HTTP_CREDENTIALS