Merge pull request #39 from planetlabs/latest

expose latest file functionality in client

Author: bcavagnolo

datalake/archive.py

@@ -117,6 +117,15 @@ def list(self, what, start=None, end=None, where=None, work_id=None):
             else:
                 break
 
+    def latest(self, what, where, lookback=None):
+        url = self.http_url + '/v0/archive/latest/{}/{}'.format(what, where)
+        params = dict(
+            lookback=lookback,
+        )
+        response = self._requests_get(url, params=params)
+        self._check_http_response(response)
+        return response.json()
+
     @property
     def http_url(self):
         self._http_url = self._http_url or environ.get('DATALAKE_HTTP_URL')
@@ -125,7 +134,7 @@ def http_url(self):
         return self._http_url.rstrip('/')
 
     def _check_http_response(self, response):
-        if response.status_code == 400:
+        if response.status_code in (400, 404):
             err = response.json()
             msg = '{} ({})'.format(err['message'], err['code'])
             raise DatalakeHttpError(msg)

datalake/scripts/cli.py

@@ -297,3 +297,23 @@ def _cat(url):
         f = archive.fetch(url)
         out.write(f.read())
     out.close()
+
+
+@cli.command()
+@click.option('--lookback', type=int)
+@click.option('--format', type=click.Choice(_list_result_formats),
+              default='url')
+@click.argument('what')
+@click.argument('where')
+def latest(**kwargs):
+    _prepare_archive_or_fail()
+    _latest(**kwargs)
+
+
+@clean_up_datalake_errors
+def _latest(**kwargs):
+    format = kwargs.pop('format')
+    what = kwargs.pop('what')
+    where = kwargs.pop('where')
+    result = archive.latest(what, where, **kwargs)
+    _print_list_results([result], format)

test/conftest.py

@@ -20,6 +20,7 @@
 import os
 from click.testing import CliRunner
 import stat
+import responses
 
 from datalake.scripts.cli import cli
 from datalake import Archive
@@ -103,3 +104,12 @@ def maker(metadata=random_metadata, content=b''):
 if crtime_available:
     s = os.stat(crtime)
     crtime_setuid = s.st_mode & stat.S_ISUID and s.st_uid == 0
+
+
+def prepare_response(response, status=200, url=None, **query_params):
+    url = url or 'http://datalake.example.com/v0/archive/files/'
+    if len(query_params):
+        q = ['{}={}'.format(k, query_params[k]) for k in query_params.keys()]
+        url = url + '?' + '&'.join(q)
+    responses.add(responses.GET, url, json=response, status=status,
+                  match_querystring=True)

test/test_latest.py

@@ -0,0 +1,68 @@
+import pytest
+import responses
+from conftest import prepare_response
+from datalake import DatalakeHttpError
+
+
+@responses.activate
+def test_latest(archive, random_metadata):
+    r = {
+        'url': 's3://bucket/file',
+        'metadata': random_metadata,
+    }
+    url = 'http://datalake.example.com/v0/archive/latest/{}/{}'
+    url = url.format(random_metadata['what'], random_metadata['where'])
+    prepare_response(r, url=url)
+    l = archive.latest(random_metadata['what'], random_metadata['where'])
+    assert l['url'] == 's3://bucket/file'
+    assert l['metadata'] == random_metadata
+
+
+@responses.activate
+def test_latest_cli(cli_tester, random_metadata):
+    r = {
+        'url': 's3://bucket/file',
+        'metadata': random_metadata,
+    }
+    url = 'http://datalake.example.com/v0/archive/latest/{}/{}'
+    url = url.format(random_metadata['what'], random_metadata['where'])
+    prepare_response(r, url=url)
+
+    cmd = 'latest {what} {where}'
+    cmd = cmd.format(**random_metadata)
+    output = cli_tester(cmd)
+    assert output == 's3://bucket/file\n'
+
+
+@responses.activate
+def test_latest_with_lookback_cli(cli_tester, random_metadata):
+    r = {
+        'url': 's3://bucket/file',
+        'metadata': random_metadata,
+    }
+    url = 'http://datalake.example.com/v0/archive/latest/{}/{}?lookback=42'
+    url = url.format(random_metadata['what'], random_metadata['where'])
+    prepare_response(r, url=url)
+
+    cmd = 'latest {what} {where} --lookback 42'
+    cmd = cmd.format(**random_metadata)
+    output = cli_tester(cmd)
+    assert output == 's3://bucket/file\n'
+
+
+@responses.activate
+def test_no_such_latest(archive):
+    r = {
+        'message': 'not found',
+        'code': 'NoSuchDatalakeFile',
+    }
+    url = 'http://datalake.example.com/v0/archive/latest/not/here'
+    prepare_response(r, status=404, url=url)
+    with pytest.raises(DatalakeHttpError):
+        archive.latest('not', 'here')
+
+
+@responses.activate
+def test_latest_cli_bad_lookback(cli_tester, random_metadata):
+    cmd = 'latest foo bar --lookback nine'
+    cli_tester(cmd, expected_exit=2)

test/test_list.py

@@ -6,15 +6,7 @@
 from datetime import datetime, timedelta
 from pytz import utc
 import simplejson as json
-
-
-def _prepare_response(response, status=200, url=None, **query_params):
-    url = url or 'http://datalake.example.com/v0/archive/files/'
-    if len(query_params):
-        q = ['{}={}'.format(k, query_params[k]) for k in query_params.keys()]
-        url = url + '?' + '&'.join(q)
-    responses.add(responses.GET, url, json=response, status=status,
-                  match_querystring=True)
+from conftest import prepare_response
 
 
 @responses.activate
@@ -28,9 +20,9 @@ def test_list_one_page(archive, random_metadata):
         ],
         'next': None,
     }
-    _prepare_response(r, what=random_metadata['what'],
-                      start=random_metadata['start'],
-                      end=random_metadata['end'])
+    prepare_response(r, what=random_metadata['what'],
+                     start=random_metadata['start'],
+                     end=random_metadata['end'])
     l = list(archive.list(random_metadata['what'],
                           start=random_metadata['start'],
                           end=random_metadata['end']))
@@ -52,8 +44,8 @@ def test_list_two_pages(archive, random_metadata):
         ],
         'next': 'http://the-next-url/',
     }
-    _prepare_response(r1, what=random_metadata['what'], start=m1['start'],
-                      end=m1['end'])
+    prepare_response(r1, what=random_metadata['what'], start=m1['start'],
+                     end=m1['end'])
 
     m2 = copy(random_metadata)
     m2['id'] = '2'
@@ -66,7 +58,7 @@ def test_list_two_pages(archive, random_metadata):
         ],
         'next': None,
     }
-    _prepare_response(r2, url='http://the-next-url/')
+    prepare_response(r2, url='http://the-next-url/')
     l = list(archive.list(m1['what'],
                           start=random_metadata['start'],
                           end=random_metadata['end']))
@@ -84,7 +76,7 @@ def test_bad_request(archive):
         "code": "NoWorkInterval",
         "message": "You must provide either work_id or start/end"
     }
-    _prepare_response(r, status=400, what='syslog')
+    prepare_response(r, status=400, what='syslog')
 
     with pytest.raises(DatalakeHttpError):
         list(archive.list('syslog'))
@@ -94,7 +86,7 @@ def test_bad_request(archive):
 def test_internal_server_error(archive):
 
     r = 'INTERNAL SERVER ERROR'
-    _prepare_response(r, status=500, what='syslog')
+    prepare_response(r, status=500, what='syslog')
 
     with pytest.raises(DatalakeHttpError):
         list(archive.list('syslog'))
@@ -116,9 +108,9 @@ def tester(start, end):
             'next': None,
         }
 
-        _prepare_response(r, what=random_metadata['what'],
-                          start=random_metadata['start'],
-                          end=random_metadata['end'])
+        prepare_response(r, what=random_metadata['what'],
+                         start=random_metadata['start'],
+                         end=random_metadata['end'])
         l = list(archive.list(random_metadata['what'], start=start, end=end))
         assert len(l) == 1
         assert l[0]['url'] == 's3://bucket/file'
@@ -152,10 +144,10 @@ def test_with_where(archive, random_metadata):
         ],
         'next': None,
     }
-    _prepare_response(r, what=random_metadata['what'],
-                      where=random_metadata['where'],
-                      start=random_metadata['start'],
-                      end=random_metadata['end'])
+    prepare_response(r, what=random_metadata['what'],
+                     where=random_metadata['where'],
+                     start=random_metadata['start'],
+                     end=random_metadata['end'])
     l = list(archive.list(random_metadata['what'],
                           where=random_metadata['where'],
                           start=random_metadata['start'],
@@ -178,8 +170,8 @@ def test_with_work_id(archive, random_metadata):
         ],
         'next': None,
     }
-    _prepare_response(r, what=random_metadata['what'],
-                      work_id=random_metadata['work_id'])
+    prepare_response(r, what=random_metadata['what'],
+                     work_id=random_metadata['work_id'])
     l = list(archive.list(random_metadata['what'],
                           work_id='foo123'))
     assert len(l) == 1
@@ -202,9 +194,9 @@ def test_list_cli_url_format(cli_tester, random_metadata):
         ],
         'next': None,
     }
-    _prepare_response(r, what=random_metadata['what'],
-                      start=random_metadata['start'],
-                      end=random_metadata['end'])
+    prepare_response(r, what=random_metadata['what'],
+                     start=random_metadata['start'],
+                     end=random_metadata['end'])
     cmd = 'list {what} --start={start} --end={end}'
     cmd = cmd.format(**random_metadata)
     output = cli_tester(cmd)
@@ -232,7 +224,7 @@ def test_list_cli_json_format(cli_tester, random_metadata):
         ],
         'next': None,
     }
-    _prepare_response(r, what=m1['what'], work_id=m1['work_id'])
+    prepare_response(r, what=m1['what'], work_id=m1['work_id'])
     cmd = 'list {what} --work-id={work_id} --format=json'
     cmd = cmd.format(**m1)
     output_lines = cli_tester(cmd).rstrip('\n').split('\n')