From 2109f59c34f335afbe6c4a3fbbe8c3ea29206993 Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 16 Jun 2017 20:24:39 +0000 Subject: [PATCH 01/16] Updating places API for statistics endpoints --- descarteslabs/services/places.py | 146 ++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index b1bfd18a..927ed39d 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -18,7 +18,7 @@ from cachetools import TTLCache, cachedmethod from cachetools.keys import hashkey from .service import Service - +from six import string_types class Places(Service): TIMEOUT = (9.5, 120) @@ -48,6 +48,28 @@ def placetypes(self): return r.json() + def random(self, geom='low', placetype=None): + """Get a random location + + geom: string + Resolution for the shape [low (default), medium, high] + + return: geojson + """ + params = {} + + if geom: + params['geom'] = geom + + if placetype: + params['placetype'] = placetype + + r = self.session.get('%s/random' % self.url, params=params) + + if r.status_code != 200: + raise RuntimeError("%s: %s" % (r.status_code, r.text)) + + return r.json() @cachedmethod(operator.attrgetter('cache'), key=partial(hashkey, 'find')) def find(self, path, **kwargs): """Find candidate slugs based on full or partial path. @@ -130,3 +152,125 @@ def prefix(self, slug, output='geojson', placetype=None, geom='low'): params=params, timeout=self.TIMEOUT) return r.json() + + def sources(self): + """Get a list of sources + """ + r = self.session.get('%s/sources' % (self.url), timeout=self.TIMEOUT) + + return r.json() + + def categories(self): + """Get a list of categories + """ + r = self.session.get('%s/categories' % (self.url), timeout=self.TIMEOUT) + + return r.json() + + def metrics(self): + """Get a list of metrics + """ + r = self.session.get('%s/metrics' % (self.url), timeout=self.TIMEOUT) + + return r.json() + + def data(self, slug, source=None, category=None, metric=None, date=None, placetype='county'): + """Get all values for a prefix search and point in time + + :param str slug: Slug identifier (or shape id). + :param str source: Source + :param str category: Category + :param str metric: Metric + :param str date: Date + :param str placetype: Restrict results to a particular place type. + + """ + params = {} + + if source: + params['source'] = source + + if category: + params['category'] = category + + if metric: + params['metric'] = metric + + if date: + params['date'] = date + + if placetype: + params['placetype'] = placetype + + r = self.session.get('%s/data/%s' % (self.url, slug), + params=params, timeout=self.TIMEOUT) + + return r.json() + + def statistics(self, slug, source=None, category=None, metric=None): + """Get a time series for a specific place + + :param str slug: Slug identifier (or shape id). + :param str slug: Slug identifier (or shape id). + :param str source: Source + :param str category: Category + :param str metric: Metric + + """ + params = {} + + if source: + params['source'] = source + + if category: + params['category'] = category + + if metric: + params['metric'] = metric + + r = self.session.get('%s/statistics/%s' % (self.url, slug), + params=params, timeout=self.TIMEOUT) + + return r.json() + + def value(self, slug, source=None, category=None, metric=None, date=None): + """Get point values for a specific place + + :param str slug: Slug identifier (or shape id). + :param list(str) source: Source(s) + :param list(str) category: Category(s) + :param list(str) metric: Metric(s) + :param str date: Date + + + """ + params = {} + + if source: + + if isinstance(source, string_types): + source = [source] + + params['source'] = source + + if category: + + if isinstance(category, string_types): + category = [category] + + params['category'] = category + + if metric: + + if isinstance(metric, string_types): + metric = [metric] + + params['metric'] = metric + + if date: + params['date'] = date + + r = self.session.get('%s/value/%s' % (self.url, slug), + params=params, timeout=self.TIMEOUT) + + return r.json() \ No newline at end of file From 642efe820f18a608573fd19512f9090899ba2b6d Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 16 Jun 2017 21:08:19 +0000 Subject: [PATCH 02/16] Starting with dev endpoint --- descarteslabs/services/places.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 927ed39d..2c4786f2 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -30,7 +30,7 @@ def __init__(self, url=None, token=None, maxsize=10, ttl=600): of the backing service. """ if url is None: - url = os.environ.get("DESCARTESLABS_PLACES_URL", "https://platform-services.descarteslabs.com/waldo/v1") + url = os.environ.get("DESCARTESLABS_PLACES_URL", "https://platform-services.descarteslabs.com/waldo/dev") Service.__init__(self, url, token) self.cache = TTLCache(maxsize, ttl) From 8483d89b7cffbc22c7d3859171c9bfc6d517f91c Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 16 Jun 2017 21:14:20 +0000 Subject: [PATCH 03/16] Fix the tests --- descarteslabs/services/places.py | 4 ++-- descarteslabs/tests/test_places.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 2c4786f2..a9dcee00 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -41,8 +41,8 @@ def placetypes(self): Example:: >>> import descarteslabs as dl >>> dl.places.placetypes() - ['country', 'region', 'district', 'mesoregion', 'microregion', - 'county'] + ['continent', 'country', 'dependency', 'macroregion', 'region', 'district', 'mesoregion', 'microregion', 'county'] + """ r = self.session.get('%s/placetypes' % self.url, timeout=self.TIMEOUT) diff --git a/descarteslabs/tests/test_places.py b/descarteslabs/tests/test_places.py index 4eeb8cd2..1802c3a1 100644 --- a/descarteslabs/tests/test_places.py +++ b/descarteslabs/tests/test_places.py @@ -26,7 +26,7 @@ def setUpClass(cls): def test_placetypes(self): data = self.instance.placetypes() - self.assertEqual(6, len(data)) + self.assertEqual(9, len(data)) def test_find(self): r = self.instance.find('united-states_iowa') From 0bc77b1cf581327f6629d64cd39dc2fa51f6131c Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 16 Jun 2017 21:23:19 +0000 Subject: [PATCH 04/16] Thank you flake8! --- descarteslabs/services/places.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index a9dcee00..2e565823 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -20,6 +20,7 @@ from .service import Service from six import string_types + class Places(Service): TIMEOUT = (9.5, 120) """Places and statistics service https://iam.descarteslabs.com/service/waldo""" @@ -41,8 +42,8 @@ def placetypes(self): Example:: >>> import descarteslabs as dl >>> dl.places.placetypes() - ['continent', 'country', 'dependency', 'macroregion', 'region', 'district', 'mesoregion', 'microregion', 'county'] - + ['continent', 'country', 'dependency', 'macroregion', 'region', + 'district', 'mesoregion', 'microregion', 'county'] """ r = self.session.get('%s/placetypes' % self.url, timeout=self.TIMEOUT) @@ -50,26 +51,27 @@ def placetypes(self): def random(self, geom='low', placetype=None): """Get a random location - + geom: string Resolution for the shape [low (default), medium, high] return: geojson """ params = {} - + if geom: params['geom'] = geom - + if placetype: params['placetype'] = placetype - + r = self.session.get('%s/random' % self.url, params=params) - + if r.status_code != 200: raise RuntimeError("%s: %s" % (r.status_code, r.text)) - return r.json() + return r.json() + @cachedmethod(operator.attrgetter('cache'), key=partial(hashkey, 'find')) def find(self, path, **kwargs): """Find candidate slugs based on full or partial path. @@ -206,7 +208,7 @@ def data(self, slug, source=None, category=None, metric=None, date=None, placety params=params, timeout=self.TIMEOUT) return r.json() - + def statistics(self, slug, source=None, category=None, metric=None): """Get a time series for a specific place @@ -232,7 +234,7 @@ def statistics(self, slug, source=None, category=None, metric=None): params=params, timeout=self.TIMEOUT) return r.json() - + def value(self, slug, source=None, category=None, metric=None, date=None): """Get point values for a specific place @@ -241,8 +243,6 @@ def value(self, slug, source=None, category=None, metric=None, date=None): :param list(str) category: Category(s) :param list(str) metric: Metric(s) :param str date: Date - - """ params = {} @@ -273,4 +273,4 @@ def value(self, slug, source=None, category=None, metric=None, date=None): r = self.session.get('%s/value/%s' % (self.url, slug), params=params, timeout=self.TIMEOUT) - return r.json() \ No newline at end of file + return r.json() From f5b1774e560a3178f413d3ee553275d932f332de Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Thu, 29 Jun 2017 16:08:14 +0000 Subject: [PATCH 05/16] Updating places to v1 --- descarteslabs/services/places.py | 41 +++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 96489d4c..64d54329 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -35,7 +35,7 @@ def __init__(self, url=None, token=None, maxsize=10, ttl=600): Service.__init__(self, url, token) self.cache = TTLCache(maxsize, ttl) - + def placetypes(self): """Get a list of place types. @@ -96,6 +96,45 @@ def find(self, path, **kwargs): return r.json() + def search(self, q, country=None, region=None, placetype=None): + """Search for shapes + + :param q: A query string. + :param str country: Restrict search to a specific country + :param str region: Restrict search to a specific region + :param str placetype: Restrict search to a specific placetype + + :return: list of candidates + + Example:: + >>> import descarteslabs as dl + >>> from pprint import pprint + >>> results = dl.places.search('texas') + >>> pprint(results[0]) + {u'bbox': [-106.645584, 25.837395, -93.508039, 36.50035], + u'id': 85688753, + u'name': u'Texas', + u'placetype': u'region', + u'slug': u'north-america_united-states_texas'} + """ + params = {} + + if q: + params['q'] = q + + if country: + params['country'] = country + + if region: + params['region'] = region + + if placetype: + params['placetype'] = placetype + + r = self.session.get('%s/search' % self.url, params=params, timeout=self.TIMEOUT) + + return r.json() + @cachedmethod(operator.attrgetter('cache'), key=partial(hashkey, 'shape')) def shape(self, slug, output='geojson', geom='low'): """Get the geometry for a specific slug From 6fa59f6cc8a92e9c17051fcfa4efc97ba090ce8a Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Thu, 29 Jun 2017 16:15:08 +0000 Subject: [PATCH 06/16] Now 10 placetypes --- descarteslabs/services/places.py | 2 +- descarteslabs/tests/test_places.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 64d54329..93822a14 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -43,7 +43,7 @@ def placetypes(self): >>> import descarteslabs as dl >>> dl.places.placetypes() ['continent', 'country', 'dependency', 'macroregion', 'region', - 'district', 'mesoregion', 'microregion', 'county'] + 'district', 'mesoregion', 'microregion', 'county', 'locality'] """ r = self.session.get('%s/placetypes' % self.url, timeout=self.TIMEOUT) diff --git a/descarteslabs/tests/test_places.py b/descarteslabs/tests/test_places.py index 1802c3a1..2dd8fe1a 100644 --- a/descarteslabs/tests/test_places.py +++ b/descarteslabs/tests/test_places.py @@ -26,7 +26,7 @@ def setUpClass(cls): def test_placetypes(self): data = self.instance.placetypes() - self.assertEqual(9, len(data)) + self.assertEqual(10, len(data)) def test_find(self): r = self.instance.find('united-states_iowa') From 3540775bce39decd069d6b9802ffb94cd842094a Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Thu, 29 Jun 2017 16:20:43 +0000 Subject: [PATCH 07/16] Updating search doc --- descarteslabs/services/places.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 93822a14..57d6d5ae 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -111,11 +111,11 @@ def search(self, q, country=None, region=None, placetype=None): >>> from pprint import pprint >>> results = dl.places.search('texas') >>> pprint(results[0]) - {u'bbox': [-106.645584, 25.837395, -93.508039, 36.50035], - u'id': 85688753, - u'name': u'Texas', - u'placetype': u'region', - u'slug': u'north-america_united-states_texas'} + {'bbox': [-106.645584, 25.837395, -93.508039, 36.50035], + 'id': 85688753, + 'name': 'Texas', + 'placetype': 'region', + 'slug': 'north-america_united-states_texas'} """ params = {} From dd6dea524becabfaab0ed7e97b26377603dae2b3 Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 7 Jul 2017 00:33:58 +0000 Subject: [PATCH 08/16] Switching to v1 --- descarteslabs/services/places.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 57d6d5ae..22c04fde 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -31,7 +31,7 @@ def __init__(self, url=None, token=None, maxsize=10, ttl=600): of the backing service. """ if url is None: - url = os.environ.get("DESCARTESLABS_PLACES_URL", "https://platform-services.descarteslabs.com/waldo/dev") + url = os.environ.get("DESCARTESLABS_PLACES_URL", "https://platform-services.descarteslabs.com/waldo/v1") Service.__init__(self, url, token) self.cache = TTLCache(maxsize, ttl) From f3f13e8c31abf744f11375a76d802a6ccd6650f0 Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 7 Jul 2017 00:57:02 +0000 Subject: [PATCH 09/16] Remove url from path --- descarteslabs/services/places.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 7c8c3558..a9bbf1c4 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -193,21 +193,21 @@ def prefix(self, slug, output='geojson', placetype=None, geom='low'): def sources(self): """Get a list of sources """ - r = self.session.get('%s/sources' % (self.url), timeout=self.TIMEOUT) + r = self.session.get('/sources', timeout=self.TIMEOUT) return r.json() def categories(self): """Get a list of categories """ - r = self.session.get('%s/categories' % (self.url), timeout=self.TIMEOUT) + r = self.session.get('/categories', timeout=self.TIMEOUT) return r.json() def metrics(self): """Get a list of metrics """ - r = self.session.get('%s/metrics' % (self.url), timeout=self.TIMEOUT) + r = self.session.get('/metrics', timeout=self.TIMEOUT) return r.json() @@ -239,7 +239,7 @@ def data(self, slug, source=None, category=None, metric=None, date=None, placety if placetype: params['placetype'] = placetype - r = self.session.get('%s/data/%s' % (self.url, slug), + r = self.session.get('/data/%s' % (slug), params=params, timeout=self.TIMEOUT) return r.json() @@ -265,7 +265,7 @@ def statistics(self, slug, source=None, category=None, metric=None): if metric: params['metric'] = metric - r = self.session.get('%s/statistics/%s' % (self.url, slug), + r = self.session.get('/statistics/%s' % (slug), params=params, timeout=self.TIMEOUT) return r.json() @@ -305,7 +305,7 @@ def value(self, slug, source=None, category=None, metric=None, date=None): if date: params['date'] = date - r = self.session.get('%s/value/%s' % (self.url, slug), + r = self.session.get('/value/%s' % (slug), params=params, timeout=self.TIMEOUT) return r.json() From 20319356d40dec2c60b85bcc13cdb298ee41d95e Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 7 Jul 2017 01:02:09 +0000 Subject: [PATCH 10/16] ...and search --- descarteslabs/services/places.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index a9bbf1c4..5c11d688 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -129,7 +129,7 @@ def search(self, q, country=None, region=None, placetype=None): if placetype: params['placetype'] = placetype - r = self.session.get('%s/search' % self.url, params=params, timeout=self.TIMEOUT) + r = self.session.get('/search', params=params, timeout=self.TIMEOUT) return r.json() From 11574410762c18c0ff2b23916f7ac6e7d2936ecb Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 7 Jul 2017 01:07:02 +0000 Subject: [PATCH 11/16] flake8 --- descarteslabs/services/places.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 5c11d688..489c7bcc 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -35,7 +35,7 @@ def __init__(self, url=None, token=None, maxsize=10, ttl=600): Service.__init__(self, url, token) self.cache = TTLCache(maxsize, ttl) - + def placetypes(self): """Get a list of place types. @@ -119,7 +119,7 @@ def search(self, q, country=None, region=None, placetype=None): if q: params['q'] = q - + if country: params['country'] = country From eb602f775f78f7bbe0b8b57c059b49eaad697df8 Mon Sep 17 00:00:00 2001 From: Mark Mathis Date: Fri, 7 Jul 2017 01:11:00 +0000 Subject: [PATCH 12/16] Upping timeout --- descarteslabs/services/places.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descarteslabs/services/places.py b/descarteslabs/services/places.py index 489c7bcc..655134f2 100644 --- a/descarteslabs/services/places.py +++ b/descarteslabs/services/places.py @@ -22,7 +22,7 @@ class Places(Service): - TIMEOUT = (9.5, 120) + TIMEOUT = (9.5, 360) """Places and statistics service https://iam.descarteslabs.com/service/waldo""" def __init__(self, url=None, token=None, maxsize=10, ttl=600): From 38a84292b02ae9c412ae58d696ee55698bca1bc1 Mon Sep 17 00:00:00 2001 From: aliasmrchips Date: Thu, 10 Aug 2017 21:16:53 +0000 Subject: [PATCH 13/16] update tests --- descarteslabs/tests/test_places.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/descarteslabs/tests/test_places.py b/descarteslabs/tests/test_places.py index 2dd8fe1a..1aab7665 100644 --- a/descarteslabs/tests/test_places.py +++ b/descarteslabs/tests/test_places.py @@ -44,6 +44,29 @@ def test_prefix(self): r = self.instance.prefix('north-america_united-states_iowa', placetype='district') self.assertEqual(9, len(r['features'])) + def test_sources(self): + r = self.instance.sources() + self.assertIn('nass', r) + + def test_categories(self): + r = self.instance.categories() + self.assertIn('corn', r) + + def test_metrics(self): + r = self.instance.metrics() + self.assertIn('yield', r) + + def test_value(self): + r = self.instance.value('north-america_united-states', source='nass', category='corn', metric='yield') + self.assertEqual(1, len(r)) + + def test_statistics(self): + r = self.instance.statistics('north-america_united-states', source='nass', category='corn', metric='yield') + self.assertEqual(36, len(r)) + + def test_data(self): + r = self.instance.data('north-america_united-states', source='nass', category='corn', metric='yield', date='2015-01-01') + self.assertEqual(1439, len(r)) if __name__ == '__main__': unittest.main() From 6020be7bfedff1ae02169ffeb7abc73f9ffca417 Mon Sep 17 00:00:00 2001 From: aliasmrchips Date: Thu, 10 Aug 2017 21:21:26 +0000 Subject: [PATCH 14/16] flake8 --- descarteslabs/tests/test_places.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/descarteslabs/tests/test_places.py b/descarteslabs/tests/test_places.py index 1aab7665..0a56d9c6 100644 --- a/descarteslabs/tests/test_places.py +++ b/descarteslabs/tests/test_places.py @@ -65,7 +65,8 @@ def test_statistics(self): self.assertEqual(36, len(r)) def test_data(self): - r = self.instance.data('north-america_united-states', source='nass', category='corn', metric='yield', date='2015-01-01') + r = self.instance.data('north-america_united-states', source='nass', category='corn', metric='yield', + date='2015-01-01') self.assertEqual(1439, len(r)) if __name__ == '__main__': From e38f58a402144cb026db29a60d0864aad70daf56 Mon Sep 17 00:00:00 2001 From: aliasmrchips Date: Thu, 10 Aug 2017 21:24:10 +0000 Subject: [PATCH 15/16] flake8 --- descarteslabs/tests/test_places.py | 1 + 1 file changed, 1 insertion(+) diff --git a/descarteslabs/tests/test_places.py b/descarteslabs/tests/test_places.py index 0a56d9c6..c3481024 100644 --- a/descarteslabs/tests/test_places.py +++ b/descarteslabs/tests/test_places.py @@ -69,5 +69,6 @@ def test_data(self): date='2015-01-01') self.assertEqual(1439, len(r)) + if __name__ == '__main__': unittest.main() From ffc57ec584a6313607edab3618a2a4520c9e7411 Mon Sep 17 00:00:00 2001 From: aliasmrchips Date: Thu, 10 Aug 2017 21:40:15 +0000 Subject: [PATCH 16/16] One more test --- descarteslabs/tests/test_places.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/descarteslabs/tests/test_places.py b/descarteslabs/tests/test_places.py index c3481024..f7c97981 100644 --- a/descarteslabs/tests/test_places.py +++ b/descarteslabs/tests/test_places.py @@ -32,6 +32,19 @@ def test_find(self): r = self.instance.find('united-states_iowa') self.assertEqual(1, len(r)) + def test_search(self): + r = self.instance.search('texas') + self.assertEqual(8, len(r)) + + r = self.instance.search('texas', country='united-states') + self.assertEqual(7, len(r)) + + r = self.instance.search('texas', country='united-states', placetype='county') + self.assertEqual(2, len(r)) + + r = self.instance.search('texas', country='united-states', region='oklahoma', placetype='county') + self.assertEqual(1, len(r)) + def test_shape(self): r = self.instance.shape('north-america_united-states_iowa') self.assertEqual(85688713, r['id'])