Flake8 fixes

Author: Matt Bertrand

.travis.yml

@@ -21,9 +21,10 @@ install:
   - pip install -e .
   - git clone -b 2.4.x https://github.com/GeoNode/geonode.git
   - cp local_settings.py geonode/geonode/.
-  - pushd geonode
-  - pip install -e .
+  - pip install -e geonode
 
 script:
-
-  - python manage.py test dataqs
+  - flake8 --config flake8.cfg dataqs
+  - pushd geonode
+  - python manage.py test dataqs
+  - popd

dataqs/airnow/airnow.py

@@ -7,7 +7,7 @@
 import re
 import shutil
 from django.conf import settings
-from dataqs.processor_base import GeoDataMosaicProcessor, DEFAULT_WORKSPACE
+from dataqs.processor_base import GeoDataMosaicProcessor
 from dataqs.helpers import warp_image, style_exists
 
 logger = logging.getLogger("dataqs.processors")
@@ -27,8 +27,9 @@ class AirNowGRIB2HourlyProcessor(GeoDataMosaicProcessor):
     """
     prefix = "airnow"
     base_url = "ftp.airnowapi.org"
-    layer_names = ["airnow_aqi_ozone", "airnow_aqi_pm25",
-                         "airnow_aqi_combined"]
+    layer_names = ["airnow_aqi_ozone",
+                   "airnow_aqi_pm25",
+                   "airnow_aqi_combined"]
     img_patterns = ["", "_pm25", "_combined"]
     layer_titles = ["Ozone", "PM25", "Combined Ozone & PM25"]
 

dataqs/aqicn/aqicn.py

@@ -13,8 +13,8 @@
 import traceback
 from dateutil.parser import parse
 from dateutil.tz import tzutc
-from dataqs.helpers import postgres_query, layer_exists, table_exists, style_exists, \
-    asciier
+from dataqs.helpers import postgres_query, layer_exists, table_exists, \
+    style_exists, asciier
 from dataqs.processor_base import GeoDataProcessor, DEFAULT_WORKSPACE
 from geonode.geoserver.helpers import ogc_server_settings
 
@@ -23,11 +23,11 @@
 script_dir = os.path.dirname(os.path.realpath(__file__))
 
 REQ_HEADER = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-    'User-Agent':
-                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) \
-                AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 '
-                'Safari/537.36'
+    'Accept': 'text/html,application/xhtml+xml,application/xml;'
+              'q=0.9,image/webp,*/*;q=0.8',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) '
+                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 '
+                  'Safari/537.36'
 }
 
 AQICN_SQL = u"""
@@ -79,6 +79,7 @@
 CREATE INDEX {table}_the_geom ON {table} USING gist (the_geom);
 """
 
+
 def thread_parse(table, cities):
     """
     Thread worker for a list of cities
@@ -135,8 +136,7 @@ def handle_city(self, i, city):
                 return
             city['dateTime'] = self.get_time(city)
             city["data"] = {}
-
-            cur_list = soup.find_all("td", {"id" : re.compile('^cur_')})
+            cur_list = soup.find_all("td", {"id": re.compile('^cur_')})
             # Go on to the next city if we don't find anything
             if not cur_list:
                 logger.debug("Nothing found for %s" % city['city'])
@@ -160,7 +160,7 @@ def handle_city(self, i, city):
 
         except KeyboardInterrupt:
             sys.exit()
-        except Exception as e:
+        except Exception:
             logger.error('Error with city {}'.format(city['url']))
             logger.error(traceback.format_exc())
 
@@ -221,8 +221,8 @@ def run(self):
 class AQICNProcessor(GeoDataProcessor):
     prefix = 'aqicn'
     directory = 'output'
-    cities=None
-    countries=None
+    cities = None
+    countries = None
     pool_size = 6
     base_url = 'http://aqicn.org/city/all/'
     layers = {

dataqs/aqicn/tests.py

@@ -30,12 +30,14 @@ def mock_saveData(self, city):
     with open(tmpfile, 'w') as outfile:
         outfile.write(json.dumps(city))
 
+
 def mock_worker_init(self, table, cities):
     self.cities = cities
     self.prefix = table
     self.archive = self.prefix + "_archive"
     self.max_wait = 5
 
+
 class AQICNTest(TestCase):
     """
     Tests the dataqs.aqicn module.  Since each processor is highly

dataqs/csv_helpers.py

@@ -62,4 +62,4 @@ def writerow(self, row):
 
     def writerows(self, rows):
         for row in rows:
-            self.writerow(row)
+            self.writerow(row)

dataqs/forecastio/tests.py

@@ -110,4 +110,3 @@ def test_cleanup(self):
         self.processor.cleanup()
         self.assertEquals([], glob.glob(os.path.join(
             self.processor.tmp_dir, self.processor.prefix + '*')))
-

dataqs/gdacs/tests.py

@@ -1,5 +1,3 @@
-import glob
-import json
 import os
 import datetime
 from django.test import TestCase
@@ -9,6 +7,7 @@
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 
+
 class GdacsTest(TestCase):
     """
     Tests the dataqs.gdacs module.  Since each processor is highly

dataqs/gfms/tests.py

@@ -91,7 +91,7 @@ def test_cleanup(self):
         httpretty.register_uri(httpretty.GET, current_url,
                                body=get_mock_image())
         imgfile = self.processor.download(current_url)
-        tif_file = self.processor.convert(imgfile)
+        self.processor.convert(imgfile)
         self.assertNotEqual([], glob.glob(os.path.join(
             self.processor.tmp_dir, self.processor.prefix + '*')))
         self.processor.cleanup()

dataqs/helpers.py

@@ -3,8 +3,6 @@
 import traceback
 import os
 import subprocess
-
-import functools
 import requests
 from geoserver.catalog import Catalog, FailedRequestError
 import psycopg2
@@ -340,4 +338,4 @@ def __init__(self, content):
         self.status_code = 200
 
     def raise_for_status(self):
-        pass
+        pass

dataqs/nasa_gpm/tests.py

@@ -27,7 +27,8 @@ def mock_nlst(self, *args):
     current_date = datetime.datetime.utcnow()
     start_date = current_date - datetime.timedelta(days=7)
     while start_date < current_date:
-        file = '3B-HHR-E.MS.MRG.3IMERG.{}-S120000-E175959.1050.V03E.1day.tif'.format(
+        f = '3B-HHR-E.MS.MRG.3IMERG.{}-S120000-E175959.1050.V03E.1day.tif'
+        file = f.format(
             current_date.strftime('%Y%m%d'))
         file_list.append(file)
         current_date = current_date - datetime.timedelta(days=1)
@@ -73,10 +74,10 @@ def test_parse_name(self):
         Layer title should contain date of image
         :return:
         """
-        imgfile = '3B-HHR-E.MS.MRG.3IMERG.20151027-S133000-E135959.0810.V03E.1day.tif'
-        title = self.processor.parse_name(imgfile)[0]
+        f = '3B-HHR-E.MS.MRG.3IMERG.20151027-S133000-E135959.0810.V03E.1day.tif'
+        title = self.processor.parse_name(f)[0]
         self.assertTrue('NASA Global Precipitation Estimate (1day) - 2015-10-27'
-                       in title)
+                        in title)
 
     @patch('ftplib.FTP', autospec=True)
     @patch('ftplib.FTP.retrbinary', mock_retrbinary)
@@ -106,9 +107,9 @@ def test_cleanup(self, mock_ftp):
         :return:
         """
         dl_tif = self.processor.download()[0]
-        convert_tif = self.processor.convert(dl_tif)
+        self.processor.convert(dl_tif)
         self.assertNotEqual([], glob.glob(os.path.join(
             self.processor.tmp_dir, self.processor.prefix + '*')))
         self.processor.cleanup()
         self.assertEquals([], glob.glob(os.path.join(
-            self.processor.tmp_dir, self.processor.prefix + '*')))
+            self.processor.tmp_dir, self.processor.prefix + '*')))

dataqs/processor_base.py

@@ -71,8 +71,9 @@ class GeoDataProcessor(object):
     for import into GeoNode/GeoServer
     """
 
-    gs_url = "http://{}:8080/geoserver/rest/workspaces/{}/coveragestores/{}/file.geotiff"
-    gs_vec_url = "http://{}:8080/geoserver/rest/workspaces/{}/datastores/{}/featuretypes"
+    base_url = "http://{}:8080/geoserver/rest/workspaces/"
+    gs_url = base_url + "{}/coveragestores/{}/file.geotiff"
+    gs_vec_url = base_url + "{}/datastores/{}/featuretypes"
     gs_style_url = "http://{}:8080/geoserver/rest/styles/"
 
     def __init__(self, workspace=DEFAULT_WORKSPACE, tmp_dir=GS_TMP_DIR,
@@ -111,17 +112,17 @@ def truncate_gs_cache(self, layer_name):
             ws=self.workspace,
             layer=layer_name
         )
-        truncate_json = json.dumps({'seedRequest':
-                                        {'name': 'geonode:{}'.format(
-                                            layer_name),
-                                         'srs': {'number': 900913},
-                                         'zoomStart': 0,
-                                         'zoomStop': 19,
-                                         'format': 'image/png',
-                                         'type': 'truncate',
-                                         'threadCount': 4
-                                         }
-                                    })
+        truncate_json = json.dumps({
+            'seedRequest': {
+                'name': 'geonode:{}'.format(layer_name),
+                'srs': {'number': 900913},
+                'zoomStart': 0,
+                'zoomStop': 19,
+                'format': 'image/png',
+                'type': 'truncate',
+                'threadCount': 4
+            }
+        })
         res = requests.post(url=gwc_url, data=truncate_json,
                             auth=(_user, _password),
                             headers={"Content-type": "application/json"})
@@ -249,8 +250,8 @@ def set_default_style(self, layer_name, sld_name, sld_content):
         gs_url = self.gs_style_url.format(ogc_server_settings.hostname)
 
         # Create the style
-        data = "<style><name>{name}</name><filename>{name}.sld</filename></style>".format(
-            name=sld_name)
+        s = "<style><name>{name}</name><filename>{name}.sld</filename></style>"
+        data = s.format(name=sld_name)
         _user, _password = ogc_server_settings.credentials
         res = requests.post(url=gs_url,
                             data=data,
@@ -274,7 +275,8 @@ def set_default_style(self, layer_name, sld_name, sld_content):
 
         # Assign to the layer
         layer_typename = "{}%3A{}".format(DEFAULT_WORKSPACE, layer_name)
-        data = '<layer><defaultStyle><name>{}</name></defaultStyle></layer>'.format(
+        s = '<layer><defaultStyle><name>{}</name></defaultStyle></layer>'
+        data = s.format(
             sld_name)
         url = urljoin(gs_url.replace("styles", "layers"), layer_typename)
         logger.debug(url)
@@ -303,11 +305,11 @@ def run(self):
 class GeoDataMosaicProcessor(GeoDataProcessor):
     """
     Processor for handling raster mosaic data stores
-    http://docs.geoserver.org/latest/en/user/tutorials/imagemosaic_timeseries/imagemosaic_timeseries.html
+    http://bit.ly/1oMPIE7
     http://geoserver.geo-solutions.it/multidim/en/rest/index.html
     """
-
-    gs_url = "http://{}:8080/geoserver/rest/workspaces/{}/coveragestores/{}/external.imagemosaic"
+    gs_url = "http://{}:8080/geoserver/rest/workspaces/{}/" \
+             "coveragestores/{}/external.imagemosaic"
     mosaic_url = gs_url.replace('external.imagemosaic',
                                 'coverages/{}/index/granules')
     create_url = gs_url.replace('external.imagemosaic', 'file.imagemosaic')

dataqs/spei/spei.py

@@ -2,7 +2,7 @@
 
 import logging
 import os
-from dataqs.processor_base import GeoDataProcessor, DEFAULT_WORKSPACE
+from dataqs.processor_base import GeoDataProcessor
 from dataqs.helpers import get_band_count, gdal_translate, cdo_invert, \
     nc_convert, style_exists
 

dataqs/spei/tests.py

@@ -52,7 +52,7 @@ def test_cleanup(self):
         imgurl = "{}spei03.nc".format(self.processor.base_url)
         httpretty.register_uri(httpretty.GET, imgurl,
                                body=get_mock_image())
-        dl_tif = self.processor.download(imgurl, 'spei03.tif')
+        self.processor.download(imgurl, 'spei03.tif')
         self.assertNotEqual([], glob.glob(os.path.join(
             self.processor.tmp_dir, self.processor.prefix + '*')))
         self.processor.cleanup()

dataqs/usgs_quakes/usgs_quakes.py

@@ -13,6 +13,7 @@
 logger = logging.getLogger("dataqs.processors")
 script_dir = os.path.dirname(os.path.realpath(__file__))
 
+
 class USGSQuakeProcessor(GeoDataProcessor):
     """
     Class for retrieving and processing the latest earthquake data from USGS.
@@ -21,9 +22,11 @@ class USGSQuakeProcessor(GeoDataProcessor):
     are removed.
     """
     prefix = 'usgs_quakes'
-    tables = ("quakes_weekly", "quakes_monthly", "quakes_yearly", "quakes_archive")
+    tables = ("quakes_weekly", "quakes_monthly",
+              "quakes_yearly", "quakes_archive")
     titles = ("Last 7 Days", "Last 30 Days", "Last 365 Days", "Archive")
-    base_url = "http://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson&starttime={}&endtime={}"
+    base_url = "http://earthquake.usgs.gov/fdsnws/event/1/query?" \
+               "format=geojson&starttime={}&endtime={}"
     params = {}
 
     def __init__(self, *args, **kwargs):
@@ -64,7 +67,7 @@ def run(self, rss_file=None):
         """
         if not rss_file:
             rss = self.download(self.base_url.format(self.params['sdate'],
-                                                  self.params['edate']),
+                                                     self.params['edate']),
                                 filename=self.prefix + '.rss')
             rss_file = os.path.join(self.tmp_dir, rss)
 
@@ -93,11 +96,10 @@ def run(self, rss_file=None):
             datastore = ogc_server_settings.server.get('DATASTORE')
             if not layer_exists(table, datastore, DEFAULT_WORKSPACE):
                 c = connections[datastore].cursor()
+                q = 'ALTER TABLE {tb} ADD CONSTRAINT {tb}_ids UNIQUE (ids);'
                 try:
-                    c.execute(
-                        'ALTER TABLE {tb} ADD CONSTRAINT {tb}_ids UNIQUE (ids);'.
-                            format(tb=table))
-                except:
+                    c.execute(q.format(tb=table))
+                except Exception:
                     c.close()
                 self.post_geoserver_vector(table)
             if not style_exists(table):

dataqs/wqp/tests.py

@@ -39,7 +39,8 @@ def test_download(self):
         """
         for qtype in ('Result', 'Station'):
             url = ('http://www.waterqualitydata.us/{}/search?'.format(qtype) +
-                   'countrycode=US&startDateLo=12-27-2015&startDateHi=01-26-2016' +
+                   'countrycode=US&startDateLo=12-27-2015' +
+                   '&startDateHi=01-26-2016' +
                    '&characteristicName=pH')
             httpretty.register_uri(httpretty.GET, url,
                                    body=get_mock_response(

dataqs/wqp/wqp.py

@@ -5,7 +5,7 @@
 import datetime
 import re
 import requests
-from dataqs.helpers import gdal_translate, postgres_query, ogr2ogr_exec, \
+from dataqs.helpers import postgres_query, ogr2ogr_exec, \
     table_exists, purge_old_data, layer_exists, style_exists
 from dataqs.processor_base import GeoDataProcessor, DEFAULT_WORKSPACE
 import unicodecsv as csv
@@ -55,21 +55,22 @@ def update_station_table(self, csvfile):
         :return: None
         """
         vrt_content = (
-        """<OGRVRTDataSource>
-            <OGRVRTLayer name="{name}">
-                <SrcDataSource>{csv}</SrcDataSource>
-                <GeometryType>wkbPoint</GeometryType>
-                <LayerSRS>WGS84</LayerSRS>
-                <GeometryField encoding="PointFromColumns" x="LongitudeMeasure" y="LatitudeMeasure"/>
-            </OGRVRTLayer>
-        </OGRVRTDataSource>
-        """)
+            """<OGRVRTDataSource>
+                <OGRVRTLayer name="{name}">
+                    <SrcDataSource>{csv}</SrcDataSource>
+                    <GeometryType>wkbPoint</GeometryType>
+                    <LayerSRS>WGS84</LayerSRS>
+                    <GeometryField encoding="PointFromColumns"
+                    x="LongitudeMeasure" y="LatitudeMeasure"/>
+                </OGRVRTLayer>
+            </OGRVRTDataSource>
+            """)
         station_table = self.station_table
         needs_index = not table_exists(station_table)
 
         db = ogc_server_settings.datastore_db
         vrt_file = os.path.join(self.tmp_dir, csvfile.replace('.csv', '.vrt'))
-        csv_name = os.path.basename(csvfile).replace(".csv","")
+        csv_name = os.path.basename(csvfile).replace(".csv", "")
         if not os.path.exists(vrt_file):
             with open(vrt_file, 'w') as vrt:
                 vrt.write(vrt_content.format(