Add pycodestyle.

shyuep · shyuep · commit c109f28a73cf · 2019-09-09T15:47:23.000-07:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -25,6 +25,7 @@ jobs:
             source activate test-environment
             pip install --quiet -e .
             pytest fireworks
+            pycodestyle fireworks
   py2test:
     working_directory: ~/fireworks
     docker:
diff --git a/.travis.yml b/.travis.yml
@@ -9,6 +9,7 @@ before_script:
   - python setup.py develop
 script:
   - pytest fireworks
+  - pycodestyle fireworks
   - source fireworks/tests/cmd_line_test.sh
 after_success:
   - coveralls
diff --git a/fireworks/core/firework.py b/fireworks/core/firework.py
@@ -51,7 +51,9 @@ class FiretaskBase(defaultdict, FWSerializable):
     You can set parameters of a Firetask like you'd use a dict.
     """
     required_params = None  # list of str of required parameters to check for consistency upon init
-    optional_params = None  # if set to a list of str, only required and optional kwargs are allowed; consistency checked upon init
+
+    # if set to a list of str, only required and optional kwargs are allowed; consistency checked upon init
+    optional_params = None
 
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)
diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py
@@ -119,7 +119,9 @@ def __init__(self, host=None, port=None, name=None, username=None, password=None
                  authsource=None, uri_mode=False):
         """
         Args:
-            host (str): hostname. If uri_mode is True, a MongoDB connection string URI (https://docs.mongodb.com/manual/reference/connection-string/) can be used instead of the remaining options below.
+            host (str): hostname. If uri_mode is True, a MongoDB connection string URI
+                (https://docs.mongodb.com/manual/reference/connection-string/) can be used instead of the remaining
+                options below.
             port (int): port number
             name (str): database name
             username (str)
@@ -133,8 +135,10 @@ def __init__(self, host=None, port=None, name=None, username=None, password=None
             ssl_certfile (str): path to the client certificate to be used for mongodb connection
             ssl_keyfile (str): path to the client private key
             ssl_pem_passphrase (str): passphrase for the client private key
-            authsource (str): authsource parameter for MongoDB authentication; defaults to "name" (i.e., db name) if not set
-            uri_mode (bool): if set True, all Mongo connection parameters occur through a MongoDB URI string (set as the host).
+            authsource (str): authsource parameter for MongoDB authentication; defaults to "name" (i.e., db name) if
+                not set
+            uri_mode (bool): if set True, all Mongo connection parameters occur through a MongoDB URI string (set as
+                the host).
         """
 
         self.host = host if (host or uri_mode) else "localhost"
@@ -292,8 +296,8 @@ def reset(self, password, require_password=True, max_reset_wo_password=25):
             self.m_logger.info('LaunchPad was RESET.')
         elif not require_password:
             raise ValueError("Password check cannot be overridden since the size of DB ({} workflows) "
-                             "is greater than the max_reset_wo_password parameter ({}).".format(
-                self.fireworks.count(), max_reset_wo_password))
+                             "is greater than the max_reset_wo_password parameter ({}).".format(self.fireworks.count(),
+                                                                                                max_reset_wo_password))
         else:
             raise ValueError("Invalid password! Password is today's date: {}".format(m_password))
 
@@ -1612,7 +1616,9 @@ def _steal_launches(self, thief_fw):
                 except NotImplementedError:
                     verified = True  # no dupefinder.verify() implemented, skip verification
 
-                except Exception:  # we want to catch any exceptions from testing an empty dict, which the dupefinder might not be designed for
+                except Exception:
+                    # we want to catch any exceptions from testing an empty dict, which the dupefinder might not be
+                    # designed for
                     pass
 
                 if not verified:
@@ -1710,7 +1716,8 @@ def recover_offline(self, launch_id, ignore_errors=False, print_errors=False):
                     self.ping_launch(launch_id, ptime=ping_dict['ping_time'], checkpoint=checkpoint)
                 else:
                     warnings.warn(
-                        "Unable to find FW_ping.json in {}! State history updated_on might be incorrect, trackers may not update.".format(
+                        "Unable to find FW_ping.json in {}! State history updated_on might be incorrect, trackers "
+                        "may not update.".format(
                             m_launch.launch_dir))
                     m_launch.touch_history(checkpoint=checkpoint)
 
@@ -1728,9 +1735,7 @@ def recover_offline(self, launch_id, ignore_errors=False, print_errors=False):
                         s['created_on'] = reconstitute_dates(offline_data['completed_on'])
                 self.launches.find_one_and_update(
                     {'launch_id': m_launch.launch_id},
-                    {'$set':
-                         {'state_history': m_launch.state_history}
-                     })
+                    {'$set': {'state_history': m_launch.state_history}})
 
                 self.offline_runs.update_one({"launch_id": launch_id},
                                              {"$set": {"completed": True}})
@@ -1740,11 +1745,8 @@ def recover_offline(self, launch_id, ignore_errors=False, print_errors=False):
                                                        m_launch.to_db_dict(), upsert=True)
                 fw_id = l['fw_id']
                 f = self.fireworks.find_one_and_update({'fw_id': fw_id},
-                                                       {'$set':
-                                                            {'state': 'RUNNING',
-                                                             'updated_on': datetime.datetime.utcnow()
-                                                             }
-                                                        })
+                                                       {'$set': {'state': 'RUNNING',
+                                                                 'updated_on': datetime.datetime.utcnow()}})
                 if f:
                     self._refresh_wf(fw_id)
 
diff --git a/fireworks/features/stats.py b/fireworks/features/stats.py
@@ -269,13 +269,15 @@ def _aggregate(coll, group_by="state", match=None, project=None, unwind=None,
         :param match: (dict) Query for the match step in Mongodb aggregation framework.
         :param project: (dict) Query for the project step in Mongodb aggregation framework
         :param unwind: (dict) Query for the unwind step in Mongodb aggregation framework
-        :param group_op: (dict) Additional operations to generate values in the group step in Mongodb aggregation framework.
+        :param group_op: (dict) Additional operations to generate values in the group step in Mongodb aggregation
+            framework.
         :param sort: (tuple) Defines how to sort the aggregation results. Default is to sort by field in group_by.
         :param return_query_only:  (bool) If only return the query expression for aggregation. Default is False.
         :return: (list) Aggregation results if the operation is successful.
         """
         for arg in [match, project, unwind, group_op]:
-            if arg is None: arg = {}
+            if arg is None:
+                arg = {}
         group_op.update({"_id": "$" + group_by})
         if sort is None:
             sort_query = ("_id", 1)
diff --git a/fireworks/flask_site/app.py b/fireworks/flask_site/app.py
@@ -317,8 +317,7 @@ def wf_metadata_find(key, value, state):
             page = int(request.args.get('page', 1))
         except ValueError:
             page = 1
-        rows = list(db.find(q).sort([('_id', DESCENDING)]). \
-                    skip(page - 1).limit(PER_PAGE))
+        rows = list(db.find(q).sort([('_id', DESCENDING)]).skip(page - 1).limit(PER_PAGE))
         for r in rows:
             r["fw_id"] = r["nodes"][0]
         pagination = Pagination(page=page, total=wf_count,
diff --git a/fireworks/flask_site/gunicorn.py b/fireworks/flask_site/gunicorn.py
@@ -31,6 +31,7 @@ def load_config(self):
     def load(self):
         return self.application
 
+
 if __name__ == '__main__':
     options = {
         'bind': '%s:%s' % ('127.0.0.1', '8080'),
diff --git a/fireworks/flask_site/helpers.py b/fireworks/flask_site/helpers.py
@@ -4,7 +4,7 @@ def get_totals(states, lp):
     for state in states:
         fw_stats[state] = lp.get_fw_ids(query={'state': state}, count_only=True)
         wf_stats[state] = lp.get_wf_ids(query={'state': state}, count_only=True)
-    return {"fw_stats": fw_stats, "wf_stats":wf_stats}
+    return {"fw_stats": fw_stats, "wf_stats": wf_stats}
 
 
 def fw_filt_given_wf_filt(filt, lp):
@@ -20,6 +20,7 @@ def wf_filt_given_fw_filt(filt, lp):
         wf_ids.add(doc['fw_id'])
     return {"nodes": {"$in": list(wf_ids)}}
 
+
 def uses_index(filt, coll):
     ii = coll.index_information()
     fields_filtered = set(filt.keys())
diff --git a/fireworks/fw_config.py b/fireworks/fw_config.py
@@ -93,7 +93,7 @@
 
 WEBSERVER_PORT = 5000  # default port on which the Flask web server runs
 
-WEBSERVER_PERFWARNINGS = False # enable performance-related warnings
+WEBSERVER_PERFWARNINGS = False  # enable performance-related warnings
 
 # value of socketTimeoutMS when connection to mongoDB.  See pymongo official
 # documentation http://api.mongodb.org/python/current/api/pymongo/mongo_client.html
@@ -117,7 +117,7 @@ def override_user_settings():
         if fp not in config_paths and os.path.exists(fp):
             config_paths.append(fp)
 
-    if "FW_CONFIG_FILE" in os.environ and os.environ["FW_CONFIG_FILE"] not in\
+    if "FW_CONFIG_FILE" in os.environ and os.environ["FW_CONFIG_FILE"] not in \
             config_paths:
         config_paths.append(os.environ["FW_CONFIG_FILE"])
 
diff --git a/fireworks/scripts/__init__.py b/fireworks/scripts/__init__.py
@@ -3,4 +3,4 @@
 __version__ = '0.1'
 __maintainer__ = 'Anubhav Jain'
 __email__ = 'ajain@lbl.gov'
-__date__ = 'Feb 28, 2013'
+__date__ = 'Feb 28, 2013'
diff --git a/fireworks/scripts/lpad_run.py b/fireworks/scripts/lpad_run.py
@@ -127,7 +127,8 @@ def init_yaml(args):
             ("host", None, "Example: mongodb+srv://USER:PASSWORD@CLUSTERNAME.mongodb.net/fireworks"),
             ("ssl_ca_file", None, "Path to any client certificate to be used for mongodb connection"),
             ("authsource", None,
-             "Database used for authentication, if not connection db. e.g., for MongoDB Atlas this is sometimes 'admin'."))
+             "Database used for authentication, if not connection db. e.g., for MongoDB Atlas this is sometimes "
+             "'admin'."))
     else:
         fields = (
             ("host", "localhost", "Example: 'localhost' or 'mongodb+srv://CLUSTERNAME.mongodb.net'"),
@@ -137,12 +138,15 @@ def init_yaml(args):
             ("password", None, "Password for MongoDB authentication"),
             ("ssl_ca_file", None, "Path to any client certificate to be used for Mongodb connection"),
             ("authsource", None,
-             "Database used for authentication, if not connection db. e.g., for MongoDB Atlas this is sometimes 'admin'."))
+             "Database used for authentication, if not connection db. e.g., for MongoDB Atlas this is sometimes "
+             "'admin'."))
 
     doc = {}
     if args.uri_mode:
         print(
-            "Note 1: You are in URI format mode. This means that all database parameters (username, password, host, port, database name, etc.) must be present in the URI. See: https://docs.mongodb.com/manual/reference/connection-string/ for details.")
+            "Note 1: You are in URI format mode. This means that all database parameters (username, password, host, "
+            "port, database name, etc.) must be present in the URI. See: "
+            "https://docs.mongodb.com/manual/reference/connection-string/ for details.")
         print("(Enter your connection URI in under the 'host' parameter)")
     print("Please supply the following configuration values")
     print("(press Enter if you want to accept the defaults)\n")
@@ -750,7 +754,8 @@ def lpad():
         'init', help='Initialize a Fireworks launchpad YAML file.')
     init_parser.add_argument('-u', '--uri_mode',
                              action="store_true",
-                             help="Connect via a URI, see: https://docs.mongodb.com/manual/reference/connection-string/")
+                             help="Connect via a URI, see: "
+                                  "https://docs.mongodb.com/manual/reference/connection-string/")
     init_parser.add_argument('--config-file', default=DEFAULT_LPAD_YAML,
                              type=str,
                              help="Filename to write to.")
@@ -776,14 +781,17 @@ def lpad():
     check_wf_parser = subparsers.add_parser('check_wflow', help='validate and graph a workflow from launchpad')
     check_wf_parser.add_argument('-i', '--fw_id', type=int, help='the id of a firework from the workflow')
     check_wf_parser.add_argument('-g', '--graph', type=str,
-                                 help='graph the workflow in DOT format; allowed views: dataflow, controlflow, combined.',
+                                 help='graph the workflow in DOT format; allowed views: dataflow, controlflow, '
+                                      'combined.',
                                  dest='view', default=None)
     check_wf_parser.add_argument('-f', '--dot_file', help='path to store the workflow graph, default: workflow.dot',
                                  default='workflow.dot')
     check_wf_parser.set_defaults(func=check_wf, control_flow=False, data_flow=False)
 
     get_launchdir_parser = subparsers.add_parser('get_launchdir',
-                                                 help='get the directory of the most recent launch of the given fw_id. A common usage is "cd `get_launchdir <FW_ID>`" to change the working directory that of the FW launch')
+                                                 help='get the directory of the most recent launch of the given fw_id.'
+                                                      ' A common usage is "cd `get_launchdir <FW_ID>`" to change the '
+                                                      'working directory that of the FW launch')
     get_launchdir_parser.add_argument('fw_id', type=int, help='fw_id to chdir to')
     get_launchdir_parser.add_argument('--launch_idx', type=int,
                                       help='the index of the launch to get (default of -1 is most recent launch)',
@@ -1075,9 +1083,11 @@ def lpad():
 
     webgui_parser = subparsers.add_parser('webgui', help='launch the web GUI')
     webgui_parser.add_argument("--port", dest="port", type=int, default=WEBSERVER_PORT,
-                               help="Port to run the web server on (default: 5000 or WEBSERVER_PORT arg in FW_config.yaml)")
+                               help="Port to run the web server on (default: 5000 or WEBSERVER_PORT arg in "
+                                    "FW_config.yaml)")
     webgui_parser.add_argument("--host", dest="host", type=str, default=WEBSERVER_HOST,
-                               help="Host to run the web server on (default: 127.0.0.1 or WEBSERVER_HOST arg in FW_config.yaml)")
+                               help="Host to run the web server on (default: 127.0.0.1 or WEBSERVER_HOST arg in "
+                                    "FW_config.yaml)")
     webgui_parser.add_argument('--debug', help='print debug messages', action='store_true')
     webgui_parser.add_argument('-s', '--server_mode', help='run in server mode (skip opening the browser)',
                                action='store_true')
diff --git a/fireworks/user_objects/dupefinders/__init__.py b/fireworks/user_objects/dupefinders/__init__.py
@@ -3,4 +3,4 @@
 __version__ = '0.1'
 __maintainer__ = 'Anubhav Jain'
 __email__ = 'ajain@lbl.gov'
-__date__ = 'Mar 01, 2013'
+__date__ = 'Mar 01, 2013'
diff --git a/fireworks/user_objects/firetasks/fileio_tasks.py b/fireworks/user_objects/firetasks/fileio_tasks.py
@@ -55,7 +55,6 @@ class FileDeleteTask(FiretaskBase):
     _fw_name = 'FileDeleteTask'
     required_params = ["files_to_delete"]
 
-
     def run_task(self, fw_spec):
         pth = self.get("dest", os.getcwd())
         ignore_errors = self.get('ignore_errors', True)
@@ -89,13 +88,13 @@ class FileTransferTask(FiretaskBase):
     optional_params = ["server", "user", "key_filename", "max_retry", "retry_delay"]
 
     fn_list = {
-            "move": shutil.move,
-            "mv": shutil.move,
-            "copy": shutil.copy,
-            "cp": shutil.copy,
-            "copy2": shutil.copy2,
-            "copytree": shutil.copytree,
-            "copyfile": shutil.copyfile,
+        "move": shutil.move,
+        "mv": shutil.move,
+        "copy": shutil.copy,
+        "cp": shutil.copy,
+        "copy2": shutil.copy2,
+        "copytree": shutil.copytree,
+        "copyfile": shutil.copyfile,
     }
 
     def run_task(self, fw_spec):
@@ -128,7 +127,7 @@ def run_task(self, fw_spec):
                             sftp.mkdir(dest)
 
                         for f in os.listdir(src):
-                            if os.path.isfile(os.path.join(src,f)):
+                            if os.path.isfile(os.path.join(src, f)):
                                 sftp.put(os.path.join(src, f), os.path.join(dest, f))
                     else:
                         if not self._rexists(sftp, dest):
diff --git a/fireworks/user_objects/firetasks/filepad_tasks.py b/fireworks/user_objects/firetasks/filepad_tasks.py
@@ -105,8 +105,8 @@ class GetFilesByQueryTask(FiretaskBase):
     """
     _fw_name = 'GetFilesByQueryTask'
     required_params = ["query"]
-    optional_params = ["sort_key","sort_direction", "limit",
-        "filepad_file", "dest_dir", "new_file_names"]
+    optional_params = ["sort_key", "sort_direction", "limit",
+                       "filepad_file", "dest_dir", "new_file_names"]
 
     def run_task(self, fw_spec):
         import pymongo
@@ -116,14 +116,15 @@ def run_task(self, fw_spec):
         query = self.get("query", {})
         sort_key = self.get("sort_key", None)
         sort_direction = self.get("sort_direction", pymongo.DESCENDING)
-        limit = self.get("limit",None)
+        limit = self.get("limit", None)
 
-        l = fpad.get_file_by_query(query,sort_key,sort_direction)
+        l = fpad.get_file_by_query(query, sort_key, sort_direction)
         for i, (file_contents, doc) in enumerate(l[:limit]):
             file_name = new_file_names[i] if new_file_names else doc["original_file_name"]
             with open(os.path.join(dest_dir, file_name), "wb") as f:
                 f.write(file_contents)
 
+
 class DeleteFilesTask(FiretaskBase):
     """
     A Firetask to delete files from the filepad
@@ -148,4 +149,4 @@ def get_fpad(fpad_file):
     if fpad_file:
         return FilePad.from_db_file(fpad_file)
     else:
-        return FilePad.auto_load()
+        return FilePad.auto_load()
diff --git a/fireworks/user_objects/firetasks/templatewriter_task.py b/fireworks/user_objects/firetasks/templatewriter_task.py
@@ -9,7 +9,7 @@
 
 import os
 
-from jinja2 import Environment,FileSystemLoader
+from jinja2 import Environment, FileSystemLoader
 
 from fireworks.core.firework import FiretaskBase
 from fireworks.fw_config import TEMPLATE_DIR
diff --git a/fireworks/user_objects/queue_adapters/common_adapter.py b/fireworks/user_objects/queue_adapters/common_adapter.py
@@ -108,7 +108,8 @@ def _get_status_cmd(self, username):
             if self.get('queue'):
                 status_cmd.extend(['-p', self['queue']])
         elif self.q_type == "LoadSharingFacility":
-            # use no header and the wide format so that there is one line per job, and display only running and pending jobs
+            # use no header and the wide format so that there is one line per job, and display only running and
+            # pending jobs
             status_cmd.extend(['-p', '-r', '-o', 'jobID user queue', '-noheader', '-u', username])
         elif self.q_type == "Cobalt":
             header = "JobId:User:Queue:Jobname:Nodes:Procs:Mode:WallTime:State:RunTime:Project:Location"
diff --git a/fireworks/user_objects/queue_adapters/pbs_newt_adapter.py b/fireworks/user_objects/queue_adapters/pbs_newt_adapter.py
diff --git a/fireworks/utilities/fw_serializers.py b/fireworks/utilities/fw_serializers.py
diff --git a/fireworks/utilities/update_collection.py b/fireworks/utilities/update_collection.py
