Merge pull request #255 from crim-ca/fix-lint-ignore-comments

Address `fix-lint` target problem not respecting ignore comments

Author: fmigneault

Makefile

@@ -478,12 +478,24 @@ fix-imports-only: mkdir-reports	## apply import code checks corrections
 		isort --recursive $(APP_ROOT) \
 		1> >(tee "$(REPORTS_DIR)/fixed-imports.txt")'
 
+# FIXME: https://github.com/PyCQA/pycodestyle/issues/996
+# Tool "pycodestyle" doesn't respect "# noqa: E241" locally, but "flake8" and other tools do.
+# Because "autopep8" uses "pycodestyle", it is impossible to disable locally extra spaces (as in tests to align values).
+# Override the codes here from "setup.cfg" because "autopep8" also uses the "flake8" config, and we want to preserve
+# global detection of those errors (typos, bad indents), unless explicitly added and excluded for readability purposes.
+# WARNING: this will cause inconsistencies between what 'check-lint' detects and what 'fix-lint' can actually fix
+_DEFAULT_SETUP_ERROR := E126,E226,E402,F401,W503,W504
+_EXTRA_SETUP_ERROR := E241
+
 .PHONY: fix-lint-only
 fix-lint-only: mkdir-reports  ## fix some PEP8 code style problems automatically
 	@echo "Fixing PEP8 code style problems..."
 	@-rm -fr "$(REPORTS_DIR)/fixed-lint.txt"
 	@bash -c '$(CONDA_CMD) \
-		autopep8 -v -j 0 -i -r $(APP_ROOT) \
+		autopep8 \
+		 	--global-config "$(APP_ROOT)/setup.cfg" \
+		 	--ignore "$(_DEFAULT_SETUP_ERROR),$(_EXTRA_SETUP_ERROR)" \
+			-v -j 0 -i -r $(APP_ROOT) \
 		1> >(tee "$(REPORTS_DIR)/fixed-lint.txt")'
 
 # FIXME: move parameters to setup.cfg when implemented (https://github.com/myint/docformatter/issues/10)

setup.cfg

@@ -5,21 +5,21 @@ tag = True
 tag_name = {new_version}
 
 [bumpversion:file:CHANGES.rst]
-search = 
+search =
 	`Unreleased <https://github.com/crim-ca/weaver/tree/master>`_ (latest)
 	========================================================================
-replace = 
+replace =
 	`Unreleased <https://github.com/crim-ca/weaver/tree/master>`_ (latest)
 	========================================================================
-	
+
 	Changes:
 	--------
 	- No change.
-	
+
 	Fixes:
 	------
 	- No change.
-	
+
 	`{new_version} <https://github.com/crim-ca/weaver/tree/{new_version}>`_ ({now:%%Y-%%m-%%d})
 	========================================================================
 
@@ -40,12 +40,12 @@ search = LABEL version="{current_version}"
 replace = LABEL version="{new_version}"
 
 [tool:pytest]
-addopts = 
+addopts =
 	--strict-markers
 	--tb=native
 	weaver/
 python_files = test_*.py
-markers = 
+markers =
 	testbed14: mark test as 'testbed14' validation
 	functional: mark test as functionality validation
 	workflow: mark test as workflow execution (E2E)
@@ -70,9 +70,12 @@ exclude = *.egg-info,build,dist,env,tests,./tests,test_*
 targets = .
 
 [flake8]
+# FIXME: https://github.com/PyCQA/pycodestyle/issues/996
+# see Makefile for some specific overrides to avoid false-positive during 'fix-lint'
+# new codes must be synced between below list and corresponding one in Makefile
 ignore = E126,E226,E402,F401,W503,W504
 max-line-length = 120
-exclude = 
+exclude =
 	src,
 	.git,
 	__pycache__,
@@ -88,18 +91,19 @@ max-line-length = 120
 ignore-path = docs/build,docs/source/autoapi
 
 [pylint]
+# use .pylintrc
 
 [coverage:run]
 branch = true
 source = ./
 include = weaver/*
-omit = 
+omit =
 	setup.py
 	docs/*
 	tests/*
 
 [coverage:report]
-exclude_lines = 
+exclude_lines =
 	pragma: no cover
 	raise AssertionError
 	raise NotImplementedError

weaver/datatype.py

@@ -57,6 +57,7 @@ class Base(dict):
     Explicitly overridden ``getter``/``setter`` attributes are called instead of ``dict``-key ``get``/``set``-item
     to ensure corresponding checks and/or value adjustments are executed before applying it to the sub-``dict``.
     """
+
     def __setattr__(self, item, value):
         # use the existing property setter if defined
         prop = getattr(type(self), item)

weaver/processes/wps_workflow.py

@@ -110,6 +110,7 @@ class WpsWorkflow(ProcessCWL):
     Definition of a ``CWL`` ``workflow`` that can execute ``WPS`` application packages as defined by
     :class:`weaver.processes.wps_package.WpsPackage` as job steps.
     """
+
     def __init__(self, toolpath_object, loading_context, get_job_process_definition):
         # type: (Dict[Text, Any], LoadingContext, GetJobProcessDefinitionFunction) -> None
         super(WpsWorkflow, self).__init__(toolpath_object, loading_context)

weaver/store/mongodb.py

@@ -177,6 +177,7 @@ class MongodbProcessStore(StoreProcesses, MongodbStore):
     """
     Registry for processes. Uses `MongoDB` to store processes and attributes.
     """
+
     def __init__(self, *args, **kwargs):
         db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
         StoreProcesses.__init__(self)
@@ -388,6 +389,7 @@ class MongodbJobStore(StoreJobs, MongodbStore):
     """
     Registry for process jobs tracking. Uses `MongoDB` to store job attributes.
     """
+
     def __init__(self, *args, **kwargs):
         db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
         StoreJobs.__init__(self)
@@ -656,6 +658,7 @@ class MongodbQuoteStore(StoreQuotes, MongodbStore):
     """
     Registry for quotes. Uses `MongoDB` to store quote attributes.
     """
+
     def __init__(self, *args, **kwargs):
         db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
         StoreQuotes.__init__(self)
@@ -727,6 +730,7 @@ class MongodbBillStore(StoreBills, MongodbStore):
     """
     Registry for bills. Uses `MongoDB` to store bill attributes.
     """
+
     def __init__(self, *args, **kwargs):
         db_args, db_kwargs = MongodbStore.get_args_kwargs(*args, **kwargs)
         StoreBills.__init__(self)

weaver/wps/service.py

@@ -99,6 +99,7 @@ class WorkerService(ServiceWPS):
     When receiving ``Execute`` request, convert the XML payload to corresponding JSON and
     dispatch it to the Celery Worker to actually process it after job setup for monitoring.
     """
+
     def __init__(self, *_, is_worker=False, settings=None, **__):
         super(WorkerService, self).__init__(*_, **__)
         self.is_worker = is_worker

weaver/wps_restapi/colander_extras.py

@@ -104,6 +104,7 @@ class OneOfCaseInsensitive(colander.OneOf):
     """
     Validator that ensures the given value matches one of the available choices, but allowing case insensitive values.
     """
+
     def __call__(self, node, value):
         if str(value).lower() not in (choice.lower() for choice in self.choices):
             return super(OneOfCaseInsensitive, self).__call__(node, value)
@@ -113,6 +114,7 @@ class StringRange(colander.Range):
     """
     Validator that provides the same functionalities as :class:`colander.Range` for a numerical string value.
     """
+
     def __init__(self, min=None, max=None, **kwargs):
         try:
             if isinstance(min, str):
@@ -139,6 +141,7 @@ class SchemeURL(colander.Regex):
         :class:`colander.url` [remote http(s)/ftp(s)]
         :class:`colander.file_uri` [local file://]
     """
+
     def __init__(self, schemes=None, msg=None, flags=re.IGNORECASE):
         if not schemes:
             schemes = [""]
@@ -875,6 +878,7 @@ class AnyKeyObject(PermissiveMappingSchema):
         This class is only a shorthand definition of ``unknown`` keyword for convenience.
         All :class:`colander.MappingSchema` support this natively.
     """
+
     def __init__(self, *args, **kwargs):
         kwargs["unknown"] = "preserve"
         super(PermissiveMappingSchema, self).__init__(*args, **kwargs)
@@ -1509,6 +1513,7 @@ def _deserialize_keyword(self, cstruct):
 
 class KeywordTypeConverter(TypeConverter):
     """Generic keyword converter that builds schema with a list of sub-schemas under the keyword."""
+
     def convert_type(self, schema_node):
         keyword = schema_node.get_keyword_name()
         keyword_schema = {
@@ -1531,6 +1536,7 @@ class OneOfKeywordTypeConverter(KeywordTypeConverter):
     .. seealso::
         - :class:`OneOfKeywordSchema`
     """
+
     def convert_type(self, schema_node):
         # type: (OneOfKeywordSchema) -> Dict
         keyword = schema_node.get_keyword_name()
@@ -1599,6 +1605,7 @@ class VariableObjectTypeConverter(ObjectTypeConverter):
     Updates the mapping object's ``additionalProperties`` for each ``properties``
     that a marked as :class:`VariableSchemaNode`.
     """
+
     def convert_type(self, schema_node):
         converted = super(VariableObjectTypeConverter, self).convert_type(schema_node)
         converted.setdefault("additionalProperties", {})