Fix flake8 failures and make its execution more consistent

- upgrade flake8 to 3.6.0
- remove use of hacking since pre-commit replaced it
- mentioned minimal flake8 version on pre-commit config
- fixed newer linting errors
- disable W504 as it seems not possible to fix (conflict with W403)
- replaced buggy tox deps installation with native pip install, which
  address the issue of missing to install new deps when the reqs files
  are updated, developer being forced to to recreate the virtualenv.
- prepare for migration from tox-pep8 to tox-linters, now these being
  just aliases.

Change-Id: I310578dce215aaf00a5b2d54716f90da9a1ecb4d
Depends-On: https://review.openstack.org/#/c/613726/

.pre-commit-config.yaml

@@ -2,10 +2,13 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.2.1-1
+    rev: v2.0.0
     hooks:
     -   id: check-added-large-files
     -   id: end-of-file-fixer
     -   id: flake8
         args: ['--config=setup.cfg']
+        language: python
+        types: [python]
+        additional_dependencies: [flake8>=3.6.0]
     -   id: trailing-whitespace

jenkins_jobs/alphanum.py

@@ -32,9 +32,9 @@ sorted( foo, key=AlphanumSort)
 import re
 
 
-re_chunk = re.compile("([\D]+|[\d]+)")
-re_letters = re.compile("\D+")
-re_numbers = re.compile("\d+")
+re_chunk = re.compile(r"([\D]+|[\d]+)")
+re_letters = re.compile(r"\D+")
+re_numbers = re.compile(r"\d+")
 
 
 def getchunk(item):

jenkins_jobs/cache.py

@@ -43,7 +43,7 @@ class JobCache(object):
     def __init__(self, jenkins_url, flush=False):
         cache_dir = self.get_cache_dir()
         # One cache per remote Jenkins URL:
-        host_vary = re.sub('[^A-Za-z0-9\-\~]', '_', jenkins_url)
+        host_vary = re.sub(r'[^A-Za-z0-9\-\~]', '_', jenkins_url)
         self.cachefilename = os.path.join(
             cache_dir, 'cache-host-jobs-' + host_vary + '.yml')
 

jenkins_jobs/formatter.py

@@ -81,7 +81,7 @@ class CustomFormatter(Formatter):
     Custom formatter to allow non-existing key references when formatting a
     string
     """
-    _expr = """
+    _expr = r"""
         (?<!{){({{)*                # non-pair opening {
         (?:obj:)?                   # obj:
         (?P<key>\w+)                # key

jenkins_jobs/modules/builders.py

@@ -2888,7 +2888,7 @@ def cmake(registry, xml_parent, data):
 
 
 def dsl(registry, xml_parent, data):
-    """yaml: dsl
+    r"""yaml: dsl
     Process Job DSL
 
     Requires the Jenkins :jenkins-wiki:`Job DSL plugin <Job+DSL+Plugin>`.
@@ -3475,7 +3475,7 @@ def jms_messaging(registry, xml_parent, data):
 
 
 def openshift_build_verify(registry, xml_parent, data):
-    """yaml: openshift-build-verify
+    r"""yaml: openshift-build-verify
     Performs the equivalent of an 'oc get builds` command invocation for the
     provided buildConfig key provided; once the list of builds are obtained,
     the state of the latest build is inspected for up to a minute to see if
@@ -3524,7 +3524,7 @@ def openshift_build_verify(registry, xml_parent, data):
 
 
 def openshift_builder(registry, xml_parent, data):
-    """yaml: openshift-builder
+    r"""yaml: openshift-builder
     Perform builds in OpenShift for the job.
     Requires the Jenkins :jenkins-wiki:`OpenShift
     Pipeline Plugin <OpenShift+Pipeline+Plugin>`.
@@ -3580,7 +3580,7 @@ def openshift_builder(registry, xml_parent, data):
 
 
 def openshift_creator(registry, xml_parent, data):
-    """yaml: openshift-creator
+    r"""yaml: openshift-creator
     Performs the equivalent of an oc create command invocation;
     this build step takes in the provided JSON or YAML text, and if it
     conforms to OpenShift schema, creates whichever
@@ -3628,7 +3628,7 @@ def openshift_creator(registry, xml_parent, data):
 
 
 def openshift_dep_verify(registry, xml_parent, data):
-    """yaml: openshift-dep-verify
+    r"""yaml: openshift-dep-verify
     Determines whether the expected set of DeploymentConfig's,
     ReplicationController's, and active replicas are present based on prior
     use of the scaler (2) and deployer (3) steps
@@ -3679,7 +3679,7 @@ def openshift_dep_verify(registry, xml_parent, data):
 
 
 def openshift_deployer(registry, xml_parent, data):
-    """yaml: openshift-deployer
+    r"""yaml: openshift-deployer
     Start a deployment in OpenShift for the job.
     Requires the Jenkins :jenkins-wiki:`OpenShift
     Pipeline Plugin <OpenShift+Pipeline+Plugin>`.
@@ -3725,7 +3725,7 @@ def openshift_deployer(registry, xml_parent, data):
 
 
 def openshift_img_tagger(registry, xml_parent, data):
-    """yaml: openshift-img-tagger
+    r"""yaml: openshift-img-tagger
     Performs the equivalent of an oc tag command invocation in order to
     manipulate tags for images in OpenShift ImageStream's
     Requires the Jenkins :jenkins-wiki:`OpenShift
@@ -3776,7 +3776,7 @@ def openshift_img_tagger(registry, xml_parent, data):
 
 
 def openshift_scaler(registry, xml_parent, data):
-    """yaml: openshift-scaler
+    r"""yaml: openshift-scaler
     Scale deployments in OpenShift for the job.
     Requires the Jenkins :jenkins-wiki:`OpenShift
     Pipeline Plugin <OpenShift+Pipeline+Plugin>`.
@@ -3823,7 +3823,7 @@ def openshift_scaler(registry, xml_parent, data):
 
 
 def openshift_svc_verify(registry, xml_parent, data):
-    """yaml: openshift-svc-verify
+    r"""yaml: openshift-svc-verify
     Verify a service is up in OpenShift for the job.
     Requires the Jenkins :jenkins-wiki:`OpenShift
     Pipeline Plugin <OpenShift+Pipeline+Plugin>`.

jenkins_jobs/modules/project_multibranch.py

@@ -290,7 +290,7 @@ class WorkflowMultiBranchDefaults(WorkflowMultiBranch):
 
 
 def bitbucket_scm(xml_parent, data):
-    """Configure BitBucket scm
+    r"""Configure BitBucket scm
 
     Requires the :jenkins-wiki:`Bitbucket Branch Source Plugin
     <Bitbucket+Branch+Source+Plugin>`.
@@ -626,7 +626,7 @@ def gerrit_scm(xml_parent, data):
 
 
 def git_scm(xml_parent, data):
-    """Configure Git SCM
+    r"""Configure Git SCM
 
     Requires the :jenkins-wiki:`Git Plugin <Git+Plugin>`.
 
@@ -738,7 +738,7 @@ def git_scm(xml_parent, data):
 
 
 def github_scm(xml_parent, data):
-    """Configure GitHub SCM
+    r"""Configure GitHub SCM
 
     Requires the :jenkins-wiki:`GitHub Branch Source Plugin
     <GitHub+Branch+Source+Plugin>`.

jenkins_jobs/modules/publishers.py

@@ -1658,7 +1658,7 @@ def violations(registry, xml_parent, data):
 
 
 def findbugs(registry, xml_parent, data):
-    """yaml: findbugs
+    r"""yaml: findbugs
     FindBugs reporting for builds
 
     Requires the Jenkins :jenkins-wiki:`FindBugs Plugin
@@ -2962,7 +2962,7 @@ def sounds(parser, xml_parent, data):
 
 
 def performance(registry, xml_parent, data):
-    """yaml: performance
+    r"""yaml: performance
     Publish performance test results from jmeter and junit.
     Requires the Jenkins :jenkins-wiki:`Performance Plugin
     <Performance+Plugin>`.
@@ -4391,7 +4391,7 @@ def warnings(registry, xml_parent, data):
 
 
 def sloccount(registry, xml_parent, data):
-    """yaml: sloccount
+    r"""yaml: sloccount
     Generates the trend report for SLOCCount
 
     Requires the Jenkins :jenkins-wiki:`SLOCCount Plugin <SLOCCount+Plugin>`.
@@ -7220,7 +7220,7 @@ def jms_messaging(registry, xml_parent, data):
 
 
 def openshift_build_canceller(registry, xml_parent, data):
-    """yaml: openshift-build-canceller
+    r"""yaml: openshift-build-canceller
     This action is intended to provide cleanup for a Jenkins job which failed
     because a build is hung (instead of terminating with a failure code);
     this step will allow you to perform the equivalent of a oc cancel-build
@@ -7271,7 +7271,7 @@ def openshift_build_canceller(registry, xml_parent, data):
 
 
 def openshift_deploy_canceller(registry, xml_parent, data):
-    """yaml: openshift-deploy-canceller
+    r"""yaml: openshift-deploy-canceller
     This action is intended to provide cleanup for any OpenShift deployments
     left running when the Job completes; this step will allow you to perform
     the equivalent of a oc deploy --cancel for the provided deployment config.

jenkins_jobs/modules/scm.py

@@ -46,7 +46,7 @@ import jenkins_jobs.modules.helpers as helpers
 
 
 def git(registry, xml_parent, data):
-    """yaml: git
+    r"""yaml: git
     Specifies the git SCM repository for this job.
     Requires the Jenkins :jenkins-wiki:`Git Plugin <Git+Plugin>`.
 
@@ -994,7 +994,7 @@ def svn(registry, xml_parent, data):
 
 
 def tfs(registry, xml_parent, data):
-    """yaml: tfs
+    r"""yaml: tfs
     Specifies the Team Foundation Server repository for this job.
     Requires the Jenkins :jenkins-wiki:`Team Foundation Server Plugin
     <Team+Foundation+Server+Plugin>`.
@@ -1216,7 +1216,7 @@ def hg(self, xml_parent, data):
 
 
 def openshift_img_streams(registry, xml_parent, data):
-    """yaml: openshift-img-streams
+    r"""yaml: openshift-img-streams
     Rather than a Build step extension plugin, this is an extension of the
     Jenkins SCM plugin, where this baked-in polling mechanism provided by
     Jenkins is leveraged by exposing some of the common semantics between

jenkins_jobs/sphinx/yaml.py

@@ -29,7 +29,7 @@ from sphinx.ext.autodoc import FunctionDocumenter
 from sphinx.locale import _
 
 
-yaml_sig_re = re.compile('yaml:\s*(.*)')
+yaml_sig_re = re.compile(r'yaml:\s*(.*)')
 
 
 class PyYAMLFunction(PyModulelevel):

setup.cfg

@@ -101,6 +101,8 @@ jenkins_jobs.modules =
 [flake8]
 # These are ignored intentionally in openstack-infra projects; please
 # don't submit patches that solely correct them or enable them.
-ignore = E125,E128,H
+# W504 is controversial an apparently conflicting with W503, being impossible
+# to solve both of them while still keeping the line length limited.
+ignore = E125,E128,H,W504
 show-source = True
 exclude = .virtualenv,.venv,.tox,dist,build,*.egg,.test

test-requirements.txt

@@ -1,7 +1,6 @@
 # The order of packages is significant, because pip processes them in the order
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
-hacking>=1.1.0 # Apache-2.0
 
 coverage>=4.0 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
@@ -13,4 +12,4 @@ stestr>=2.0.0 # Apache-2.0/BSD
 tox>=2.9.1 # MIT
 mock>=2.0 # BSD
 sphinxcontrib-programoutput
-pre-commit
+pre-commit>=1.12.0

tox.ini

@@ -1,7 +1,9 @@
 [tox]
 minversion = 2.2
-envlist = docs, pep8, py34, py35, py36, py27, cover
+envlist = linters, docs, py34, py35, py36, py27, cover
 skip_missing_interpreters = true
+# custom vars (no meaning to tox)
+install_test_deps = pip install -q -r test-requirements.txt
 
 [testenv]
 setenv =
@@ -11,11 +13,11 @@ setenv =
          VIRTUAL_ENV={envdir}
 usedevelop = True
 install_command = pip install {opts} {packages}
-deps = -r{toxinidir}/test-requirements.txt
 # cleanup is needed mostly for dev environmnts and assures that if  dev
 # is switching branch, no lefovers will impact execution.
 # No git commands here because test should run even from a source archive.
 commands =
+    {[tox]install_test_deps}
     - find . -type f -name "*.pyc" -delete
     - find . -type d -name "__pycache__" -delete
     # test that we can call jjb using both variants with same results
@@ -39,15 +41,22 @@ setenv =
     {[testenv]setenv}
     PYTHON=coverage run --source jenkins_jobs --parallel-mode
 commands =
+    {[tox]install_test_deps}
     stestr run {posargs}
     coverage combine
     coverage html -d cover
     coverage xml -o cover/coverage.xml
 
-[testenv:pep8]
+[testenv:linters]
 basepython = python3
 commands =
-   python -m pre_commit run --all
+    {[tox]install_test_deps}
+    python -m pre_commit run --all
+
+# points to linters, kept only for developer convenience
+[testenv:pep8]
+envdir={toxworkdir}/linters
+commands = {[testenv:linters]commands}
 
 [testenv:pyflakes]
 deps = pyflakes
@@ -63,7 +72,9 @@ commands = jenkins-jobs test -o .test/old/out/ .test/old/config/
 commands = jenkins-jobs test -o .test/new/out/ .test/new/config/
 
 [testenv:docs]
-commands = python setup.py build_sphinx {posargs}
+commands =
+    {[tox]install_test_deps}
+    python setup.py build_sphinx {posargs}
 
 [testenv:docs-linkcheck]
 # If you are behind a proxy, for this test to work you will need to set