Custom formatter

Implements: custom formatter

Custom formatter can be used to output a machine-readable, easily
parsable and customizable format using set of predefined tags
to suite various needs.

Output string is formatted using python string.format() standards
and therefore provides familiar usage.

Usage: bandit --format custom [--msg-template MSG-TEMPLATE] targets

See bandit --help for additional information and list of available tags

modified:   bandit/cli/main.py
modified:   bandit/core/manager.py
modified:   README.rst
modified:   setup.cfg
new file:   bandit/formatters/custom.py

Change-Id: I900c9689cddb048db58608c443305e05e7a4be14
Signed-off-by: Marek Cermak <macermak@redhat.com>

README.rst

@@ -87,8 +87,9 @@ Usage::
     $ bandit -h
     usage: bandit [-h] [-r] [-a {file,vuln}] [-n CONTEXT_LINES] [-c CONFIG_FILE]
                   [-p PROFILE] [-t TESTS] [-s SKIPS] [-l] [-i]
-                  [-f {csv,html,json,screen,txt,xml,yaml}] [-o [OUTPUT_FILE]] [-v]
+                  [-f {csv,custom,html,json,screen,txt,xml,yaml}]
-                  [-d] [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE]
+                  [--msg-template MSG_TEMPLATE] [-o [OUTPUT_FILE]] [-v] [-d]
+                  [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE]
                   [--ini INI_PATH] [--version]
                   targets [targets ...]
 
@@ -118,8 +119,12 @@ Usage::
                             (-l for LOW, -ll for MEDIUM, -lll for HIGH)
       -i, --confidence      report only issues of a given confidence level or
                             higher (-i for LOW, -ii for MEDIUM, -iii for HIGH)
-      -f {csv,html,json,screen,txt,xml,yaml}, --format {csv,html,json,screen,txt,xml,yaml}
+      -f {csv,custom,html,json,screen,txt,xml,yaml}, --format {csv,custom,html,json,screen,txt,xml,yaml}
                             specify output format
+      --msg-template        MSG_TEMPLATE
+                            specify output message template (only usable with
+                            --format custom), see CUSTOM FORMAT section for list
+                            of available values
       -o [OUTPUT_FILE], --output [OUTPUT_FILE]
                             write report to filename
       -v, --verbose         output extra information like excluded and included
@@ -137,7 +142,33 @@ Usage::
                             arguments
       --version             show program's version number and exit
 
+    CUSTOM FORMATTING
+    -----------------
+
+    Available tags:
+
+        {abspath}, {relpath}, {line},  {test_id},
+        {severity}, {msg}, {confidence}, {range}
+
+    Example usage:
+
+        Default template:
+        bandit -r examples/ --format custom --msg-template \
+        "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}"
+
+        Provides same output as:
+        bandit -r examples/ --format custom
+
+        Tags can also be formatted in python string.format() style:
+        bandit -r examples/ --format custom --msg-template \
+        "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}"
+
+        See python documentation for more information about formatting style:
+        https://docs.python.org/3.4/library/string.html
+
     The following tests were discovered and loaded:
+    -----------------------------------------------
+
       B101  assert_used
       B102  exec_used
       B103  set_bad_file_permissions
@@ -339,6 +370,33 @@ To register your plugin, you have two options:
         bandit.plugins =
             mako = bandit_mako
 
+
+Custom Formatting
+-----------------
+
+Available tags:
+
+::
+    {abspath}, {relpath}, {line},  {test_id},
+    {severity}, {msg}, {confidence}, {range}
+
+Example usage:
+
+  Default template::
+    bandit -r examples/ --format custom --msg-template \
+    "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}"
+
+  Provides same output as::
+    bandit -r examples/ --format custom
+
+  Tags can also be formatted in python string.format() style::
+    bandit -r examples/ --format custom --msg-template \
+    "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}"
+
+See python documentation for more information about formatting style:
+https://docs.python.org/3.4/library/string.html
+
+
 Contributing
 ------------
 Contributions to Bandit are always welcome! We can be found on

bandit/cli/main.py

@@ -18,6 +18,7 @@ import fnmatch
 import logging
 import os
 import sys
+import textwrap
 
 
 import bandit
@@ -205,6 +206,13 @@ def main():
         default=output_format, help='specify output format',
         choices=sorted(extension_mgr.formatter_names)
     )
+    parser.add_argument(
+        '--msg-template', action='store',
+        default=None, help='specify output message template'
+                           ' (only usable with --format custom),'
+                           ' see CUSTOM FORMAT section'
+                           ' for list of available values',
+    )
     parser.add_argument(
         '-o', '--output', dest='output_file', action='store', nargs='?',
         type=argparse.FileType('w'), default=sys.stdout,
@@ -253,11 +261,41 @@ def main():
             blacklist_info.append('%s\t%s' % (b['id'], b['name']))
 
     plugin_list = '\n\t'.join(sorted(set(plugin_info + blacklist_info)))
-    parser.epilog = ('The following tests were discovered and'
+    dedent_text = textwrap.dedent('''
-                     ' loaded:\n\t{0}\n'.format(plugin_list))
+    CUSTOM FORMATTING
+    -----------------
+
+    Available tags:
+
+        {abspath}, {relpath}, {line},  {test_id},
+        {severity}, {msg}, {confidence}, {range}
+
+    Example usage:
+
+        Default template:
+        bandit -r examples/ --format custom --msg-template \\
+        "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}"
+
+        Provides same output as:
+        bandit -r examples/ --format custom
+
+        Tags can also be formatted in python string.format() style:
+        bandit -r examples/ --format custom --msg-template \\
+        "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}"
+
+        See python documentation for more information about formatting style:
+        https://docs.python.org/3.4/library/string.html
+
+    The following tests were discovered and loaded:
+    -----------------------------------------------
+    ''')
+    parser.epilog = dedent_text + "\t{0}".format(plugin_list)
 
     # setup work - parse arguments, and initialize BanditManager
     args = parser.parse_args()
+    # Check if `--msg-template` is not present without custom formatter
+    if args.output_format != 'custom' and args.msg_template is not None:
+        parser.error("--msg-template can only be used with --format=custom")
 
     try:
         b_conf = b_config.BanditConfig(config_file=args.config_file)
@@ -341,7 +379,8 @@ def main():
                          sev_level,
                          conf_level,
                          args.output_file,
-                         args.output_format)
+                         args.output_format,
+                         args.msg_template)
 
     # return an exit code of 1 if there are results, 0 otherwise
     if b_mgr.results_count(sev_filter=sev_level, conf_filter=conf_level) > 0:

bandit/core/manager.py

@@ -136,7 +136,7 @@ class BanditManager(object):
         return len(self.get_issue_list(sev_filter, conf_filter))
 
     def output_results(self, lines, sev_level, conf_level, output_file,
-                       output_format):
+                       output_format, template=None):
         '''Outputs results from the result store
 
         :param lines: How many surrounding lines to show per result
@@ -144,6 +144,9 @@ class BanditManager(object):
         :param conf_level: Which confidence levels to show (LOW, MEDIUM, HIGH)
         :param output_file: File to store results
         :param output_format: output format plugin name
+        :param template: Output template with non-terminal tags <N>
+                         (default:  {abspath}:{line}:
+                         {test_id}[bandit]: {severity}: {msg})
         :return: -
         '''
         try:
@@ -153,8 +156,13 @@ class BanditManager(object):
 
             formatter = formatters_mgr[output_format]
             report_func = formatter.plugin
-            report_func(self, fileobj=output_file, sev_level=sev_level,
+            if output_format == 'custom':
-                        conf_level=conf_level, lines=lines)
+                report_func(self, fileobj=output_file, sev_level=sev_level,
+                            conf_level=conf_level, lines=lines,
+                            template=template)
+            else:
+                report_func(self, fileobj=output_file, sev_level=sev_level,
+                            conf_level=conf_level, lines=lines)
 
         except Exception as e:
             raise RuntimeError("Unable to output report using '%s' formatter: "

bandit/formatters/custom.py

@@ -0,0 +1,163 @@
+# Copyright (c) 2017 Hewlett Packard Enterprise
+# -*- coding:utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+r"""
+================
+Custom Formatter
+================
+
+This formatter outputs the issues in custom machine-readable format.
+
+default template: {abspath}:{line}: {test_id}[bandit]: {severity}: {msg}
+
+:Example:
+
+/usr/lib/python3.6/site-packages/openlp/core/utils/__init__.py: \
+405: B310[bandit]: MEDIUM: Audit url open for permitted schemes. \
+Allowing use of file:/ or custom schemes is often unexpected.
+
+"""
+
+import logging
+import os
+import re
+import string
+import sys
+
+from bandit.core import test_properties
+
+
+LOG = logging.getLogger(__name__)
+
+
+class SafeMapper(dict):
+    """Safe mapper to handle format key errors"""
+    @classmethod  # To prevent PEP8 warnings in the test suite
+    def __missing__(cls, key):
+        return "{%s}" % key
+
+
+@test_properties.accepts_baseline
+def report(manager, fileobj, sev_level, conf_level, lines=-1, template=None):
+    """Prints issues in custom format
+
+    :param manager: the bandit manager object
+    :param fileobj: The output file object, which may be sys.stdout
+    :param sev_level: Filtering severity level
+    :param conf_level: Filtering confidence level
+    :param lines: Number of lines to report, -1 for all
+    :param template: Output template with non-terminal tags <N>
+                    (default: '{abspath}:{line}:
+                    {test_id}[bandit]: {severity}: {msg}')
+    """
+
+    machine_output = {'results': [], 'errors': []}
+    for (fname, reason) in manager.get_skipped():
+        machine_output['errors'].append({'filename': fname,
+                                         'reason': reason})
+
+    results = manager.get_issue_list(sev_level=sev_level,
+                                     conf_level=conf_level)
+
+    msg_template = template
+    if template is None:
+        msg_template = "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}"
+
+    # Dictionary of non-terminal tags that will be expanded
+    tag_mapper = {
+        'abspath': lambda issue: os.path.abspath(issue.fname),
+        'relpath': lambda issue: os.path.relpath(issue.fname),
+        'line': lambda issue: issue.lineno,
+        'test_id': lambda issue: issue.test_id,
+        'severity': lambda issue: issue.severity,
+        'msg': lambda issue: issue.text,
+        'confidence': lambda issue: issue.confidence,
+        'range': lambda issue: issue.linerange
+    }
+
+    # Create dictionary with tag sets to speed up search for similar tags
+    tag_sim_dict = dict(
+        [(tag, set(tag)) for tag, _ in tag_mapper.items()]
+    )
+
+    # Parse the format_string template and check the validity of tags
+    try:
+        parsed_template_orig = list(string.Formatter().parse(msg_template))
+        # of type (literal_text, field_name, fmt_spec, conversion)
+
+        # Check the format validity only, ignore keys
+        string.Formatter().vformat(msg_template, (), SafeMapper(line=0))
+    except ValueError as e:
+        LOG.error("Template is not in valid format: %s", e.args[0])
+        sys.exit(2)
+
+    tag_set = {t[1] for t in parsed_template_orig if t[1] is not None}
+    if not tag_set:
+        LOG.error("No tags were found in the template. Are you missing '{}'?")
+        sys.exit(2)
+
+    def get_similar_tag(tag):
+        similarity_list = [(len(set(tag) & t_set), t)
+                           for t, t_set in tag_sim_dict.items()]
+        return sorted(similarity_list)[-1][1]
+
+    tag_blacklist = []
+    for tag in tag_set:
+        # check if the tag is in dictionary
+        if tag not in tag_mapper:
+            similar_tag = get_similar_tag(tag)
+            LOG.warning(
+                "Tag '%s' was not recognized and will be skipped, "
+                "did you mean to use '%s'?", tag, similar_tag
+            )
+            tag_blacklist += [tag]
+
+    # Compose the message template back with the valid values only
+    msg_parsed_template_list = []
+    for literal_text, field_name, fmt_spec, conversion in parsed_template_orig:
+        if literal_text:
+            # if there is '{' or '}', double it to prevent expansion
+            literal_text = re.sub('{', '{{', literal_text)
+            literal_text = re.sub('}', '}}', literal_text)
+            msg_parsed_template_list.append(literal_text)
+
+        if field_name is not None:
+            if field_name in tag_blacklist:
+                msg_parsed_template_list.append(field_name)
+                continue
+            # Append the fmt_spec part
+            params = [field_name, fmt_spec, conversion]
+            markers = ['', ':', '!']
+            msg_parsed_template_list.append(
+                ['{'] +
+                ["%s" % (m + p) if p else ''
+                 for m, p in zip(markers, params)] +
+                ['}']
+            )
+
+    msg_parsed_template = "".join([item for lst in msg_parsed_template_list
+                                   for item in lst]) + "\n"
+    limit = lines if lines > 0 else None
+    with fileobj:
+        for defect in results[:limit]:
+            evaluated_tags = SafeMapper(
+                (k, v(defect)) for k, v in tag_mapper.items()
+            )
+            output = msg_parsed_template.format(**evaluated_tags)
+
+            fileobj.write(output)
+
+    if fileobj.name != sys.stdout.name:
+        LOG.info("Result written to file: %s", fileobj.name)

doc/source/man/bandit.rst

@@ -7,8 +7,9 @@ SYNOPSIS
 
 bandit [-h] [-r] [-a {file,vuln}] [-n CONTEXT_LINES] [-c CONFIG_FILE]
             [-p PROFILE] [-t TESTS] [-s SKIPS] [-l] [-i]
-            [-f {csv,html,json,screen,txt,xml,yaml}] [-o OUTPUT_FILE] [-v]
+            [-f {csv,custom,html,json,screen,txt,xml,yaml}]
-            [-d] [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE]
+            [--msg-template MSG_TEMPLATE] [-o OUTPUT_FILE] [-v] [-d]
+            [--ignore-nosec] [-x EXCLUDED_PATHS] [-b BASELINE]
             [--ini INI_PATH] [--version]
             targets [targets ...]
 
@@ -43,8 +44,12 @@ OPTIONS
                         (-l for LOW, -ll for MEDIUM, -lll for HIGH)
   -i, --confidence      report only issues of a given confidence level or
                         higher (-i for LOW, -ii for MEDIUM, -iii for HIGH)
-  -f {csv,html,json,screen,txt,xml,yaml}, --format {csv,html,json,screen,txt,xml,yaml}
+  -f {csv,custom,html,json,screen,txt,xml,yaml}, --format {csv,custom,html,json,screen,txt,xml,yaml}
                         specify output format
+  --msg-template MSG_TEMPLATE
+                        specify output message template (only usable with
+                        --format custom), see CUSTOM FORMAT section for list
+                        of available values
   -o OUTPUT_FILE, --output OUTPUT_FILE
                         write report to filename
   -v, --verbose         output extra information like excluded and included
@@ -62,6 +67,30 @@ OPTIONS
                         arguments
   --version             show program's version number and exit
 
+CUSTOM FORMATTING
+-----------------
+
+Available tags:
+
+    {abspath}, {relpath}, {line},  {test_id},
+    {severity}, {msg}, {confidence}, {range}
+
+Example usage:
+
+    Default template:
+    bandit -r examples/ --format custom --msg-template \
+    "{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}"
+
+    Provides same output as:
+    bandit -r examples/ --format custom
+
+    Tags can also be formatted in python string.format() style:
+    bandit -r examples/ --format custom --msg-template \
+    "{relpath:20.20s}: {line:03}: {test_id:^8}: DEFECT: {msg:>20}"
+
+    See python documentation for more information about formatting style:
+    https://docs.python.org/3.4/library/string.html
+
 FILES
 =====
 

setup.cfg

@@ -37,6 +37,7 @@ bandit.formatters =
     html = bandit.formatters.html:report
     screen = bandit.formatters.screen:report
     yaml = bandit.formatters.yaml:report
+    custom = bandit.formatters.custom:report
 bandit.plugins =
     # bandit/plugins/app_debug.py
     flask_debug_true = bandit.plugins.app_debug:flask_debug_true

tests/functional/test_runtime.py

@@ -77,7 +77,7 @@ class RuntimeTests(testtools.TestCase):
         self.assertIn("tests were discovered and loaded:", output)
 
     def test_help_in_readme(self):
-        replace_list = [' ', '\t']
+        replace_list = [' ', '\t', '\n']
         (retcode, output) = self._test_runtime(['bandit', '-h'])
         for i in replace_list:
             output = output.replace(i, '')