From e477e023506548c4d24ef30b0c2740e0a98c1731 Mon Sep 17 00:00:00 2001 From: Andrew Krasichkov Date: Fri, 19 May 2017 19:31:20 +0300 Subject: [PATCH] Support multiple config files (#47) --- gixy/cli/main.py | 57 +++++++++++++++++----------- gixy/core/manager.py | 16 +++++--- gixy/formatters/base.py | 19 +++++++--- gixy/formatters/console.py | 3 +- gixy/formatters/json.py | 16 +++++++- gixy/formatters/templates/console.j2 | 34 ++++++++++------- gixy/formatters/templates/text.j2 | 32 ++++++++++------ gixy/formatters/text.py | 3 +- tests/plugins/test_simply.py | 9 ++--- tests/utils.py | 5 --- 10 files changed, 122 insertions(+), 72 deletions(-) diff --git a/gixy/cli/main.py b/gixy/cli/main.py index 64e9200..4155039 100644 --- a/gixy/cli/main.py +++ b/gixy/cli/main.py @@ -35,7 +35,7 @@ def _create_plugin_help(option): def _get_cli_parser(): parser = create_parser() - parser.add_argument('nginx_file', nargs='?', type=str, default='/etc/nginx/nginx.conf', metavar='nginx.conf', + parser.add_argument('nginx_files', nargs='*', type=str, default=['/etc/nginx/nginx.conf'], metavar='nginx.conf', help='Path to nginx.conf, e.g. /etc/nginx/nginx.conf') parser.add_argument( @@ -96,11 +96,12 @@ def main(): args = parser.parse_args() _init_logger(args.debug) - path = os.path.expanduser(args.nginx_file) - if path != '-' and not os.path.exists(path): - sys.stderr.write('Please specify path to Nginx configuration.\n\n') - parser.print_help() - sys.exit(1) + if len(args.nginx_files) == 1 and args.nginx_files[0] != '-': + path = os.path.expanduser(args.nginx_files[0]) + if not os.path.exists(path): + sys.stderr.write('File {path!r} was not found.\nPlease specify correct path to configuration.\n'.format( + path=path)) + sys.exit(1) try: severity = gixy.severity.ALL[args.level] @@ -148,22 +149,32 @@ def main(): options[opt_key] = val config.set_for(name, options) - with Gixy(config=config) as yoda: - if path == '-': - with os.fdopen(sys.stdin.fileno(), 'rb') as fdata: - yoda.audit('', fdata, is_stdin=True) - else: - with open(path, mode='rb') as fdata: - yoda.audit(path, fdata, is_stdin=False) + formatter = formatters()[config.output_format]() + failed = False + for input_path in args.nginx_files: + path = os.path.abspath(os.path.expanduser(input_path)) + if not os.path.exists(path): + LOG.error('File %s was not found', path) + continue - formatted = formatters()[config.output_format]().format(yoda) - if args.output_file: - with open(config.output_file, 'w') as f: - f.write(formatted) - else: - print(formatted) + with Gixy(config=config) as yoda: + if path == '-': + with os.fdopen(sys.stdin.fileno(), 'rb') as fdata: + yoda.audit('', fdata, is_stdin=True) + else: + with open(path, mode='rb') as fdata: + yoda.audit(path, fdata, is_stdin=False) - if sum(yoda.stats.values()) > 0: - # If something found - exit code must be 1, otherwise 0 - sys.exit(1) - sys.exit(0) + formatter.feed(path, yoda) + failed = failed or sum(yoda.stats.values()) > 0 + + if args.output_file: + with open(config.output_file, 'w') as f: + f.write(formatter.flush()) + else: + print(formatter.flush()) + + if failed: + # If something found - exit code must be 1, otherwise 0 + sys.exit(1) + sys.exit(0) diff --git a/gixy/core/manager.py b/gixy/core/manager.py index 1147382..0f4b1ed 100644 --- a/gixy/core/manager.py +++ b/gixy/core/manager.py @@ -15,10 +15,6 @@ class Manager(object): self.root = None self.config = config or Config() self.auditor = PluginsManager(config=self.config) - self.stats = {gixy.severity.UNSPECIFIED: 0, - gixy.severity.LOW: 0, - gixy.severity.MEDIUM: 0, - gixy.severity.HIGH: 0} def audit(self, file_path, file_data, is_stdin=False): LOG.debug("Audit config file: {fname}".format(fname=file_path)) @@ -30,12 +26,20 @@ class Manager(object): push_context(self.root) self._audit_recursive(self.root.children) - def get_results(self): + @property + def results(self): for plugin in self.auditor.plugins: if plugin.issues: - self.stats[plugin.severity] += len(plugin.issues) yield plugin + @property + def stats(self): + stats = dict.fromkeys(gixy.severity.ALL, 0) + for plugin in self.auditor.plugins: + if plugin.issues: + stats[plugin.severity] += len(plugin.issues) + return stats + def _audit_recursive(self, tree): for directive in tree: self._update_variables(directive) diff --git a/gixy/formatters/base.py b/gixy/formatters/base.py index f4b0d1e..e441407 100644 --- a/gixy/formatters/base.py +++ b/gixy/formatters/base.py @@ -1,17 +1,25 @@ from __future__ import absolute_import +import gixy from gixy.directives import block class BaseFormatter(object): skip_parents = set([block.Root, block.HttpBlock]) + def __init__(self): + self.reports = {} + self.stats = dict.fromkeys(gixy.severity.ALL, 0) + def format_reports(self, reports, stats): raise NotImplementedError("Formatter must override format_reports function") - def format(self, manager): - reports = [] - for result in manager.get_results(): + def feed(self, path, manager): + for severity in gixy.severity.ALL: + self.stats[severity] += manager.stats[severity] + + self.reports[path] = [] + for result in manager.results: report = self._prepare_result(manager.root, summary=result.summary, severity=result.severity, @@ -19,9 +27,10 @@ class BaseFormatter(object): issues=result.issues, plugin=result.name, help_url=result.help_url) - reports.extend(report) + self.reports[path].extend(report) - return self.format_reports(reports, manager.stats) + def flush(self): + return self.format_reports(self.reports, self.stats) def _prepare_result(self, root, issues, severity, summary, description, plugin, help_url): result = {} diff --git a/gixy/formatters/console.py b/gixy/formatters/console.py index 560ab66..18016c7 100644 --- a/gixy/formatters/console.py +++ b/gixy/formatters/console.py @@ -6,8 +6,9 @@ from gixy.formatters.base import BaseFormatter class ConsoleFormatter(BaseFormatter): def __init__(self): + super(ConsoleFormatter, self).__init__() env = Environment(loader=PackageLoader('gixy', 'formatters/templates'), trim_blocks=True, lstrip_blocks=True) self.template = env.get_template('console.j2') def format_reports(self, reports, stats): - return self.template.render(issues=reports, stats=stats) + return self.template.render(reports=reports, stats=stats) diff --git a/gixy/formatters/json.py b/gixy/formatters/json.py index b80e3e2..cf5c29f 100644 --- a/gixy/formatters/json.py +++ b/gixy/formatters/json.py @@ -7,4 +7,18 @@ from gixy.formatters.base import BaseFormatter class JsonFormatter(BaseFormatter): def format_reports(self, reports, stats): - return json.dumps(reports, sort_keys=True, indent=2, separators=(',', ': ')) + result = [] + for path, issues in reports.items(): + for issue in issues: + result.append(dict( + path=path, + plugin=issue['plugin'], + summary=issue['summary'], + severity=issue['severity'], + description=issue['description'], + reference=issue['help_url'], + reason=issue['reason'], + config=issue['config'] + )) + + return json.dumps(result, sort_keys=True, indent=2, separators=(',', ': ')) diff --git a/gixy/formatters/templates/console.j2 b/gixy/formatters/templates/console.j2 index 067f4ed..01e953d 100644 --- a/gixy/formatters/templates/console.j2 +++ b/gixy/formatters/templates/console.j2 @@ -1,35 +1,43 @@ {% set colors = {'DEF': '\033[0m', 'TITLE': '\033[95m', 'UNSPECIFIED': '\033[0m', 'LOW': '\033[94m', 'MEDIUM': '\033[93m', 'HIGH': '\033[91m'} %} -{{ colors['TITLE'] }}==================== Results ==================={{ colors['DEF'] }} +{{ colors.TITLE }}==================== Results ==================={{ colors.DEF }} +{% for path, issues in reports.items() %} +File path: {{ path }} {% if not issues %} No issues found. + {% else %} {% for issue in issues|sort(attribute='severity') %} -{{ colors[issue.severity] }}Problem: [{{ issue.plugin }}] {{ issue.summary }} +{{ colors[issue.severity] }}>> Problem: [{{ issue.plugin }}] {{ issue.summary }} {% if issue.description %} -Description: {{ issue.description }} + Description: {{ issue.description }} {% endif %} {% if issue.help_url %} -Additional info: {{ issue.help_url }} + Additional info: {{ issue.help_url }} {% endif %} {% if issue.reason %} -Reason: {{ issue.reason }} + Reason: {{ issue.reason }} {% endif %} -{{ colors['DEF'] }}Pseudo config:{{ issue.config }} + Pseudo config:{{ colors.DEF }} +{{ issue.config }} +{% if not loop.last %} +------------------------------------------------ + +{% endif %} +{% endfor %} +{% endif %} {% if not loop.last %} --------8<--------8<--------8<--------8<-------- {% endif %} {% endfor %} -{% endif %} - {% if stats %} -{{ colors['TITLE'] }}==================== Summary ==================={{ colors['DEF'] }} +{{ colors.TITLE }}==================== Summary ==================={{ colors.DEF }} Total issues: - Unspecified: {{ stats['UNSPECIFIED'] }} - Low: {{ stats['LOW'] }} - Medium: {{ stats['MEDIUM'] }} - High: {{ stats['HIGH'] }} + Unspecified: {{ stats.UNSPECIFIED }} + Low: {{ stats.LOW }} + Medium: {{ stats.MEDIUM }} + High: {{ stats.HIGH }} {% endif %} diff --git a/gixy/formatters/templates/text.j2 b/gixy/formatters/templates/text.j2 index 0f0362a..c2ef679 100644 --- a/gixy/formatters/templates/text.j2 +++ b/gixy/formatters/templates/text.j2 @@ -1,35 +1,43 @@ ==================== Results =================== +{% for path, issues in reports.items() %} +File path: {{ path }} {% if not issues %} No issues found. + {% else %} {% for issue in issues|sort(attribute='severity') %} -Problem: [{{ issue.plugin }}] {{ issue.summary }} -Severity: {{ issue.severity }} +>> Problem: [{{ issue.plugin }}] {{ issue.summary }} + Severity: {{ issue.severity }} {% if issue.description %} -Description: {{ issue.description }} + Description: {{ issue.description }} {% endif %} {% if issue.help_url %} -Additional info: {{ issue.help_url }} + Additional info: {{ issue.help_url }} {% endif %} {% if issue.reason %} -Reason: {{ issue.reason }} + Reason: {{ issue.reason }} {% endif %} -Pseudo config: {{ issue.config }} + Pseudo config: +{{ issue.config }} +{% if not loop.last %} +------------------------------------------------ + +{% endif %} +{% endfor %} +{% endif %} {% if not loop.last %} --------8<--------8<--------8<--------8<-------- {% endif %} {% endfor %} -{% endif %} - {% if stats %} ==================== Summary =================== Total issues: - Unspecified: {{ stats['UNSPECIFIED'] }} - Low: {{ stats['LOW'] }} - Medium: {{ stats['MEDIUM'] }} - High: {{ stats['HIGH'] }} + Unspecified: {{ stats.UNSPECIFIED }} + Low: {{ stats.LOW }} + Medium: {{ stats.MEDIUM }} + High: {{ stats.HIGH }} {% endif %} diff --git a/gixy/formatters/text.py b/gixy/formatters/text.py index addfe07..9520d05 100644 --- a/gixy/formatters/text.py +++ b/gixy/formatters/text.py @@ -6,8 +6,9 @@ from gixy.formatters.base import BaseFormatter class TextFormatter(BaseFormatter): def __init__(self): + super(TextFormatter, self).__init__() env = Environment(loader=PackageLoader('gixy', 'formatters/templates'), trim_blocks=True, lstrip_blocks=True) self.template = env.get_template('text.j2') def format_reports(self, reports, stats): - return self.template.render(issues=reports, stats=stats) + return self.template.render(reports=reports, stats=stats) diff --git a/tests/plugins/test_simply.py b/tests/plugins/test_simply.py index f8da9e3..1a33c63 100644 --- a/tests/plugins/test_simply.py +++ b/tests/plugins/test_simply.py @@ -4,7 +4,6 @@ import os from os import path import json -import gixy from ..utils import * from gixy.core.manager import Manager as Gixy from gixy.core.plugins_manager import PluginsManager @@ -83,7 +82,9 @@ def check_configuration(plugin, config_path, test_config): plugin_options = parse_plugin_options(config_path) with yoda_provider(plugin, plugin_options) as yoda: yoda.audit(config_path, open(config_path, mode='r')) - results = RawFormatter().format(yoda) + formatter = BaseFormatter() + formatter.feed(config_path, yoda) + _, results = formatter.reports.popitem() assert_equals(len(results), 1, 'Should have one report') result = results[0] @@ -104,7 +105,5 @@ def check_configuration(plugin, config_path, test_config): def check_configuration_fp(plugin, config_path, test_config): with yoda_provider(plugin) as yoda: yoda.audit(config_path, open(config_path, mode='r')) - results = RawFormatter().format(yoda) - - assert_equals(len(results), 0, + assert_equals(len([x for x in yoda.results]), 0, 'False positive configuration must not trigger any plugins') diff --git a/tests/utils.py b/tests/utils.py index c542b99..6d1494d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -62,8 +62,3 @@ class Matcher(object): else: result = dv.find(v) >= 0 return result - - -class RawFormatter(BaseFormatter): - def format_reports(self, reports, stats): - return reports