Add a query language for group, inhibit, and silence rules

The new alarm rules will each have an expression in their
definition which will need to be parsed by both the Monasca-
API and the Monasca-Notification-Engine. Documentation for
this will be included in the API along with descriptions of the
new rules.

Story: 2000939
Task: 4692

Change-Id: I1a98fafae8dfdfa6fdb2eb66f4a4a4f40e518e46

monasca_common/monasca_query_language/aql_parser.py

@@ -0,0 +1,96 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import six
+import sys
+import time
+
+import pyparsing
+
+from monasca_common.monasca_query_language import query_structures
+
+COMMA = pyparsing.Suppress(pyparsing.Literal(","))
+LPAREN = pyparsing.Suppress(pyparsing.Literal("("))
+RPAREN = pyparsing.Suppress(pyparsing.Literal(")"))
+LBRACE = pyparsing.Suppress(pyparsing.Literal("{"))
+RBRACE = pyparsing.Suppress(pyparsing.Literal("}"))
+LBRACKET = pyparsing.Suppress(pyparsing.Literal("["))
+RBRACKET = pyparsing.Suppress(pyparsing.Literal("]"))
+
+MINUS = pyparsing.Literal("-")
+
+integer_number = pyparsing.Word(pyparsing.nums)
+decimal_number = (pyparsing.Optional(MINUS) + integer_number +
+                  pyparsing.Optional("." + integer_number))
+decimal_number.setParseAction(lambda tokens: float("".join(tokens)))
+
+# Initialize non-ascii unicode code points in the Basic Multilingual Plane.
+unicode_printables = u''.join(
+    six.unichr(c) for c in range(128, 65536) if not six.unichr(c).isspace())
+
+# Does not like comma. No Literals from above allowed.
+valid_identifier_chars = (
+    (unicode_printables + pyparsing.alphanums + ".-_#$%&'*+/:;?@[\\]^`|"))
+
+metric_name = (
+    pyparsing.Word(pyparsing.alphas, valid_identifier_chars, min=1, max=255)("metric_name"))
+dimension_name = pyparsing.Word(valid_identifier_chars + ' ', min=1, max=255)
+dimension_value = pyparsing.Word(valid_identifier_chars + ' ', min=1, max=255)
+
+dim_comparison_op = pyparsing.oneOf("=")
+
+dimension = dimension_name + dim_comparison_op + dimension_value
+dimension.setParseAction(query_structures.Dimension)
+
+dimension_list = pyparsing.Group((LBRACE + pyparsing.Optional(
+    pyparsing.delimitedList(dimension)) +
+                                  RBRACE))
+
+metric = (metric_name + pyparsing.Optional(dimension_list) |
+          pyparsing.Optional(metric_name) + dimension_list)
+metric.addParseAction(query_structures.MetricSelector)
+
+source = pyparsing.Keyword("source")
+source_expression = source + metric
+source_expression.addParseAction(query_structures.SourceExpression)
+
+targets = pyparsing.Keyword("targets")
+targets_expression = targets + metric
+targets_expression.addParseAction(query_structures.TargetsExpression)
+
+excludes = pyparsing.Keyword("excluding")
+excludes_expression = excludes + metric
+excludes_expression.addParseAction(query_structures.ExcludesExpression)
+
+group_by = pyparsing.Keyword("group by")
+group_by_expr = group_by + pyparsing.delimitedList(dimension_name)
+group_by_expr.addParseAction(query_structures.GroupByExpression)
+
+grammar = (pyparsing.Optional(source_expression) +
+           pyparsing.Optional(targets_expression) +
+           pyparsing.Optional(excludes_expression) +
+           pyparsing.Optional(group_by_expr))
+grammar.addParseAction(query_structures.Rule)
+
+
+class RuleExpressionParser(object):
+
+    def __init__(self, expr):
+        self._expr = expr
+
+    def parse(self):
+        parse_result = grammar.parseString(self._expr, parseAll=True)
+        return parse_result

monasca_common/monasca_query_language/exceptions.py

@@ -0,0 +1,18 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class InvalidExpressionException(Exception):
+    pass

monasca_common/monasca_query_language/query_structures.py

@@ -0,0 +1,186 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pyparsing
+
+from monasca_common.monasca_query_language import exceptions
+
+
+class Dimension(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.key = tokens[0]
+        self.operator = tokens[1]
+        self.value = tokens[2]
+
+    def __str__(self):
+        return "Dimension(key={},operator='{}',value={})".format(
+            self.key, self.operator, self.value)
+
+
+class MetricSelector(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.name = None
+        self.dimensions = {}
+        _dimensions = []
+        for token in tokens:
+            if isinstance(token, str):
+                self.name = token
+            elif isinstance(token, pyparsing.ParseResults):
+                _dimensions = token
+
+        for dim in _dimensions:
+            self.dimensions[dim.key] = dim.value
+        if self.name is not None:
+            self.dimensions["__metricName__"] = self.name
+
+    def get_filters(self):
+        return self.dimensions
+
+    def __repr__(self):
+        return "MetricSelector(name={},dimensions={})".format(
+            self.name, self.dimensions)
+
+    def __str__(self):
+        return self.__repr__()
+
+
+class LogicalExpression(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.left_operand = tokens[0][0]
+        self.operator = None
+        self.right_operand = None
+        if len(tokens[0]) > 1:
+            self.operator = tokens[0][1]
+        if len(tokens[0]) > 2:
+            self.right_operand = tokens[0][2]
+
+    @property
+    def normalized_operator(self):
+        if self.operator == '&&':
+            result = 'and'
+        elif self.operator == '||':
+            result = 'or'
+        else:
+            result = self.operator
+        return result
+
+    def get_filters(self):
+        left_filters = self.left_operand.get_filters()
+        right_filters = self.right_operand.get_filters()
+        for key, value in right_filters.items():
+            if key in left_filters and left_filters[key] != value:
+                raise exceptions.InvalidExpressionException(
+                    "Duplicate keys specified ".format(key))
+            left_filters[key] = value
+        return left_filters
+
+    def __str__(self):
+        return "LogicalExpression(left={},operator='{}',right={})".format(
+            self.left_operand, self.operator, self.right_operand)
+        return self.__repr__()
+
+
+class SourceExpression(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.source = tokens[1]
+
+    def get_filters(self):
+        return self.source.get_filters()
+
+    def __str__(self):
+        return "SourceExpression(source={})".format(self.source)
+
+
+class TargetsExpression(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.target = tokens[1]
+
+    def get_filters(self):
+        return self.target.get_filters()
+
+    def __str__(self):
+        return "TargetExpression(target={})".format(self.target)
+
+
+class ExcludesExpression(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.exclude = tokens[1]
+
+    def get_filters(self):
+        return self.exclude.get_filters()
+
+    def __str__(self):
+        return "ExcludesExpression(exclude={})".format(self.exclude)
+
+
+class GroupByExpression(object):
+    def __init__(self, tokens):
+        self.args = tokens
+        self.group_keys = tokens[1:]
+
+    def get_filters(self):
+        return self.group_keys
+
+    def __str__(self):
+        return "GroupByExpression({})".format(self.group_keys)
+
+
+class Rule(object):
+    def __init__(self, tokens):
+        self.source = None
+        self.target = None
+        self.excludes = None
+        self.group_by = None
+        for token in tokens:
+            if isinstance(token, SourceExpression):
+                self.source = token
+            elif isinstance(token, TargetsExpression):
+                self.target = token
+            elif isinstance(token, ExcludesExpression):
+                self.excludes = token
+            elif isinstance(token, GroupByExpression):
+                self.group_by = token
+
+    def get_struct(self, _type):
+        result = {}
+        if _type == "silence":
+            result['matchers'] = self.target.get_filters() if self.target is not None else {}
+            if any([self.source, self.group_by, self.excludes]):
+                raise exceptions.InvalidExpressionException(
+                    "Silence rule contains unexpected elements")
+        elif _type == "inhibit":
+            result['source_match'] = self.source.get_filters() if self.source is not None else {}
+            result['target_match'] = self.target.get_filters() if self.target is not None else {}
+            result['equal'] = self.group_by.get_filters() if self.group_by is not None else []
+            result['exclusions'] = self.excludes.get_filters() if self.excludes is not None else {}
+        elif _type == "group":
+            result['matchers'] = self.group_by.get_filters() if self.group_by is not None else []
+            result['exclusions'] = self.excludes.get_filters() if self.excludes is not None else {}
+            if any([self.source, self.target]):
+                raise exceptions.InvalidExpressionException(
+                    "Group rule contains unexpected elements")
+        else:
+            raise exceptions.InvalidExpressionException("Unknown type for expression")
+        return result
+
+    def __str__(self):
+        return "Rule(source={},target={},excludes={},group_by={})".format(
+            self.source, self.target, self.excludes, self.group_by)

monasca_common/tests/test_monasca_query_language.py

@@ -0,0 +1,186 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslotest import base
+import pyparsing
+
+from monasca_common.monasca_query_language import aql_parser
+from monasca_common.monasca_query_language import exceptions
+from monasca_common.monasca_query_language import query_structures
+
+
+class TestMonascaQueryLanguage(base.BaseTestCase):
+
+    def test_parse_group_expression(self):
+        expressions = [
+            "",
+            "excluding metric_two",
+            "group by hostname, service",
+            "excluding metric_two group by hostname, service",
+            "group by __severity__",
+            "excluding {__severity__=HIGH} group by __severity__",
+            "excluding {__severity__=HIGH, hostname=host1} group by __severity__, hostname",
+            "group by excluding"  # excluding is an acceptable metric name
+        ]
+        negative_expressions = [
+            "group by hostname excluding {__metricName__=metric_two}",
+            "excluding metric_one excluding metric_two",
+            "targets metric_one",
+        ]
+        matchers = [
+            [],
+            [],
+            ["hostname", "service"],
+            ["hostname", "service"],
+            ["__severity__"],
+            ["__severity__"],
+            ["__severity__", "hostname"],
+            ["excluding"]
+        ]
+        exclusions = [
+            {},
+            {"__metricName__": "metric_two"},
+            {},
+            {"__metricName__": "metric_two"},
+            {},
+            {"__severity__": "HIGH"},
+            {"__severity__": "HIGH", "hostname": "host1"},
+            {},
+        ]
+        for i in range(len(expressions)):
+            result = aql_parser.RuleExpressionParser(expressions[i]).parse()
+            result = result[0].get_struct("group")
+            self.assertEqual(result['matchers'], matchers[i])
+            self.assertEqual(result['exclusions'], exclusions[i])
+        for negative_expression in negative_expressions:
+            try:
+                result = aql_parser.RuleExpressionParser(negative_expression)
+                self.assertRaises(exceptions.InvalidExpressionException,
+                                  result.parse())
+            except TypeError:
+                pass
+            except pyparsing.ParseException:
+                pass
+
+    def test_parse_inhibit_rule(self):
+        expressions = [
+            "",
+            "source metric_one",
+            "targets metric_two",
+            "source metric_one targets metric_two",
+            "source metric_one targets metric_two excluding metric_three",
+            "source metric_one targets metric_two excluding metric_three group by hostname",
+            "source metric_one targets metric_two group by hostname",
+            "source metric_one group by hostname",
+            "source {__severity__=HIGH} targets {__severity__=LOW} excluding "
+                "{__alarmName__=alarm_one} group by __alarmName__"
+        ]
+        negative_expressions = [
+            "targets metric_two source_metric_one"
+        ]
+        source = [
+            {},
+            {"__metricName__": "metric_one"},
+            {},
+            {"__metricName__": "metric_one"},
+            {"__metricName__": "metric_one"},
+            {"__metricName__": "metric_one"},
+            {"__metricName__": "metric_one"},
+            {"__metricName__": "metric_one"},
+            {"__severity__": "HIGH"},
+        ]
+        target = [
+            {},
+            {},
+            {"__metricName__": "metric_two"},
+            {"__metricName__": "metric_two"},
+            {"__metricName__": "metric_two"},
+            {"__metricName__": "metric_two"},
+            {"__metricName__": "metric_two"},
+            {},
+            {"__severity__": "LOW"}
+        ]
+        equals = [
+            [],
+            [],
+            [],
+            [],
+            [],
+            ["hostname"],
+            ["hostname"],
+            ["hostname"],
+            ["__alarmName__"]
+        ]
+        exclusions = [
+            {},
+            {},
+            {},
+            {},
+            {"__metricName__": "metric_three"},
+            {"__metricName__": "metric_three"},
+            {},
+            {},
+            {"__alarmName__": "alarm_one"}
+        ]
+        for i in range(len(expressions)):
+            result = aql_parser.RuleExpressionParser(expressions[i]).parse()
+            result = result[0].get_struct("inhibit")
+            self.assertEqual(result['source_match'], source[i])
+            self.assertEqual(result['target_match'], target[i])
+            self.assertEqual(result['equal'], equals[i])
+            self.assertEqual(result['exclusions'], exclusions[i])
+
+        for expression in negative_expressions:
+            try:
+                result = aql_parser.RuleExpressionParser(expression)
+                self.assertRaises(exceptions.InvalidExpressionException,
+                                  result.parse())
+            except pyparsing.ParseException:
+                pass
+
+    def test_parse_silence_rule(self):
+        expressions = [
+            "",
+            "targets metric_one",
+            "targets metric_one{}",
+            "targets metric_one{hostname=host_one}",
+            "targets metric_one{hostname=host_one, region=region_one}",
+        ]
+        negative_expressions = [
+            "excludes metric_one",
+            "source metric_one",
+            "group by hostname",
+            "targets metric_one, {hostname=host_one}",
+        ]
+        matchers = [
+            {},
+            {"__metricName__": "metric_one"},
+            {"__metricName__": "metric_one"},
+            {"__metricName__": "metric_one", "hostname": "host_one"},
+            {"__metricName__": "metric_one", "hostname": "host_one", "region": "region_one"},
+        ]
+        for i in range(len(expressions)):
+            result = aql_parser.RuleExpressionParser(expressions[i]).parse()
+            result = result[0].get_struct("silence")
+            self.assertEqual(result['matchers'], matchers[i])
+
+        for expression in negative_expressions:
+            try:
+                self.assertRaises(exceptions.InvalidExpressionException,
+                                  aql_parser.RuleExpressionParser(expression).parse())
+            except TypeError:
+                pass
+            except pyparsing.ParseException:
+                pass

requirements.txt

@@ -7,4 +7,5 @@ pykafka>=2.5.0 # Apache 2.0 License
 PyMySQL>=0.7.6 # MIT License
 oslo.config>=4.0.0 # Apache-2.0
 pbr!=2.1.0,>=2.0.0 # Apache-2.0
+pyparsing>=2.1.0 # MIT
 ujson>=1.35 # BSD