From 41800dd195efaf99119b44a6681ec36ea7bcde38 Mon Sep 17 00:00:00 2001
From: Andrea Adams <aadams@hpe.com>
Date: Fri, 2 Jun 2017 13:39:21 -0600
Subject: [PATCH] Add a query language for group, inhibit, and silence rules
The new alarm rules will each have an expression in their
definition which will need to be parsed by both the Monasca-
API and the Monasca-Notification-Engine. Documentation for
this will be included in the API along with descriptions of the
new rules.
Story: 2000939
Task: 4692
Change-Id: I1a98fafae8dfdfa6fdb2eb66f4a4a4f40e518e46
---
.../monasca_query_language/__init__.py | 0
.../monasca_query_language/aql_parser.py | 96 +++++++++
.../monasca_query_language/exceptions.py | 18 ++
.../query_structures.py | 186 ++++++++++++++++++
.../tests/test_monasca_query_language.py | 186 ++++++++++++++++++
requirements.txt | 1 +
6 files changed, 487 insertions(+)
create mode 100644 monasca_common/monasca_query_language/__init__.py
create mode 100644 monasca_common/monasca_query_language/aql_parser.py
create mode 100644 monasca_common/monasca_query_language/exceptions.py
create mode 100644 monasca_common/monasca_query_language/query_structures.py
create mode 100644 monasca_common/tests/test_monasca_query_language.py
diff --git a/monasca_common/monasca_query_language/__init__.py b/monasca_common/monasca_query_language/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/monasca_common/monasca_query_language/aql_parser.py b/monasca_common/monasca_query_language/aql_parser.py
new file mode 100644
index 00000000..da741881
--- /dev/null
+++ b/monasca_common/monasca_query_language/aql_parser.py
@@ -0,0 +1,96 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import six
+import sys
+import time
+
+import pyparsing
+
+from monasca_common.monasca_query_language import query_structures
+
+COMMA = pyparsing.Suppress(pyparsing.Literal(","))
+LPAREN = pyparsing.Suppress(pyparsing.Literal("("))
+RPAREN = pyparsing.Suppress(pyparsing.Literal(")"))
+LBRACE = pyparsing.Suppress(pyparsing.Literal("{"))
+RBRACE = pyparsing.Suppress(pyparsing.Literal("}"))
+LBRACKET = pyparsing.Suppress(pyparsing.Literal("["))
+RBRACKET = pyparsing.Suppress(pyparsing.Literal("]"))
+
+MINUS = pyparsing.Literal("-")
+
+integer_number = pyparsing.Word(pyparsing.nums)
+decimal_number = (pyparsing.Optional(MINUS) + integer_number +
+ pyparsing.Optional("." + integer_number))
+decimal_number.setParseAction(lambda tokens: float("".join(tokens)))
+
+# Initialize non-ascii unicode code points in the Basic Multilingual Plane.
+unicode_printables = u''.join(
+ six.unichr(c) for c in range(128, 65536) if not six.unichr(c).isspace())
+
+# Does not like comma. No Literals from above allowed.
+valid_identifier_chars = (
+ (unicode_printables + pyparsing.alphanums + ".-_#$%&'*+/:;?@[\\]^`|"))
+
+metric_name = (
+ pyparsing.Word(pyparsing.alphas, valid_identifier_chars, min=1, max=255)("metric_name"))
+dimension_name = pyparsing.Word(valid_identifier_chars + ' ', min=1, max=255)
+dimension_value = pyparsing.Word(valid_identifier_chars + ' ', min=1, max=255)
+
+dim_comparison_op = pyparsing.oneOf("=")
+
+dimension = dimension_name + dim_comparison_op + dimension_value
+dimension.setParseAction(query_structures.Dimension)
+
+dimension_list = pyparsing.Group((LBRACE + pyparsing.Optional(
+ pyparsing.delimitedList(dimension)) +
+ RBRACE))
+
+metric = (metric_name + pyparsing.Optional(dimension_list) |
+ pyparsing.Optional(metric_name) + dimension_list)
+metric.addParseAction(query_structures.MetricSelector)
+
+source = pyparsing.Keyword("source")
+source_expression = source + metric
+source_expression.addParseAction(query_structures.SourceExpression)
+
+targets = pyparsing.Keyword("targets")
+targets_expression = targets + metric
+targets_expression.addParseAction(query_structures.TargetsExpression)
+
+excludes = pyparsing.Keyword("excluding")
+excludes_expression = excludes + metric
+excludes_expression.addParseAction(query_structures.ExcludesExpression)
+
+group_by = pyparsing.Keyword("group by")
+group_by_expr = group_by + pyparsing.delimitedList(dimension_name)
+group_by_expr.addParseAction(query_structures.GroupByExpression)
+
+grammar = (pyparsing.Optional(source_expression) +
+ pyparsing.Optional(targets_expression) +
+ pyparsing.Optional(excludes_expression) +
+ pyparsing.Optional(group_by_expr))
+grammar.addParseAction(query_structures.Rule)
+
+
+class RuleExpressionParser(object):
+
+ def __init__(self, expr):
+ self._expr = expr
+
+ def parse(self):
+ parse_result = grammar.parseString(self._expr, parseAll=True)
+ return parse_result
diff --git a/monasca_common/monasca_query_language/exceptions.py b/monasca_common/monasca_query_language/exceptions.py
new file mode 100644
index 00000000..1da5c25f
--- /dev/null
+++ b/monasca_common/monasca_query_language/exceptions.py
@@ -0,0 +1,18 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class InvalidExpressionException(Exception):
+ pass
diff --git a/monasca_common/monasca_query_language/query_structures.py b/monasca_common/monasca_query_language/query_structures.py
new file mode 100644
index 00000000..2b689d96
--- /dev/null
+++ b/monasca_common/monasca_query_language/query_structures.py
@@ -0,0 +1,186 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pyparsing
+
+from monasca_common.monasca_query_language import exceptions
+
+
+class Dimension(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.key = tokens[0]
+ self.operator = tokens[1]
+ self.value = tokens[2]
+
+ def __str__(self):
+ return "Dimension(key={},operator='{}',value={})".format(
+ self.key, self.operator, self.value)
+
+
+class MetricSelector(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.name = None
+ self.dimensions = {}
+ _dimensions = []
+ for token in tokens:
+ if isinstance(token, str):
+ self.name = token
+ elif isinstance(token, pyparsing.ParseResults):
+ _dimensions = token
+
+ for dim in _dimensions:
+ self.dimensions[dim.key] = dim.value
+ if self.name is not None:
+ self.dimensions["__metricName__"] = self.name
+
+ def get_filters(self):
+ return self.dimensions
+
+ def __repr__(self):
+ return "MetricSelector(name={},dimensions={})".format(
+ self.name, self.dimensions)
+
+ def __str__(self):
+ return self.__repr__()
+
+
+class LogicalExpression(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.left_operand = tokens[0][0]
+ self.operator = None
+ self.right_operand = None
+ if len(tokens[0]) > 1:
+ self.operator = tokens[0][1]
+ if len(tokens[0]) > 2:
+ self.right_operand = tokens[0][2]
+
+ @property
+ def normalized_operator(self):
+ if self.operator == '&&':
+ result = 'and'
+ elif self.operator == '||':
+ result = 'or'
+ else:
+ result = self.operator
+ return result
+
+ def get_filters(self):
+ left_filters = self.left_operand.get_filters()
+ right_filters = self.right_operand.get_filters()
+ for key, value in right_filters.items():
+ if key in left_filters and left_filters[key] != value:
+ raise exceptions.InvalidExpressionException(
+ "Duplicate keys specified ".format(key))
+ left_filters[key] = value
+ return left_filters
+
+ def __str__(self):
+ return "LogicalExpression(left={},operator='{}',right={})".format(
+ self.left_operand, self.operator, self.right_operand)
+ return self.__repr__()
+
+
+class SourceExpression(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.source = tokens[1]
+
+ def get_filters(self):
+ return self.source.get_filters()
+
+ def __str__(self):
+ return "SourceExpression(source={})".format(self.source)
+
+
+class TargetsExpression(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.target = tokens[1]
+
+ def get_filters(self):
+ return self.target.get_filters()
+
+ def __str__(self):
+ return "TargetExpression(target={})".format(self.target)
+
+
+class ExcludesExpression(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.exclude = tokens[1]
+
+ def get_filters(self):
+ return self.exclude.get_filters()
+
+ def __str__(self):
+ return "ExcludesExpression(exclude={})".format(self.exclude)
+
+
+class GroupByExpression(object):
+ def __init__(self, tokens):
+ self.args = tokens
+ self.group_keys = tokens[1:]
+
+ def get_filters(self):
+ return self.group_keys
+
+ def __str__(self):
+ return "GroupByExpression({})".format(self.group_keys)
+
+
+class Rule(object):
+ def __init__(self, tokens):
+ self.source = None
+ self.target = None
+ self.excludes = None
+ self.group_by = None
+ for token in tokens:
+ if isinstance(token, SourceExpression):
+ self.source = token
+ elif isinstance(token, TargetsExpression):
+ self.target = token
+ elif isinstance(token, ExcludesExpression):
+ self.excludes = token
+ elif isinstance(token, GroupByExpression):
+ self.group_by = token
+
+ def get_struct(self, _type):
+ result = {}
+ if _type == "silence":
+ result['matchers'] = self.target.get_filters() if self.target is not None else {}
+ if any([self.source, self.group_by, self.excludes]):
+ raise exceptions.InvalidExpressionException(
+ "Silence rule contains unexpected elements")
+ elif _type == "inhibit":
+ result['source_match'] = self.source.get_filters() if self.source is not None else {}
+ result['target_match'] = self.target.get_filters() if self.target is not None else {}
+ result['equal'] = self.group_by.get_filters() if self.group_by is not None else []
+ result['exclusions'] = self.excludes.get_filters() if self.excludes is not None else {}
+ elif _type == "group":
+ result['matchers'] = self.group_by.get_filters() if self.group_by is not None else []
+ result['exclusions'] = self.excludes.get_filters() if self.excludes is not None else {}
+ if any([self.source, self.target]):
+ raise exceptions.InvalidExpressionException(
+ "Group rule contains unexpected elements")
+ else:
+ raise exceptions.InvalidExpressionException("Unknown type for expression")
+ return result
+
+ def __str__(self):
+ return "Rule(source={},target={},excludes={},group_by={})".format(
+ self.source, self.target, self.excludes, self.group_by)
diff --git a/monasca_common/tests/test_monasca_query_language.py b/monasca_common/tests/test_monasca_query_language.py
new file mode 100644
index 00000000..c2df3442
--- /dev/null
+++ b/monasca_common/tests/test_monasca_query_language.py
@@ -0,0 +1,186 @@
+# (C) Copyright 2017 Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslotest import base
+import pyparsing
+
+from monasca_common.monasca_query_language import aql_parser
+from monasca_common.monasca_query_language import exceptions
+from monasca_common.monasca_query_language import query_structures
+
+
+class TestMonascaQueryLanguage(base.BaseTestCase):
+
+ def test_parse_group_expression(self):
+ expressions = [
+ "",
+ "excluding metric_two",
+ "group by hostname, service",
+ "excluding metric_two group by hostname, service",
+ "group by __severity__",
+ "excluding {__severity__=HIGH} group by __severity__",
+ "excluding {__severity__=HIGH, hostname=host1} group by __severity__, hostname",
+ "group by excluding" # excluding is an acceptable metric name
+ ]
+ negative_expressions = [
+ "group by hostname excluding {__metricName__=metric_two}",
+ "excluding metric_one excluding metric_two",
+ "targets metric_one",
+ ]
+ matchers = [
+ [],
+ [],
+ ["hostname", "service"],
+ ["hostname", "service"],
+ ["__severity__"],
+ ["__severity__"],
+ ["__severity__", "hostname"],
+ ["excluding"]
+ ]
+ exclusions = [
+ {},
+ {"__metricName__": "metric_two"},
+ {},
+ {"__metricName__": "metric_two"},
+ {},
+ {"__severity__": "HIGH"},
+ {"__severity__": "HIGH", "hostname": "host1"},
+ {},
+ ]
+ for i in range(len(expressions)):
+ result = aql_parser.RuleExpressionParser(expressions[i]).parse()
+ result = result[0].get_struct("group")
+ self.assertEqual(result['matchers'], matchers[i])
+ self.assertEqual(result['exclusions'], exclusions[i])
+ for negative_expression in negative_expressions:
+ try:
+ result = aql_parser.RuleExpressionParser(negative_expression)
+ self.assertRaises(exceptions.InvalidExpressionException,
+ result.parse())
+ except TypeError:
+ pass
+ except pyparsing.ParseException:
+ pass
+
+ def test_parse_inhibit_rule(self):
+ expressions = [
+ "",
+ "source metric_one",
+ "targets metric_two",
+ "source metric_one targets metric_two",
+ "source metric_one targets metric_two excluding metric_three",
+ "source metric_one targets metric_two excluding metric_three group by hostname",
+ "source metric_one targets metric_two group by hostname",
+ "source metric_one group by hostname",
+ "source {__severity__=HIGH} targets {__severity__=LOW} excluding "
+ "{__alarmName__=alarm_one} group by __alarmName__"
+ ]
+ negative_expressions = [
+ "targets metric_two source_metric_one"
+ ]
+ source = [
+ {},
+ {"__metricName__": "metric_one"},
+ {},
+ {"__metricName__": "metric_one"},
+ {"__metricName__": "metric_one"},
+ {"__metricName__": "metric_one"},
+ {"__metricName__": "metric_one"},
+ {"__metricName__": "metric_one"},
+ {"__severity__": "HIGH"},
+ ]
+ target = [
+ {},
+ {},
+ {"__metricName__": "metric_two"},
+ {"__metricName__": "metric_two"},
+ {"__metricName__": "metric_two"},
+ {"__metricName__": "metric_two"},
+ {"__metricName__": "metric_two"},
+ {},
+ {"__severity__": "LOW"}
+ ]
+ equals = [
+ [],
+ [],
+ [],
+ [],
+ [],
+ ["hostname"],
+ ["hostname"],
+ ["hostname"],
+ ["__alarmName__"]
+ ]
+ exclusions = [
+ {},
+ {},
+ {},
+ {},
+ {"__metricName__": "metric_three"},
+ {"__metricName__": "metric_three"},
+ {},
+ {},
+ {"__alarmName__": "alarm_one"}
+ ]
+ for i in range(len(expressions)):
+ result = aql_parser.RuleExpressionParser(expressions[i]).parse()
+ result = result[0].get_struct("inhibit")
+ self.assertEqual(result['source_match'], source[i])
+ self.assertEqual(result['target_match'], target[i])
+ self.assertEqual(result['equal'], equals[i])
+ self.assertEqual(result['exclusions'], exclusions[i])
+
+ for expression in negative_expressions:
+ try:
+ result = aql_parser.RuleExpressionParser(expression)
+ self.assertRaises(exceptions.InvalidExpressionException,
+ result.parse())
+ except pyparsing.ParseException:
+ pass
+
+ def test_parse_silence_rule(self):
+ expressions = [
+ "",
+ "targets metric_one",
+ "targets metric_one{}",
+ "targets metric_one{hostname=host_one}",
+ "targets metric_one{hostname=host_one, region=region_one}",
+ ]
+ negative_expressions = [
+ "excludes metric_one",
+ "source metric_one",
+ "group by hostname",
+ "targets metric_one, {hostname=host_one}",
+ ]
+ matchers = [
+ {},
+ {"__metricName__": "metric_one"},
+ {"__metricName__": "metric_one"},
+ {"__metricName__": "metric_one", "hostname": "host_one"},
+ {"__metricName__": "metric_one", "hostname": "host_one", "region": "region_one"},
+ ]
+ for i in range(len(expressions)):
+ result = aql_parser.RuleExpressionParser(expressions[i]).parse()
+ result = result[0].get_struct("silence")
+ self.assertEqual(result['matchers'], matchers[i])
+
+ for expression in negative_expressions:
+ try:
+ self.assertRaises(exceptions.InvalidExpressionException,
+ aql_parser.RuleExpressionParser(expression).parse())
+ except TypeError:
+ pass
+ except pyparsing.ParseException:
+ pass
diff --git a/requirements.txt b/requirements.txt
index 648d9e06..2dedcee5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,5 @@ pykafka>=2.5.0 # Apache 2.0 License
PyMySQL>=0.7.6 # MIT License
oslo.config>=4.0.0 # Apache-2.0
pbr!=2.1.0,>=2.0.0 # Apache-2.0
+pyparsing>=2.1.0 # MIT
ujson>=1.35 # BSD