From 6a67fecc80c211e9a33445bd969c3fbf8c7a55c5 Mon Sep 17 00:00:00 2001 From: Craig Bryant Date: Sun, 24 Jul 2016 20:58:34 -0600 Subject: [PATCH] Add Last function Add the LAST function to the Java parser and its associated classes. It will save the last value received by its timestamp Update SubAlarmDb to have a state field. This will be used in the Threshold Engine for the new LAST function Add a force argument to SubAlarmStats.addValue() because the Threshold Engine will always need to add the values for the alarm because they are only sent once The changes to Java Monasca API and Monasca Thresh for this last-value will require these changes, first Also, remove all trailing spaces from AlarmExpression.g4 Change-Id: Ie3192370057676e89bfebbc5ce4243b1a3ecf0ea Implements: Blueprint last-value --- java/monasca-common-hibernate/pom.xml | 6 ++ .../common/hibernate/db/SubAlarmDb.java | 44 +++++++++- .../common/model/alarm/AggregateFunction.java | 8 +- .../common/model/alarm/AlarmExpression.java | 2 +- .../common/model/alarm/AlarmExpression.g4 | 26 +++--- .../common/util/stats/SlidingWindowStats.java | 34 ++++++-- .../monasca/common/util/stats/Statistic.java | 4 +- .../monasca/common/util/stats/Statistics.java | 28 +++++-- .../util/stats/SlidingWindowStatsTest.java | 82 ++++++++++++------- .../common/util/stats/StatisticsTest.java | 9 +- 10 files changed, 175 insertions(+), 68 deletions(-) diff --git a/java/monasca-common-hibernate/pom.xml b/java/monasca-common-hibernate/pom.xml index 82f99536..02920189 100644 --- a/java/monasca-common-hibernate/pom.xml +++ b/java/monasca-common-hibernate/pom.xml @@ -24,6 +24,12 @@ hibernate-core ${hibernate-core.version} + + monasca-common + monasca-common-testing + ${project.version} + test + diff --git a/java/monasca-common-hibernate/src/main/java/monasca/common/hibernate/db/SubAlarmDb.java b/java/monasca-common-hibernate/src/main/java/monasca/common/hibernate/db/SubAlarmDb.java index 07235076..749fe2e8 100644 --- a/java/monasca-common-hibernate/src/main/java/monasca/common/hibernate/db/SubAlarmDb.java +++ b/java/monasca-common-hibernate/src/main/java/monasca/common/hibernate/db/SubAlarmDb.java @@ -1,5 +1,6 @@ /* * Copyright 2015 FUJITSU LIMITED + * (C) Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -14,10 +15,14 @@ */ package monasca.common.hibernate.db; +import monasca.common.model.alarm.AlarmState; + import javax.persistence.Basic; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.FetchType; import javax.persistence.JoinColumn; import javax.persistence.Lob; @@ -67,30 +72,54 @@ public class SubAlarmDb @Column(name = "expression", nullable = false, length = 16777215) private String expression = DEFAULT_EXPRESSION; + @Column(name = "state") + @Enumerated(EnumType.STRING) + private AlarmState state; + public SubAlarmDb() { super(); } + /** This constructor will be deleted once the API tests have changed to no longer use it */ public SubAlarmDb(String id, AlarmDb alarm, String expression, DateTime created_at, DateTime updated_at) { - this(id, alarm, null, expression, created_at, updated_at); - this.alarm = alarm; - this.expression = expression; + this(id, alarm, null, expression, AlarmState.OK, created_at, updated_at); } + public SubAlarmDb(String id, + AlarmDb alarm, + String expression, + AlarmState state, + DateTime created_at, + DateTime updated_at) { + this(id, alarm, null, expression, state, created_at, updated_at); + } + + /** This constructor will be deleted once the API tests have changed to no longer use it */ public SubAlarmDb(String id, AlarmDb alarm, SubAlarmDefinitionDb subExpression, String expression, DateTime created_at, DateTime updated_at) { + this(id, alarm, subExpression, expression, AlarmState.OK, created_at, updated_at); + } + + public SubAlarmDb(String id, + AlarmDb alarm, + SubAlarmDefinitionDb subExpression, + String expression, + AlarmState state, + DateTime created_at, + DateTime updated_at) { super(id, created_at, updated_at); this.alarm = alarm; this.subExpression = subExpression; this.expression = expression; + this.state = state; } public SubAlarmDb setExpression(final String expression) { @@ -125,6 +154,15 @@ public class SubAlarmDb return this.expression; } + public AlarmState getState() { + return state; + } + + public SubAlarmDb setState(AlarmState state) { + this.state = state; + return this; + } + public interface Queries { String BY_ALARMDEFINITION_ID = "SubAlarm.byAlarmDefinitionId"; String BY_ALARM_ID = "SubAlarm.byAlarmId"; diff --git a/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AggregateFunction.java b/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AggregateFunction.java index 3fe6beda..0d89f982 100644 --- a/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AggregateFunction.java +++ b/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AggregateFunction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,15 +17,17 @@ package monasca.common.model.alarm; import com.fasterxml.jackson.annotation.JsonCreator; + import monasca.common.util.stats.Statistic; import monasca.common.util.stats.Statistics.Average; import monasca.common.util.stats.Statistics.Count; +import monasca.common.util.stats.Statistics.Last; import monasca.common.util.stats.Statistics.Max; import monasca.common.util.stats.Statistics.Min; import monasca.common.util.stats.Statistics.Sum; public enum AggregateFunction { - MIN, MAX, SUM, COUNT, AVG; + MIN, MAX, SUM, COUNT, AVG, LAST; @JsonCreator public static AggregateFunction fromJson(String text) { @@ -48,6 +50,8 @@ public enum AggregateFunction { return Min.class; if (AggregateFunction.MAX.equals(this)) return Max.class; + if (AggregateFunction.LAST.equals(this)) + return Last.class; return null; } } diff --git a/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AlarmExpression.java b/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AlarmExpression.java index 23980580..c50aaed7 100644 --- a/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AlarmExpression.java +++ b/java/monasca-common-model/src/main/java/monasca/common/model/alarm/AlarmExpression.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/java/monasca-common-model/src/main/resources/monasca/common/model/alarm/AlarmExpression.g4 b/java/monasca-common-model/src/main/resources/monasca/common/model/alarm/AlarmExpression.g4 index 97f2ab65..ade3f3a9 100644 --- a/java/monasca-common-model/src/main/resources/monasca/common/model/alarm/AlarmExpression.g4 +++ b/java/monasca-common-model/src/main/resources/monasca/common/model/alarm/AlarmExpression.g4 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ start : expression EOF ; -expression +expression : compoundIdentifier relational_operator literal # relationalExprFwd | function relational_operator literal ('times' repeat)? # relationalExprFuncFwd | expression and expression # andExpr @@ -78,6 +78,7 @@ functionType | SUM | CNT | AVG + | LAST ; primary @@ -113,6 +114,7 @@ keyword | SUM | CNT | AVG + | LAST ; period @@ -135,7 +137,7 @@ repeat txt : TXT | keyword - | INTEGER + | INTEGER | STRING ; @@ -166,19 +168,19 @@ GT_S GTE : [gG][tT][eE] ; - + GTE_S : '>=' ; - + AND - : [aA][nN][dD] + : [aA][nN][dD] ; - + AND_S : '&&' ; - + OR : [oO][rR] ; @@ -207,11 +209,15 @@ AVG : [aA][vV][gG] ; +LAST + : [lL][aA][sS][tT] + ; + INTEGER : DIGIT+ ; - -DECIMAL + +DECIMAL : '-'?DIGIT+('.'DIGIT+)? ; diff --git a/java/monasca-common-util/src/main/java/monasca/common/util/stats/SlidingWindowStats.java b/java/monasca-common-util/src/main/java/monasca/common/util/stats/SlidingWindowStats.java index afde2bd9..96e19753 100644 --- a/java/monasca-common-util/src/main/java/monasca/common/util/stats/SlidingWindowStats.java +++ b/java/monasca-common-util/src/main/java/monasca/common/util/stats/SlidingWindowStats.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -97,20 +97,36 @@ public class SlidingWindowStats { /** * Adds the {@code value} to the statistics for the slot associated with the {@code timestamp} and - * returns true, else returns false if the {@code timestamp} is outside of the window. + * returns true, else returns false if the {@code timestamp} is outside of the window and + * {@code force} is false. If {@code force} is true, always add value * * @param value to add * @param timestamp to add value for + * @param force if true, add value to first window even if timestamp is outside of all windows * @return true if the value was added else false if it the {@code timestamp} was outside the - * window + * window and force was false + */ + public boolean addValue(double value, long timestamp, boolean force) { + int index = indexOfTime(timescale.adjust(timestamp)); + if (index == -1) { + if (force) { + index = 0; + } else { + return false; + } + } + slots[index].stat.addValue(value, timestamp); + return true; + } + + /** + * This will be deleted when all uses have changed to use the above method + * @param value + * @param timestamp + * @return */ public boolean addValue(double value, long timestamp) { - timestamp = timescale.adjust(timestamp); - int index = indexOfTime(timestamp); - if (index == -1) - return false; - slots[index].stat.addValue(value); - return true; + return addValue(value, timestamp, false); } /** Returns the number of slots in the window. */ diff --git a/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistic.java b/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistic.java index 8769d894..a78412cc 100644 --- a/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistic.java +++ b/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistic.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ package monasca.common.util.stats; */ public interface Statistic { /** Adds the {@code value} to the statistic. */ - void addValue(double value); + void addValue(double value, double timestamp); /** Returns true if the statistic has been initialized with a value, else false. */ boolean isInitialized(); diff --git a/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistics.java b/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistics.java index ec7aa650..7ada2564 100644 --- a/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistics.java +++ b/java/monasca-common-util/src/main/java/monasca/common/util/stats/Statistics.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,8 +50,8 @@ public final class Statistics { protected int count; @Override - public void addValue(double value) { - super.addValue(value); + public void addValue(double value, double timestamp) { + super.addValue(value, timestamp); this.count++; } @@ -69,7 +69,7 @@ public final class Statistics { public static class Count extends AbstractStatistic { @Override - public void addValue(double value) { + public void addValue(double value, double timestamp) { initialized = true; this.value++; } @@ -77,7 +77,7 @@ public final class Statistics { public static class Max extends AbstractStatistic { @Override - public void addValue(double value) { + public void addValue(double value, double timestamp) { if (!initialized) { initialized = true; this.value = value; @@ -88,7 +88,7 @@ public final class Statistics { public static class Min extends AbstractStatistic { @Override - public void addValue(double value) { + public void addValue(double value, double timestamp) { if (!initialized) { initialized = true; this.value = value; @@ -99,12 +99,26 @@ public final class Statistics { public static class Sum extends AbstractStatistic { @Override - public void addValue(double value) { + public void addValue(double value, double timestamp) { initialized = true; this.value += value; } } + public static class Last extends AbstractStatistic { + protected double lastTimestamp; + + @Override + public void addValue(double value, double timestamp) { + initialized = true; + // Ensure older measurements don't change value + if (timestamp > this.lastTimestamp) { + this.value = value; + this.lastTimestamp = timestamp; + } + } + } + private Statistics() { } } diff --git a/java/monasca-common-util/src/test/java/monasca/common/util/stats/SlidingWindowStatsTest.java b/java/monasca-common-util/src/test/java/monasca/common/util/stats/SlidingWindowStatsTest.java index a5cb62be..6a582da8 100644 --- a/java/monasca-common-util/src/test/java/monasca/common/util/stats/SlidingWindowStatsTest.java +++ b/java/monasca-common-util/src/test/java/monasca/common/util/stats/SlidingWindowStatsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -104,46 +104,46 @@ public class SlidingWindowStatsTest { SlidingWindowStats window = new SlidingWindowStats(Statistics.Average.class, TimeResolution.ABSOLUTE, 60, 1, 2, 0); - assertTrue(window.addValue(1.0, 20)); + assertTrue(window.addValue(1.0, 20, false)); window.slideViewTo(20, 30); - assertTrue(window.addValue(1.0, 0)); + assertTrue(window.addValue(1.0, 0, false)); window.slideViewTo(100, 30); - assertFalse(window.addValue(1.0, 0)); - assertTrue(window.addValue(1.0, 61)); + assertFalse(window.addValue(1.0, 0, false)); + assertTrue(window.addValue(1.0, 61, false)); window.slideViewTo(121, 30); - assertTrue(window.addValue(1.0, 61)); - assertTrue(window.addValue(1.0, 121)); + assertTrue(window.addValue(1.0, 61, false)); + assertTrue(window.addValue(1.0, 121, false)); window.slideViewTo(180, 30); - assertFalse(window.addValue(1.0, 61)); - assertTrue(window.addValue(1.0, 121)); - assertTrue(window.addValue(1.0, 181)); + assertFalse(window.addValue(1.0, 61, false)); + assertTrue(window.addValue(1.0, 121, false)); + assertTrue(window.addValue(1.0, 181, false)); window.slideViewTo(241, 30); - assertFalse(window.addValue(1.0, 121)); - assertTrue(window.addValue(1.0, 181)); - assertTrue(window.addValue(1.0, 241)); + assertFalse(window.addValue(1.0, 121, false)); + assertTrue(window.addValue(1.0, 181, false)); + assertTrue(window.addValue(1.0, 241, false)); window.slideViewTo(360, 30); - assertFalse(window.addValue(1.0, 241)); - assertTrue(window.addValue(1.0, 300)); - assertTrue(window.addValue(1.0, 361)); + assertFalse(window.addValue(1.0, 241, false)); + assertTrue(window.addValue(1.0, 300, false)); + assertTrue(window.addValue(1.0, 361, false)); } public void shouldAddValueAndGetWindowValues() { SlidingWindowStats window = new SlidingWindowStats(Statistics.Average.class, TimeResolution.ABSOLUTE, 3, 3, 2, 9); for (int i = 0; i < 5; i++) - window.addValue(999, i * 3); + window.addValue(999, i * 3, false); assertEquals(window.getWindowValues(), new double[] { 999, 999, 999, 999, 999 }); window.slideViewTo(12, 1); assertEquals(window.getWindowValues(), new double[] { 999, 999, 999, 999, Double.NaN }); - window.addValue(888, 17); + window.addValue(888, 17, false); assertEquals(window.getWindowValues(), new double[] { 999, 999, 999, 999, 888 }); } @@ -151,20 +151,42 @@ public class SlidingWindowStatsTest { SlidingWindowStats window = new SlidingWindowStats(Statistics.Average.class, TimeResolution.ABSOLUTE, 3, 3, 2, 9); for (int i = 0; i < 5; i++) - window.addValue(999, i * 3); + window.addValue(999, i * 3, false); assertEquals(window.getViewValues(), new double[] { 999, 999, 999 }); window.slideViewTo(15, 1); assertEquals(window.getViewValues(), new double[] { 999, 999, 999 }); - window.addValue(777, 15); - window.addValue(888, 18); + window.addValue(777, 15, false); + window.addValue(888, 18, false); assertEquals(window.getViewValues(), new double[] { 999, 999, 999 }); window.slideViewTo(21, 1); assertEquals(window.getViewValues(), new double[] { 999, 777, 888 }); } + public void shouldAddOutOfWindowValueWithForce() { + SlidingWindowStats window = new SlidingWindowStats(Statistics.Last.class, + TimeResolution.ABSOLUTE, 3, 1, 2, 9); + window.addValue(999, 3, true); + assertEquals(window.getViewValues(), new double[] { 999 }); + } + + public void shouldNotAddOutOfWindowValueWithoutForce() { + SlidingWindowStats window = new SlidingWindowStats(Statistics.Average.class, + TimeResolution.ABSOLUTE, 3, 1, 2, 9); + window.addValue(999, 3, false); + assertEquals(window.getViewValues(), new double[] { Double.NaN }); + } + + public void shouldIgnoreOutOfOrderValue() { + SlidingWindowStats window = new SlidingWindowStats(Statistics.Last.class, + TimeResolution.ABSOLUTE, 3, 1, 2, 9); + window.addValue(999, 3, true); + window.addValue(998, 2, true); + assertEquals(window.getViewValues(), new double[] { 999 }); + } + public void testIndexOfTime() { SlidingWindowStats window = new SlidingWindowStats(Statistics.Average.class, TimeResolution.ABSOLUTE, 3, 3, 2, 15); @@ -208,9 +230,9 @@ public class SlidingWindowStatsTest { SlidingWindowStats window = new SlidingWindowStats(Statistics.Sum.class, TimeResolution.ABSOLUTE, 5, 3, 2, 20); // Logical window is 5 10 15 - window.addValue(2, 5); - window.addValue(3, 10); - window.addValue(4, 15); + window.addValue(2, 5, false); + window.addValue(3, 10, false); + window.addValue(4, 15, false); assertEquals(window.getValue(5), 2.0); assertEquals(window.getValue(10), 3.0); @@ -218,7 +240,7 @@ public class SlidingWindowStatsTest { // Slide logical window to 10 15 20 window.slideViewTo(25, 1); - window.addValue(5, 24); + window.addValue(5, 24, false); assertEquals(window.getValue(10), 3.0); assertEquals(window.getValue(15), 4.0); @@ -264,9 +286,9 @@ public class SlidingWindowStatsTest { SlidingWindowStats window = new SlidingWindowStats(Statistics.Sum.class, TimeResolution.ABSOLUTE, 5, 3, 2, 20); // Window is 5 10 15 20 25 - window.addValue(2, 5); - window.addValue(3, 10); - window.addValue(4, 15); + window.addValue(2, 5, false); + window.addValue(3, 10, false); + window.addValue(4, 15, false); assertEquals(window.getValuesUpTo(20), new double[] { 2, 3, 4, Double.NaN }); assertEquals(window.getValuesUpTo(18), new double[] { 2, 3, 4 }); @@ -275,14 +297,14 @@ public class SlidingWindowStatsTest { // Window is 30 10 15 20 25 window.slideViewTo(22, 1); - window.addValue(5, 22); + window.addValue(5, 22, false); assertEquals(window.getValuesUpTo(22), new double[] { 3, 4, 5 }); assertEquals(window.getValuesUpTo(15), new double[] { 3, 4 }); assertEquals(window.getValuesUpTo(12), new double[] { 3 }); // Window is 30 35 15 20 25 window.slideViewTo(27, 1); - window.addValue(6, 26); + window.addValue(6, 26, false); assertEquals(window.getValuesUpTo(27), new double[] { 4, 5, 6 }); assertEquals(window.getValuesUpTo(24), new double[] { 4, 5 }); assertEquals(window.getValuesUpTo(18), new double[] { 4 }); diff --git a/java/monasca-common-util/src/test/java/monasca/common/util/stats/StatisticsTest.java b/java/monasca-common-util/src/test/java/monasca/common/util/stats/StatisticsTest.java index e545c584..01b07b4f 100644 --- a/java/monasca-common-util/src/test/java/monasca/common/util/stats/StatisticsTest.java +++ b/java/monasca-common-util/src/test/java/monasca/common/util/stats/StatisticsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * (C) Copyright 2014, 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -20,6 +20,7 @@ import org.testng.annotations.Test; import monasca.common.util.stats.Statistics.Average; import monasca.common.util.stats.Statistics.Count; +import monasca.common.util.stats.Statistics.Last; import monasca.common.util.stats.Statistics.Max; import monasca.common.util.stats.Statistics.Min; import monasca.common.util.stats.Statistics.Sum; @@ -29,13 +30,13 @@ public class StatisticsTest { @DataProvider(name = "metricTypes") public Object[][] createData1() { return new Object[][] { { new Average(), 3 }, { new Sum(), 6 }, { new Min(), 2 }, - { new Max(), 4 }, { new Count(), 2 }, }; + { new Max(), 4 }, { new Count(), 2 }, { new Last(), 4 }}; } @Test(dataProvider = "metricTypes") public void testStat(Statistic stat, double expectedValue) { - stat.addValue(2); - stat.addValue(4); + stat.addValue(2, 1); + stat.addValue(4, 2); assertEquals(stat.value(), expectedValue, stat.getClass().getName()); } }