openstack/monasca-thresh
Code Issues Proposed changes

Treat empty windows as OK for deterministic alarms

Browse Source 
Using the standard case of count(log_message) > 1, getting no
log_message measurements should be treated as OK. However, the old code
uses the emptyWindowObservationThreshold for both deterministic and
non-deterministic alarms which means that there must be 3 empty windows
before the deterministic alarm transitions to OK.

This change cause the evaluation of the Alarm to treat an empty window
as OK for deterministic alarms. So, count(log_message) > 1, getting no
measurements in a window will transition the alarm back to OK

Change-Id: I19a04bf78f907b23ef583409f2def54771c07d72
This commit is contained in:
Craig Bryant 2016-06-20 12:30:42 -06:00
parent d5d14ecdcd
commit 0d80a987db
2 changed files with 63 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
thresh/src/main/java/monasca/thresh/domain/model/SubAlarmStats.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
* (C) Copyright 2014-2016 Hewlett Packard Enterprise Development Company LP.
* Copyright 2016 FUJITSU LIMITED
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -176,29 +176,35 @@ public class SubAlarmStats {
return AlarmState.ALARM;
}
final boolean isDeterministic = this.subAlarm.isDeterministic();
// Window is empty at this point
this.emptyWindowObservations++;
final boolean emptyWindowThresholdExceeded = this.emptyWindowObservations >=
this.emptyWindowObservationThreshold;
final boolean useDefaultState;
if (isDeterministic) {
useDefaultState = true;
}
else {
useDefaultState = this.emptyWindowObservations >= this.emptyWindowObservationThreshold;
}
if (emptyWindowThresholdExceeded && this.shouldSendStateChange(AlarmState.UNDETERMINED)) {
final boolean isDeterministic = this.subAlarm.isDeterministic();
if (useDefaultState) {
final AlarmState state = SubAlarm.getDefaultState(isDeterministic);
final AlarmState subAlarmState = this.subAlarm.getState();
logger.debug(
"SubAlarm[deterministic={}] {} exceeded empty window threshold {}, transition to {} from {}",
isDeterministic,
this.subAlarm.getId(),
this.emptyWindowObservationThreshold,
state,
subAlarmState
);
return state;
if (this.shouldSendStateChange(state)) {
logger.debug(
"SubAlarm[deterministic={}] {} exceeded empty window threshold {}, transition to {} from {}",
isDeterministic,
this.subAlarm.getId(),
this.emptyWindowObservationThreshold,
state,
this.subAlarm.getState()
);
return state;
}
}
// Hasn't transitioned to UNDETERMINED yet, so use the current state
// Hasn't transitioned to default state yet, so use the current state
return null;
}

53
thresh/src/test/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBoltTest.java
View File

@ -181,8 +181,10 @@ public class MetricAggregationBoltTest {
assertEquals(subAlarm4.getState(), AlarmState.OK); // deterministic
verify(collector, times(1)).emit(new Values(subAlarm1.getAlarmId(), subAlarm1));
verify(collector, never()).emit(new Values(subAlarm2.getAlarmId(), subAlarm2));
verify(collector, never()).emit(new Values(subAlarm3.getAlarmId(), subAlarm3));
verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
// Have to reset the mock so it can tell the difference when subAlarm2 and subAlarm3 are emitted
// again.
reset(collector);
// Drive subAlarm1 to ALARM
@ -201,7 +203,7 @@ public class MetricAggregationBoltTest {
verify(collector, times(1)).emit(new Values(subAlarm1.getAlarmId(), subAlarm1));
verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2));
verify(collector, times(1)).emit(new Values(subAlarm3.getAlarmId(), subAlarm3));
verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
}
public void shouldImmediatelyEvaluateSubAlarm() {
@ -264,10 +266,11 @@ public class MetricAggregationBoltTest {
assertEquals(subAlarm2.getState(), AlarmState.OK);
assertEquals(subAlarm3.getState(), AlarmState.OK);
assertEquals(subAlarm4.getState(), AlarmState.ALARM); // still in alarm
// subAlarm4 goes to OK because of missing metrics
assertEquals(subAlarm4.getState(), AlarmState.OK);
verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2));
verify(collector, times(1)).emit(new Values(subAlarm3.getAlarmId(), subAlarm3));
verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
// Have to reset the mock so it can tell the difference when subAlarm2 and subAlarm3 are emitted
// again.
@ -296,13 +299,13 @@ public class MetricAggregationBoltTest {
// Ensure that subAlarm3 is still ALARM. subAlarm2 is still OK but because the metric
// that triggered ALARM is in the future bucket
// subAlarm4 gets back to OK due to missing metrics
// subAlarm4 is still OK due to missing metrics
assertEquals(subAlarm2.getState(), AlarmState.OK);
assertEquals(subAlarm3.getState(), AlarmState.ALARM);
assertEquals(subAlarm4.getState(), AlarmState.OK);
verify(collector, never()).emit(new Values(subAlarm2.getAlarmId(), subAlarm2));
verify(collector, never()).emit(new Values(subAlarm3.getAlarmId(), subAlarm3));
verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
}
private void sendTickTuple() {
@ -394,7 +397,35 @@ public class MetricAggregationBoltTest {
verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2));
}
public void shouldNeverLeaveOkIfThresholdNotExceededForNonDeterministic() {
public void shouldSendOkAfterAlarmIfNoMetrics() {
long t1 = 50000;
bolt.setCurrentTime(t1);
sendSubAlarmCreated(metricDef4, subAlarm4);
bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null)));
t1 += 1000;
bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null)));
t1 += 1000;
bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null)));
t1 += 1000;
bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null)));
t1 += 1000;
bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null)));
bolt.setCurrentTime(t1 += 60000);
sendTickTuple();
assertEquals(subAlarm4.getState(), AlarmState.ALARM);
verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
// Have to reset the mock so it can tell the difference when subAlarm4 is emitted again.
reset(collector);
bolt.setCurrentTime(t1 += 60000);
sendTickTuple();
assertEquals(subAlarm4.getState(), AlarmState.OK);
verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
}
public void shouldNeverLeaveOkIfThresholdNotExceededForDeterministic() {
long t1 = 50000;
bolt.setCurrentTime(t1);
sendSubAlarmCreated(metricDef4, subAlarm4);
@ -450,7 +481,7 @@ public class MetricAggregationBoltTest {
verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2));
}
public void shouldSendOKOnStartupForNonDeterministic() {
public void shouldSendOKOnStartupForDeterministic() {
long t1 = 14000;
bolt.setCurrentTime(t1);
sendSubAlarmCreated(metricDef4, subAlarm4);
@ -462,11 +493,7 @@ public class MetricAggregationBoltTest {
bolt.execute(lagTuple);
verify(collector, times(1)).ack(lagTuple);
t1 += 60000;
bolt.setCurrentTime(t1);
sendTickTuple();
verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4));
// Won't send OK on this Tick Tuple because of the METRIC_BEHIND message sent above
t1 += 60000;
bolt.setCurrentTime(t1);
sendTickTuple();