From 7aa2ce4cade9bd939eac77855f76ee0bfa6e7129 Mon Sep 17 00:00:00 2001
From: Mohammed Naser <mnaser@vexxhost.com>
Date: Wed, 15 Apr 2020 17:45:45 -0400
Subject: [PATCH] updated alerts

Change-Id: I665d6289df9337c9b43e59ba91fbc004d4a50c29
---
 controllers/mcrouter_controller.go  | 19 +++++++++++--------
 controllers/memcached_controller.go | 15 +++++++++++----
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/controllers/mcrouter_controller.go b/controllers/mcrouter_controller.go
index be7e58a5..2450ceb9 100755
--- a/controllers/mcrouter_controller.go
+++ b/controllers/mcrouter_controller.go
@@ -165,15 +165,18 @@ func (r *McrouterReconciler) ReconcilePrometheusRule(ctx context.Context, req ct
 				Name("mcrouter-rule").
 				Rules(
 					builders.Rule().
-						Alert("McrouterBackendDown").
-						Message("Backend Memcached servers are down.").
-						Priority(1).
-						Expr("mcrouter_servers{state='down'}!=0"),
+						Alert("McrouterDown").
+						Priority(2).
+						Expr("mcrouter_up != 1"),
+
 					builders.Rule().
-						Alert("McrouterBackendTimeout").
-						Message("Backend Memcached servers are timeout.").
-						Priority(1).
-						Expr("mcrouter_server_memcached_timeout_count>0"),
+						Alert("McrouterAllBackendsDown").
+						Priority(2).
+						Expr("mcrouter_servers{state='up'} / mcrouter_servers == 0"),
+					builders.Rule().
+						Alert("McrouterBackendDown").
+						Priority(3).
+						Expr("mcrouter_servers{state='down'} != 0"),
 				).
 				Interval("1m")).
 			Build()
diff --git a/controllers/memcached_controller.go b/controllers/memcached_controller.go
index c6c77aed..43390f5c 100755
--- a/controllers/memcached_controller.go
+++ b/controllers/memcached_controller.go
@@ -161,12 +161,19 @@ func (r *MemcachedReconciler) ReconcilePrometheusRule(ctx context.Context, req c
 			RuleGroups(builders.RuleGroup().
 				Name("memcached-rule").
 				Rules(
+					builders.Rule().
+						Alert("MemcachedDown").
+						Priority(3).
+						Expr("memcached_up == 0"),
 
 					builders.Rule().
-						Alert("MemcachedConnectionLimit").
-						Message("This memcached connection is over max.").
-						Priority(1).
-						Expr("memcached_current_connections/memcached_max_connections*100 >90"),
+						Alert("MemcachedMaxConnections").
+						Priority(3).
+						Expr("memcached_current_connections/memcached_max_connections * 100 > 95"),
+					builders.Rule().
+						Alert("MemcachedMaxConnections").
+						Priority(4).
+						Expr("memcached_current_connections/memcached_max_connections * 100 > 90"),
 				).
 				Interval("1m")).
 			Build()