From eb1e16685d7c6df31f2716b189656216570cabe0 Mon Sep 17 00:00:00 2001
From: Cole Walker <cole.walker@windriver.com>
Date: Mon, 9 Sep 2024 11:38:39 -0400
Subject: [PATCH] Handle incomplete pmc results
Improve the logic for handling incomplete results when a pmc query
returns empty data.
There are intermittent instances where a pmc command returns an empty
result. The previous logic would result in ptp-notification considering
the ptp state to be in Freerun, but this could lead to undesirable
results where incorrect notifications are delivered.
This update allows ptp-notification to retry the pmc commands on the
next cycle and only updates the ptp status when the results are
complete. Additional logging has been added to indicate which pmc
results are missing.
Test plan:
Pass: Verify container image builds
Pass: Verify ptp-notification deployment
Pass: Verify basic operations (get, subscribe, list, delete)
Pass: Verify that ptp-notification does not deliver incorrect freerun
notifications when pmc results are incomplete
Pass: Verify that ptp-notification delivers a notification when ptp lock
is actually lost
Story: 2011056
Task: 51049
Signed-off-by: Cole Walker <cole.walker@windriver.com>
Change-Id: Ic473fcb44e16db629f12208e581a92abc3558480
---
.../common/helpers/ptp_monitor.py | 8 ++++++--
.../trackingfunctionsdk/common/helpers/ptpsync.py | 6 +++---
.../trackingfunctionsdk/common/helpers/ptpsync.py | 12 ++++++++----
3 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py
index 38d4a64..652c2ad 100644
--- a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py
+++ b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptp_monitor.py
@@ -139,8 +139,12 @@ class PtpMonitor:
if pmc and ptp4l and phc2sys and ptp4lconf:
self.pmc_query_results, total_ptp_keywords, port_count = \
self.ptpsync()
- sync_state = utils.check_results(self.pmc_query_results,
- total_ptp_keywords, port_count)
+ try:
+ sync_state = utils.check_results(self.pmc_query_results,
+ total_ptp_keywords, port_count)
+ except RuntimeError as err:
+ LOG.warning(err)
+ sync_state = previous_sync_state
else:
LOG.warning("Missing critical resource: "
"PMC %s PTP4L %s PHC2SYS %s PTP4LCONF %s"
diff --git a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py
index 993a0d3..ae040a6 100644
--- a/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py
+++ b/notificationservice-base-v2/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py
@@ -74,9 +74,9 @@ def check_results(result, total_ptp_keywords, port_count):
# check for a healthy result
if len(result) != total_ptp_keywords:
- sync_state = constants.FREERUN_PHC_STATE
- LOG.warning('results are not complete, returning FREERUN')
- return sync_state
+ LOG.info("Results %s" % result)
+ LOG.info("Results len %s, total_ptp_keywords %s" % (len(result), total_ptp_keywords))
+ raise RuntimeError("PMC results are incomplete, retrying")
# determine the current sync state
if (result[constants.GM_PRESENT].lower() != constants.GM_IS_PRESENT
and result[constants.GRANDMASTER_IDENTITY] != result[constants.CLOCK_IDENTITY]):
diff --git a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py
index 6c5ec26..867d593 100644
--- a/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py
+++ b/notificationservice-base/docker/ptptrackingfunction/trackingfunctionsdk/common/helpers/ptpsync.py
@@ -124,9 +124,9 @@ def check_results(result, total_ptp_keywords, port_count):
# check for a healthy result
if len(result) != total_ptp_keywords:
- sync_state = constants.FREERUN_PHC_STATE
- LOG.warning('results are not complete, returning FREERUN')
- return sync_state
+ LOG.info("Results %s" % result)
+ LOG.info("Results len %s, total_ptp_keywords %s" % (len(result), total_ptp_keywords))
+ raise RuntimeError("PMC results are incomplete, retrying")
# determine the current sync state
if (result[constants.GM_PRESENT].lower() != constants.GM_IS_PRESENT
and result[constants.GRANDMASTER_IDENTITY] != result[constants.CLOCK_IDENTITY]):
@@ -225,7 +225,11 @@ def ptp_status(holdover_time, freq, sync_state, event_time):
# run pmc command if preconditions met
if pmc and ptp4l and phc2sys and ptp4lconf:
result, total_ptp_keywords, port_count = ptpsync()
- sync_state = check_results(result, total_ptp_keywords, port_count)
+ try:
+ sync_state = check_results(result, total_ptp_keywords, port_count)
+ except RuntimeError as err:
+ LOG.warning(err)
+ sync_state = previous_sync_state
else:
LOG.warning("Critical resources not available: pmc %s ptp4l %s phc2sys %s ptp4lconf %s" %
(pmc, ptp4l, phc2sys, ptp4lconf))