Add support for monitoring OS clock sync state
Adds the OsClockMonitor class to handle determining whether a PHC is being used to sync the system clock. This is done by determining which ptp device is assigned to phc2sys and then comparing the time of the ptp device against the OS clock. If the delta is within an expected threshold then the OS clock is considered LOCKED. If there is no ptp device configured then the OS clock is set to FREERUN. A holdover state will also be implemented in a future change. Unit test coverage is included for OsClockMonitor. A follow up review will also be provided to add OsClockMonitor into the main notificationservice daemon process so that the clock state is checked and reported periodically. An additional follow up review will be required to update the helm charts with the required hostPath mounts in order for the ptp-notification pods to access the expected files on the host system. Test plan: Pass: Unit tests Pass: Run code standalone in ptp-notification pod and verify that OS clock sync state can be determined Story: 2010056 Task: 45658 Signed-off-by: Cole Walker <cole.walker@windriver.com> Change-Id: I1e7dcabe99d079f4587e293ab5983cd1f5b84978
This commit is contained in:
Cole Walker
parent
13f8666882
commit
db8cf39fad
@ -33,6 +33,16 @@ GNSS_LOCKED_HO_ACK = 'locked_ho_ack'
|
||||
GNSS_DPLL_0 = "DPLL0"
|
||||
GNSS_DPLL_1 = "DPLL1"
|
||||
|
||||
UTC_OFFSET = "37"
|
||||
PTP_CONFIG_PATH = "/ptp/ptpinstance/"
|
||||
PHC_CTL_PATH = "/usr/sbin/phc_ctl"
|
||||
PHC2SYS_DEFAULT_CONFIG = "/ptp/ptpinstance/phc2sys-phc2sys-legacy.conf"
|
||||
PHC2SYS_CONF_PATH = "/ptp/ptpinstance/"
|
||||
|
||||
CLOCK_REALTIME = "CLOCK_REALTIME"
|
||||
|
||||
PHC2SYS_TOLERANCE_LOW = 36999999000
|
||||
PHC2SYS_TOLERANCE_HIGH = 37000001000
|
||||
|
||||
# testing values
|
||||
CGU_PATH_VALID = "/sys/kernel/debug/ice/0000:18:00.0/cgu"
|
||||
|
||||
@ -0,0 +1,140 @@
|
||||
#
|
||||
# Copyright (c) 2022 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
from glob import glob
|
||||
|
||||
from trackingfunctionsdk.common.helpers import log_helper
|
||||
from trackingfunctionsdk.common.helpers import constants
|
||||
from trackingfunctionsdk.model.dto.osclockstate import OsClockState
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
log_helper.config_logger(LOG)
|
||||
|
||||
|
||||
class OsClockMonitor:
|
||||
_state = OsClockState()
|
||||
last_event_time = None
|
||||
phc2sys_instance = None
|
||||
phc2sys_config = None
|
||||
phc_interface = None
|
||||
ptp_device = None
|
||||
offset = None
|
||||
|
||||
def __init__(self, init=True):
|
||||
self.phc2sys_config = os.environ.get("PHC2SYS_CONFIG", constants.PHC2SYS_DEFAULT_CONFIG)
|
||||
self.set_phc2sys_instance()
|
||||
|
||||
"""Normally initialize all fields, but allow these to be skipped to assist with unit testing
|
||||
or to short-circuit values if required.
|
||||
"""
|
||||
if init:
|
||||
self.get_os_clock_time_source()
|
||||
self.get_os_clock_offset()
|
||||
self.set_os_clock_state()
|
||||
|
||||
def set_phc2sys_instance(self):
|
||||
self.phc2sys_instance = self.phc2sys_config.split(constants.PHC2SYS_CONF_PATH
|
||||
+ "phc2sys-")[1]
|
||||
self.phc2sys_instance = self.phc2sys_instance.split(".")[0]
|
||||
LOG.debug("phc2sys config file: %s" % self.phc2sys_config)
|
||||
LOG.debug("phc2sys instance name: %s" % self.phc2sys_instance)
|
||||
|
||||
def get_os_clock_time_source(self, pidfile_path="/var/run/"):
|
||||
"""Determine which PHC is disciplining the OS clock"""
|
||||
self.phc_interface = None
|
||||
self.phc_interface = self._check_command_line_interface(pidfile_path)
|
||||
if self.phc_interface is None:
|
||||
self.phc_interface = self._check_config_file_interface()
|
||||
if self.phc_interface is None:
|
||||
LOG.info("No PHC device found for phc2sys, status is FREERUN.")
|
||||
self._state = OsClockState.Freerun
|
||||
else:
|
||||
self.ptp_device = self._get_interface_phc_device()
|
||||
|
||||
def _check_command_line_interface(self, pidfile_path):
|
||||
pidfile = pidfile_path + "phc2sys-" + self.phc2sys_instance + ".pid"
|
||||
with open(pidfile, 'r') as f:
|
||||
pid = f.readline().strip()
|
||||
# Get command line params
|
||||
cmdline_file = "/host/proc/" + pid + "/cmdline"
|
||||
with open(cmdline_file, 'r') as f:
|
||||
cmdline_args = f.readline().strip()
|
||||
cmdline_args = cmdline_args.split("\x00")
|
||||
|
||||
# The interface will be at the index after "-s"
|
||||
try:
|
||||
interface_index = cmdline_args.index('-s')
|
||||
except ValueError as ex:
|
||||
LOG.error("No interface found in cmdline args. %s" % ex)
|
||||
return None
|
||||
|
||||
phc_interface = cmdline_args[interface_index + 1]
|
||||
if phc_interface == constants.CLOCK_REALTIME:
|
||||
LOG.info("PHC2SYS is using CLOCK_REALTIME, OS Clock is not being disciplined by a PHC")
|
||||
return None
|
||||
LOG.debug("PHC interface is %s" % phc_interface)
|
||||
return phc_interface
|
||||
|
||||
def _check_config_file_interface(self):
|
||||
with open(self.phc2sys_config, 'r') as f:
|
||||
config_lines = f.readlines()
|
||||
config_lines = [line.rstrip() for line in config_lines]
|
||||
|
||||
for line in config_lines:
|
||||
# Find the interface value inside the square brackets
|
||||
if re.match(r"^\[.*\]$", line) and line != "[global]":
|
||||
phc_interface = line.strip("[]")
|
||||
|
||||
LOG.debug("PHC interface is %s" % phc_interface)
|
||||
return phc_interface
|
||||
|
||||
def _get_interface_phc_device(self):
|
||||
"""Determine the phc device for the interface"""
|
||||
pattern = "/hostsys/class/net/" + self.phc_interface + "/device/ptp/*"
|
||||
ptp_device = glob(pattern)
|
||||
if len(ptp_device) == 0:
|
||||
LOG.error("No ptp device found at %s" % pattern)
|
||||
return None
|
||||
if len(ptp_device) > 1:
|
||||
LOG.error("More than one ptp device found at %s" % pattern)
|
||||
return None
|
||||
|
||||
ptp_device = os.path.basename(ptp_device[0])
|
||||
LOG.debug("Found ptp device %s at %s" % (ptp_device, pattern))
|
||||
return ptp_device
|
||||
|
||||
def get_os_clock_offset(self):
|
||||
"""Get the os CLOCK_REALTIME offset"""
|
||||
ptp_device_path = "/dev/" + self.ptp_device
|
||||
offset = subprocess.check_output([constants.PHC_CTL_PATH, ptp_device_path, 'cmp']
|
||||
).decode().split()[-1]
|
||||
offset = offset.strip("-ns")
|
||||
LOG.debug("PHC offset is %s" % offset)
|
||||
self.offset = offset
|
||||
|
||||
def set_os_clock_state(self):
|
||||
offset_int = int(self.offset)
|
||||
if offset_int > constants.PHC2SYS_TOLERANCE_HIGH or \
|
||||
offset_int < constants.PHC2SYS_TOLERANCE_LOW:
|
||||
LOG.warning("PHC2SYS offset is outside of tolerance, handling state change.")
|
||||
# TODO Implement handler for os clock state change
|
||||
pass
|
||||
else:
|
||||
LOG.info("PHC2SYS offset is within tolerance, OS clock state is LOCKED")
|
||||
self._state = OsClockState.Locked
|
||||
|
||||
def get_os_clock_state(self):
|
||||
return self._state
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# This file can be run in a ptp-notification pod to verify the functionality of
|
||||
# os_clock_monitor.
|
||||
test_monitor = OsClockMonitor()
|
||||
@ -20,13 +20,11 @@ import datetime
|
||||
import logging
|
||||
from trackingfunctionsdk.common.helpers import constants
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# dictionary includes PMC commands used and keywords of intrest
|
||||
ptp_oper_dict = {
|
||||
#[pmc cmd, ptp keywords,...]
|
||||
# [pmc cmd, ptp keywords,...]
|
||||
1: ["'GET PORT_DATA_SET'", constants.PORT_STATE],
|
||||
2: ["'GET TIME_STATUS_NP'", constants.GM_PRESENT, constants.MASTER_OFFSET],
|
||||
3: ["'GET PARENT_DATA_SET'", constants.GM_CLOCK_CLASS, constants.GRANDMASTER_IDENTITY],
|
||||
@ -37,13 +35,14 @@ ptp_oper_dict = {
|
||||
ptp4l_service_name = os.environ.get('PTP4L_SERVICE_NAME', 'ptp4l')
|
||||
phc2sys_service_name = os.environ.get('PHC2SYS_SERVICE_NAME', 'phc2sys')
|
||||
|
||||
|
||||
# run subprocess and returns out, err, errcode
|
||||
def run_shell2(dir, ctx, args):
|
||||
cwd = os.getcwd()
|
||||
os.chdir(dir)
|
||||
|
||||
process = subprocess.Popen(args, shell=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
out, err = process.communicate()
|
||||
errcode = process.returncode
|
||||
|
||||
@ -51,6 +50,7 @@ def run_shell2(dir, ctx, args):
|
||||
|
||||
return out, err, errcode
|
||||
|
||||
|
||||
def check_critical_resources():
|
||||
pmc = False
|
||||
ptp4l = False
|
||||
@ -67,6 +67,7 @@ def check_critical_resources():
|
||||
ptp4lconf = True
|
||||
return pmc, ptp4l, phc2sys, ptp4lconf
|
||||
|
||||
|
||||
def check_results(result, total_ptp_keywords, port_count):
|
||||
# sync state is in 'Locked' state and will be overwritten if
|
||||
# it is not the case
|
||||
@ -95,11 +96,12 @@ def check_results(result, total_ptp_keywords, port_count):
|
||||
sync_state = constants.FREERUN_PHC_STATE
|
||||
if (result[constants.GM_CLOCK_CLASS] not in
|
||||
[constants.CLOCK_CLASS_VALUE1,
|
||||
constants.CLOCK_CLASS_VALUE2,
|
||||
constants.CLOCK_CLASS_VALUE3]):
|
||||
constants.CLOCK_CLASS_VALUE2,
|
||||
constants.CLOCK_CLASS_VALUE3]):
|
||||
sync_state = constants.FREERUN_PHC_STATE
|
||||
return sync_state
|
||||
|
||||
|
||||
def ptpsync():
|
||||
result = {}
|
||||
total_ptp_keywords = 0
|
||||
@ -108,7 +110,7 @@ def ptpsync():
|
||||
ptp_dict_to_use = ptp_oper_dict
|
||||
len_dic = len(ptp_dict_to_use)
|
||||
|
||||
for key in range(1,len_dic+1):
|
||||
for key in range(1, len_dic + 1):
|
||||
cmd = ptp_dict_to_use[key][0]
|
||||
cmd = "pmc -b 0 -u -f /ptp/ptpinstance/ptp4l-" + ptp4l_service_name + ".conf " + cmd
|
||||
|
||||
@ -135,14 +137,14 @@ def ptpsync():
|
||||
LOG.warning('not received the expected list length')
|
||||
sys.exit(0)
|
||||
for item in ptp_keyword:
|
||||
if state[0] == item:
|
||||
if item == constants.PORT_STATE:
|
||||
port_count += 1
|
||||
result.update({constants.PORT.format(port_count):state[1]})
|
||||
else:
|
||||
state[1] = state[1].replace('\\n','')
|
||||
state[1] = state[1].replace('\'','')
|
||||
result.update({state[0]:state[1]})
|
||||
if state[0] == item:
|
||||
if item == constants.PORT_STATE:
|
||||
port_count += 1
|
||||
result.update({constants.PORT.format(port_count): state[1]})
|
||||
else:
|
||||
state[1] = state[1].replace('\\n', '')
|
||||
state[1] = state[1].replace('\'', '')
|
||||
result.update({state[0]: state[1]})
|
||||
# making sure at least one port is available
|
||||
if port_count == 0:
|
||||
port_count = 1
|
||||
@ -150,6 +152,7 @@ def ptpsync():
|
||||
total_ptp_keywords = total_ptp_keywords + port_count - 1
|
||||
return result, total_ptp_keywords, port_count
|
||||
|
||||
|
||||
def ptp_status(holdover_time, freq, sync_state, event_time):
|
||||
result = {}
|
||||
# holdover_time - time phc can maintain clock
|
||||
|
||||
@ -0,0 +1,9 @@
|
||||
from wsme import types as wtypes
|
||||
|
||||
EnumGnssState = wtypes.Enum(str, 'Locked', 'Freerun', 'Holdover')
|
||||
|
||||
|
||||
class OsClockState(object):
|
||||
Locked = "Locked"
|
||||
Freerun = "Freerun"
|
||||
Holdover = "Holdover"
|
||||
@ -0,0 +1,9 @@
|
||||
[global]
|
||||
##
|
||||
## Default Data Set
|
||||
##
|
||||
domainNumber 24
|
||||
message_tag phc-inst1
|
||||
uds_address /var/run/ptp4l-ptp-inst1
|
||||
|
||||
[ens2f0]
|
||||
@ -0,0 +1,83 @@
|
||||
#
|
||||
# Copyright (c) 2022 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import mock_open
|
||||
|
||||
import mock
|
||||
|
||||
from trackingfunctionsdk.common.helpers import constants
|
||||
from trackingfunctionsdk.common.helpers.os_clock_monitor import OsClockMonitor
|
||||
from trackingfunctionsdk.model.dto.osclockstate import OsClockState
|
||||
|
||||
testpath = os.environ.get("TESTPATH", "")
|
||||
|
||||
class OsClockMonitorTests(unittest.TestCase):
|
||||
os.environ["PHC2SYS_CONFIG"] = constants.PTP_CONFIG_PATH + "phc2sys-phc2sys-test.conf"
|
||||
clockmon = OsClockMonitor(init=False)
|
||||
|
||||
def test_set_phc2sys_instance(self):
|
||||
self.clockmon = OsClockMonitor(init=False)
|
||||
self.clockmon.set_phc2sys_instance()
|
||||
assert self.clockmon.phc2sys_instance == "phc2sys-test"
|
||||
|
||||
def test_check_config_file_interface(self):
|
||||
self.clockmon = OsClockMonitor(init=False)
|
||||
self.clockmon.phc2sys_config = testpath + "test_input_files/phc2sys-test.conf"
|
||||
self.assertEqual(self.clockmon._check_config_file_interface(), "ens2f0")
|
||||
|
||||
@mock.patch('trackingfunctionsdk.common.helpers.os_clock_monitor.open', new_callable=mock_open,
|
||||
read_data="101")
|
||||
def test_check_command_line_interface(self, mo):
|
||||
# Use mock to return the expected readline values
|
||||
# Success path
|
||||
handlers = (mo.return_value,
|
||||
mock_open(read_data="/usr/sbin/phc2sys\x00-f\x00/etc"
|
||||
"/ptpinstance/phc2sys-phc "
|
||||
"-inst1.conf\x00-w\x00-s\x00ens1f0\x00").return_value)
|
||||
mo.side_effect = handlers
|
||||
self.assertEqual(self.clockmon._check_command_line_interface("/var/run/"), "ens1f0")
|
||||
|
||||
# Failure path - no interface in command line params
|
||||
handlers = (mo.return_value,
|
||||
mock_open(read_data="/usr/sbin/phc2sys\x00-f\x00/etc/ptpinstance/phc2sys-phc"
|
||||
"-inst1.conf\x00-w\x00").return_value)
|
||||
mo.side_effect = handlers
|
||||
self.assertEqual(self.clockmon._check_command_line_interface("/var/run/"), None)
|
||||
|
||||
@mock.patch('trackingfunctionsdk.common.helpers.os_clock_monitor.glob',
|
||||
side_effect=[['/hostsys/class/net/ens1f0/device/ptp/ptp0'],
|
||||
['/hostsys/class/net/ens1f0/device/ptp/ptp0',
|
||||
'/hostsys/class/net/ens1f0/device/ptp/ptp1'],
|
||||
[]
|
||||
])
|
||||
def test_get_interface_phc_device(self, glob_patched):
|
||||
# Success path
|
||||
self.clockmon = OsClockMonitor(init=False)
|
||||
self.clockmon.phc_interface = "ens1f0"
|
||||
self.assertEqual(self.clockmon._get_interface_phc_device(), 'ptp0')
|
||||
|
||||
# Fail path #1 - multiple devices found
|
||||
self.assertEqual(self.clockmon._get_interface_phc_device(), None)
|
||||
|
||||
# Fail path #2 - no devices found
|
||||
self.assertEqual(self.clockmon._get_interface_phc_device(), None)
|
||||
|
||||
@mock.patch('trackingfunctionsdk.common.helpers.os_clock_monitor.subprocess.check_output',
|
||||
side_effect=[b'-37000000015ns'])
|
||||
def test_get_os_clock_offset(self, subprocess_patched):
|
||||
self.clockmon = OsClockMonitor(init=False)
|
||||
self.clockmon.ptp_device = 'ptp0'
|
||||
self.clockmon.get_os_clock_offset()
|
||||
assert self.clockmon.offset == '37000000015'
|
||||
|
||||
def test_set_os_closck_state(self):
|
||||
self.clockmon = OsClockMonitor(init=False)
|
||||
self.clockmon.offset = '37000000015'
|
||||
self.clockmon.set_os_clock_state()
|
||||
self.assertEqual(self.clockmon.get_os_clock_state(), OsClockState.Locked)
|
||||
|
||||
# TODO Test for handling clock state change to LOCKED and FREERUN
|
||||
Loading…
x
Reference in New Issue
Block a user