Handle N3000 FPGA reset

A recent config commit (6d639fca5d) was introduced that resets any
Intel N3000 FPGA devices on startup.  After an FPGA reset, it is
possible that the PCI addresses of the underlying FEC device may
change, in addition to the resetting of any configured drivers and
SR-IOV VF configuration on the FPGA FEC and NIC devices.

To handle the reset, it is required to do the following:

- Do not allow a user to configure an FEC device before the first
unlock (to allow the reset to occur).

- Delay the configuration of FEC and NIC devices until the reset
  has occurred.

Depends-On: https://review.opendev.org/737388
Story: 2006740
Task: 39948

Change-Id: I6d1bd988192dca868b8cc972045933ee8ac75e7c
Signed-off-by: Steven Webster <steven.webster@windriver.com>

puppet-manifests/src/manifests/worker.pp

@@ -27,6 +27,7 @@ include ::platform::remotelogging
 include ::platform::mtce
 include ::platform::sysinv
 include ::platform::devices
+include ::platform::interfaces::sriov
 include ::platform::grub
 include ::platform::collectd
 include ::platform::filesystem::compute

puppet-manifests/src/modules/platform/manifests/devices.pp

@@ -43,14 +43,24 @@ class platform::devices::qat (
 define platform::devices::sriov_enable (
   $num_vfs,
   $addr,
-  $driver
+  $driver,
+  $device_id
 ) {
   if ($driver == 'igb_uio') {
     $vf_file = 'max_vfs'
   } else {
     $vf_file = 'sriov_numvfs'
   }
-  if $num_vfs {
+  if ($num_vfs > 0) {
+    if ($device_id == '0d8f') {
+      exec { "Waiting for n3000 reset before enabling device: ${title}":
+        command   => 'test -e /var/run/.sysinv_n3000_reset',
+        path      => '/usr/bin/',
+        tries     => 60,
+        try_sleep => 1,
+      }
+      -> Exec["sriov-enable-device: ${title}"]
+    }
     exec { "sriov-enable-device: ${title}":
       command   => template('platform/sriov.enable-device.erb'),
       logoutput => true,
@@ -61,9 +71,19 @@ define platform::devices::sriov_enable (
 define platform::devices::sriov_bind (
   $addr,
   $driver,
-  $num_vfs = undef
+  $num_vfs = undef,
+  $device_id = undef
 ) {
   if ($driver != undef) and ($addr != undef) {
+    if ($device_id != undef) and ($device_id == '0d8f') {
+      exec { "Waiting for n3000 reset before binding device: ${title}":
+        command   => 'test -e /var/run/.sysinv_n3000_reset',
+        path      => '/usr/bin/',
+        tries     => 60,
+        try_sleep => 1,
+      }
+      -> Exec["sriov-bind-device: ${title}"]
+    }
     ensure_resource(kmod::load, $driver)
     exec { "sriov-bind-device: ${title}":
       command   => template('platform/sriov.bind-device.erb'),

puppet-manifests/src/modules/platform/manifests/network.pp

@@ -178,11 +178,13 @@ class platform::network::routes (
 
 define platform::interfaces::sriov_enable (
   $addr,
+  $device_id,
   $num_vfs,
+  $port_name,
   $vf_config = undef
 ) {
   $vf_file = 'sriov_numvfs'
-  if $num_vfs {
+  if ($num_vfs > 0) {
     exec { "sriov-enable-device: ${title}":
       command   => template('platform/sriov.enable-device.erb'),
       logoutput => true,
@@ -208,9 +210,24 @@ define platform::interfaces::sriov_bind (
 
 define platform::interfaces::sriov_vf_bind (
   $addr,
+  $device_id,
   $num_vfs,
+  $port_name,
   $vf_config
 ) {
+  if ($device_id == '0d58') {
+    exec { "Waiting for n3000 reset before binding device: ${title}":
+      command   => 'test -e /var/run/.sysinv_n3000_reset',
+      path      => '/usr/bin/',
+      tries     => 60,
+      try_sleep => 1,
+    }
+    -> exec { "Restarting n3000 NICs for interface: ${title}":
+      command => "ifdown ${port_name}; ifup ${port_name}",
+      path    => '/usr/sbin/',
+    }
+    -> Anchor['platform::networking']
+  }
   create_resources('platform::interfaces::sriov_bind', $vf_config, {})
 }
 
@@ -235,8 +252,6 @@ class platform::interfaces (
   $network_config = {},
 ) {
   create_resources('network_config', $network_config, {})
-
-  include ::platform::interfaces::sriov
 }