Add get_network_details_v2 in MAAS

Includes support for static networking, bonds and VLANs.

Implements: blueprint maas-network-config
Co-Authored-By: Adrian Vladu <avladu@cloudbasesolutions.com>
Change-Id: If0949ecfc3ec0cb65d4cd89cffd3055a110e737e

cloudbaseinit/metadata/services/maasservice.py

@@ -12,19 +12,41 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
 import re
+import sys
 
+import json
+import netaddr
 from oauthlib import oauth1
 from oslo_log import log as oslo_logging
 import requests
 
 from cloudbaseinit import conf as cloudbaseinit_conf
+from cloudbaseinit import exception
 from cloudbaseinit.metadata.services import base
+from cloudbaseinit.models import network as network_model
 from cloudbaseinit.utils import x509constants
 
 CONF = cloudbaseinit_conf.CONF
 LOG = oslo_logging.getLogger(__name__)
 
+MAAS_CONFIG_TYPE_PHYSICAL = "physical"
+MAAS_CONFIG_TYPE_BOND = "bond"
+MAAS_CONFIG_TYPE_VLAN = "vlan"
+MAAS_CONGIG_TYPE_NAMESERVER = "nameserver"
+
+MAAS_BOND_LACP_RATE_SLOW = "slow"
+MAAS_BOND_LACP_RATE_FAST = "fast"
+
+MAAS_SUBNET_TYPE_STATIC = "static"
+MAAS_SUBNET_TYPE_MANUAL = "manual"
+
+BOND_LACP_RATE_MAP = {
+    MAAS_BOND_LACP_RATE_SLOW: network_model.BOND_LACP_RATE_SLOW,
+    MAAS_BOND_LACP_RATE_FAST: network_model.BOND_LACP_RATE_FAST,
+}
+
 
 class _Realm(str):
     # There's a bug in oauthlib which ignores empty realm strings,
@@ -108,3 +130,188 @@ class MaaSHttpService(base.BaseHTTPMetadataService):
 
     def get_user_data(self):
         return self._get_cache_data('%s/user-data' % self._metadata_version)
+
+    @staticmethod
+    def _get_network_data():
+        if sys.platform != "win32":
+            return
+
+        path = os.path.join(
+            os.environ["systemdrive"], "\\curtin\\network.json")
+        if not os.path.isfile(path):
+            path = os.path.join(os.environ["systemdrive"], "\\network.json")
+            if not os.path.isfile(path):
+                path = None
+
+        if path:
+            json_data = open(path, "rb").read()
+            return json.loads(json_data.decode('utf-8'))
+
+    @staticmethod
+    def _is_link_enabled(subnets):
+        return MAAS_SUBNET_TYPE_MANUAL not in [s.get("type") for s in subnets]
+
+    @staticmethod
+    def _parse_config_link(config):
+        link_id = config.get("id")
+        name = config.get("name")
+        mac = config.get("mac_address")
+        mtu = config.get("mtu")
+        maas_link_type = config.get("type")
+        subnets = config.get("subnets", [])
+        params = config.get("params", {})
+        bond = None
+        vlan_id = None
+        vlan_link = None
+        link_enabled = False
+
+        if maas_link_type == MAAS_CONFIG_TYPE_PHYSICAL:
+            link_type = network_model.LINK_TYPE_PHYSICAL
+            link_enabled = MaaSHttpService._is_link_enabled(subnets)
+        elif maas_link_type == MAAS_CONFIG_TYPE_BOND:
+            link_type = network_model.LINK_TYPE_BOND
+            bond_interfaces = config.get("bond_interfaces")
+            bond_mode = params.get("bond-mode")
+            bond_xmit_hash_policy = params.get("bond-xmit-hash-policy")
+            maas_bond_lacp_rate = params.get("bond-lacp-rate")
+
+            if bond_mode not in network_model.AVAILABLE_BOND_TYPES:
+                raise exception.CloudbaseInitException(
+                    "Unsupported bond mode: %s" % bond_mode)
+
+            if (bond_xmit_hash_policy is not None and
+                    bond_xmit_hash_policy not in
+                    network_model.AVAILABLE_BOND_LB_ALGORITHMS):
+                raise exception.CloudbaseInitException(
+                    "Unsupported bond hash policy: %s" % bond_xmit_hash_policy)
+
+            bond = network_model.Bond(
+                members=bond_interfaces,
+                type=bond_mode,
+                lb_algorithm=bond_xmit_hash_policy,
+                lacp_rate=BOND_LACP_RATE_MAP.get(maas_bond_lacp_rate))
+            link_enabled = True
+        elif maas_link_type == MAAS_CONFIG_TYPE_VLAN:
+            link_type = network_model.LINK_TYPE_VLAN
+            vlan_link = config.get("vlan_link")
+            vlan_id = config.get("vlan_id")
+            link_enabled = True
+        else:
+            raise exception.CloudbaseInitException(
+                "Unsupported MAAS link type: %s" % maas_link_type)
+
+        link = network_model.Link(
+            id=link_id,
+            name=name,
+            type=link_type,
+            enabled=link_enabled,
+            mac_address=mac,
+            mtu=mtu,
+            bond=bond,
+            vlan_id=vlan_id,
+            vlan_link=vlan_link)
+
+        networks = []
+        subnets = config.get("subnets", [])
+        for subnet in subnets:
+            maas_subnet_type = subnet.get("type")
+            if maas_subnet_type == MAAS_SUBNET_TYPE_STATIC:
+                address_cidr = subnet.get("address")
+                gateway = subnet.get("gateway")
+                dns_nameservers = subnet.get("dns_nameservers")
+
+                # TODO(alexpilotti): Add support for extra routes
+                if gateway is not None:
+                    if netaddr.valid_ipv6(gateway):
+                        default_network_cidr = u"::/0"
+                    else:
+                        default_network_cidr = u"0.0.0.0/0"
+
+                    routes = [
+                        network_model.Route(
+                            network_cidr=default_network_cidr,
+                            gateway=gateway
+                        )
+                    ]
+                else:
+                    routes = []
+                net = network_model.Network(
+                    link=link_id,
+                    address_cidr=address_cidr,
+                    dns_nameservers=dns_nameservers,
+                    routes=routes,
+                )
+                networks.append(net)
+
+        return link, networks
+
+    @staticmethod
+    def _parse_config_nameserver(config):
+        return network_model.NameServerService(
+            addresses=config.get("address", []),
+            search=config.get("search", []))
+
+    @staticmethod
+    def _parse_config_item(config):
+        link = None
+        networks = None
+        service = None
+
+        config_type = config.get("type")
+        if config_type == MAAS_CONGIG_TYPE_NAMESERVER:
+            service = MaaSHttpService._parse_config_nameserver(config)
+        elif config_type in [
+                MAAS_CONFIG_TYPE_PHYSICAL,
+                MAAS_CONFIG_TYPE_BOND,
+                MAAS_CONFIG_TYPE_VLAN]:
+            link, networks = MaaSHttpService._parse_config_link(config)
+        else:
+            raise exception.CloudbaseInitException(
+                "Unsupported item type: %s" % config_type)
+
+        return link, networks, service
+
+    @staticmethod
+    def _enable_bond_physical_links(links):
+        # The MAAS metadata sets the NIC subnet type as "manual" for both
+        # disconnected NICs and bond members. We need to make sure that the
+        # latter are enabled.
+        for link1 in links:
+            if link1.type == network_model.LINK_TYPE_BOND:
+                for index, link2 in enumerate(links):
+                    if (link2.type == network_model.LINK_TYPE_PHYSICAL and
+                            not link2.enabled and
+                            link2.id in link1.bond.members):
+                        links[index] = link2._replace(enabled=True)
+
+    def get_network_details_v2(self):
+        network_data = self._get_network_data()
+        if not network_data:
+            return
+
+        version = network_data.get("version")
+        if version != 1:
+            raise exception.CloudbaseInitException(
+                'Unsupported MAAS network metadata version: %s' % version)
+
+        links = []
+        networks = []
+        services = []
+
+        config = network_data.get("config", [])
+        for config_item in config:
+            link, link_networks, service = self._parse_config_item(config_item)
+            if link:
+                links.append(link)
+            if link_networks:
+                networks.extend(link_networks),
+            if service:
+                services.append(service)
+
+        self._enable_bond_physical_links(links)
+
+        return network_model.NetworkDetailsV2(
+            links=links,
+            networks=networks,
+            services=services
+        )

cloudbaseinit/tests/metadata/services/test_maasservice.py

@@ -20,7 +20,9 @@ except ImportError:
     import mock
 
 from cloudbaseinit import conf as cloudbaseinit_conf
+from cloudbaseinit import exception
 from cloudbaseinit.metadata.services import maasservice
+from cloudbaseinit.models import network as network_model
 from cloudbaseinit.tests import testutils
 from cloudbaseinit.utils import x509constants
 
@@ -159,3 +161,244 @@ class MaaSHttpServiceTest(unittest.TestCase):
             '%s/user-data' %
             self._maasservice._metadata_version)
         self.assertEqual(mock_get_cache_data.return_value, response)
+
+    def _get_network_data(self):
+        return {
+            "version": mock.sentinel.network_data_version,
+            "config": [{
+                "mtu": mock.sentinel.link_mtu1,
+                "name": mock.sentinel.link_name1,
+                "subnets": [{
+                    "type": maasservice.MAAS_SUBNET_TYPE_MANUAL
+                }],
+                "type": maasservice.MAAS_CONFIG_TYPE_PHYSICAL,
+                "mac_address": mock.sentinel.link_mac1,
+                "id": mock.sentinel.link_id1
+            }, {
+                "mtu": mock.sentinel.link_mtu2,
+                "name": mock.sentinel.link_name2,
+                "subnets": [{
+                    "type": maasservice.MAAS_SUBNET_TYPE_MANUAL
+                }],
+                "type": maasservice.MAAS_CONFIG_TYPE_PHYSICAL,
+                "mac_address": mock.sentinel.link_mac2,
+                "id": mock.sentinel.link_id2
+            }, {
+                "mtu": mock.sentinel.link_mtu3,
+                "name": mock.sentinel.link_name3,
+                "subnets": [{
+                    "type": maasservice.MAAS_SUBNET_TYPE_MANUAL
+                }],
+                "type": maasservice.MAAS_CONFIG_TYPE_PHYSICAL,
+                "mac_address": mock.sentinel.link_mac3,
+                "id": mock.sentinel.link_id3
+            }, {
+                "name": mock.sentinel.bond_name1,
+                "id": mock.sentinel.bond_id1,
+                "type": maasservice.MAAS_CONFIG_TYPE_BOND,
+                "mac_address": mock.sentinel.bond_mac1,
+                "bond_interfaces": [
+                    mock.sentinel.link_id1,
+                    mock.sentinel.link_id2
+                ],
+                "mtu": mock.sentinel.bond_mtu1,
+                "subnets": [{
+                    "address": mock.sentinel.bond_subnet_address1,
+                    "gateway": mock.sentinel.bond_subnet_gateway1,
+                    "type": maasservice.MAAS_SUBNET_TYPE_STATIC,
+                    "dns_nameservers": [
+                        mock.sentinel.bond_subnet_dns1,
+                        mock.sentinel.bond_subnet_dns2]
+                }, {
+                    "address": mock.sentinel.bond_subnet_address2,
+                    "type": maasservice.MAAS_SUBNET_TYPE_STATIC,
+                    "dns_nameservers": []
+                }],
+                "params": {
+                    "bond-downdelay": 0,
+                    "bond-xmit-hash-policy": mock.sentinel.bond_lb_algo1,
+                    "bond-mode": mock.sentinel.bond_mode1,
+                    "bond-updelay": 0,
+                    "bond-miimon": 100,
+                    "bond-lacp-rate": maasservice.MAAS_BOND_LACP_RATE_FAST
+                }
+            }, {
+                "type": maasservice.MAAS_CONFIG_TYPE_VLAN,
+                "mtu": mock.sentinel.vlan_mtu1,
+                "name": mock.sentinel.vlan_name1,
+                "subnets": [{
+                    "gateway": mock.sentinel.vlan_subnet_gateway1,
+                    "address": mock.sentinel.vlan_subnet_address1,
+                    "type": maasservice.MAAS_SUBNET_TYPE_STATIC,
+                    "dns_nameservers": []
+                }],
+                "vlan_id": mock.sentinel.vlan_id1,
+                "vlan_link": mock.sentinel.bond_id1,
+                "id": mock.sentinel.vlan_link_id1
+            }, {
+                "type": mock.sentinel.nameserver_config_type,
+                "search": [
+                    mock.sentinel.dns_search1
+                ],
+                "address": [
+                    mock.sentinel.bond_subnet_dns1,
+                    mock.sentinel.bond_subnet_dns2
+                ],
+            }]
+        }
+
+    @mock.patch("cloudbaseinit.metadata.services.maasservice.MaaSHttpService"
+                "._get_network_data")
+    def _test_get_network_details_v2(self, mock_get_network_data,
+                                     unsupported_version=False,
+                                     invalid_bond_type=False,
+                                     invalid_bond_lb_algo=False,
+                                     unsupported_config_type=False):
+        mock.sentinel.bond_subnet_address1 = "10.0.0.1/24"
+        mock.sentinel.bond_subnet_gateway1 = "10.0.0.254"
+        mock.sentinel.bond_subnet_address2 = "172.16.0.1/16"
+        mock.sentinel.vlan_subnet_address1 = "2001:cdba::3257:9652/24"
+        mock.sentinel.vlan_subnet_gateway1 = "2001:cdba::3257:1"
+
+        if invalid_bond_type:
+            mock.sentinel.bond_mode1 = "invalid bond type"
+        else:
+            mock.sentinel.bond_mode1 = network_model.BOND_TYPE_BALANCE_ALB
+
+        if invalid_bond_lb_algo:
+            mock.sentinel.bond_lb_algo1 = "invalid lb algorithm"
+        else:
+            mock.sentinel.bond_lb_algo1 = network_model.BOND_LB_ALGO_L2
+
+        if unsupported_version:
+            mock.sentinel.network_data_version = "unsupported"
+        else:
+            mock.sentinel.network_data_version = 1
+
+        if unsupported_config_type:
+            mock.sentinel.nameserver_config_type = "unsupported"
+        else:
+            mock.sentinel.nameserver_config_type = "nameserver"
+
+        network_data = self._get_network_data()
+        mock_get_network_data.return_value = network_data
+
+        if (unsupported_version or invalid_bond_type or invalid_bond_lb_algo or
+                unsupported_config_type):
+            with self.assertRaises(exception.CloudbaseInitException):
+                self._maasservice.get_network_details_v2()
+            return
+
+        network_details = self._maasservice.get_network_details_v2()
+
+        self.assertEqual(1, len([
+            l for l in network_details.links if
+            l.type == network_model.LINK_TYPE_PHYSICAL and
+            l.id == mock.sentinel.link_id1 and
+            l.name == mock.sentinel.link_name1 and
+            l.enabled is True and
+            l.mac_address == mock.sentinel.link_mac1 and
+            l.mtu == mock.sentinel.link_mtu1]))
+
+        self.assertEqual(1, len([
+            l for l in network_details.links if
+            l.type == network_model.LINK_TYPE_PHYSICAL and
+            l.id == mock.sentinel.link_id2 and
+            l.name == mock.sentinel.link_name2 and
+            l.enabled is True and
+            l.mac_address == mock.sentinel.link_mac2 and
+            l.mtu == mock.sentinel.link_mtu2]))
+
+        # Disconnected network adapter, ensure it's not enabled
+        self.assertEqual(1, len([
+            l for l in network_details.links if
+            l.type == network_model.LINK_TYPE_PHYSICAL and
+            l.id == mock.sentinel.link_id3 and
+            l.name == mock.sentinel.link_name3 and
+            l.enabled is False and
+            l.mac_address == mock.sentinel.link_mac3 and
+            l.mtu == mock.sentinel.link_mtu3]))
+
+        self.assertEqual(1, len([
+            l for l in network_details.links if
+            l.type == network_model.LINK_TYPE_BOND and
+            l.id == mock.sentinel.bond_id1 and
+            l.enabled is True and
+            l.name == mock.sentinel.bond_name1 and
+            l.mtu == mock.sentinel.bond_mtu1 and
+            l.mac_address == mock.sentinel.bond_mac1 and
+            l.vlan_link is None and
+            l.vlan_id is None and
+            l.bond.type == network_model.BOND_TYPE_BALANCE_ALB and
+            l.bond.members == [
+                mock.sentinel.link_id1, mock.sentinel.link_id2] and
+            l.bond.lb_algorithm == network_model.BOND_LB_ALGO_L2 and
+            l.bond.lacp_rate == network_model.BOND_LACP_RATE_FAST]))
+
+        self.assertEqual(1, len([
+            l for l in network_details.links if
+            l.type == network_model.LINK_TYPE_VLAN and
+            l.id == mock.sentinel.vlan_link_id1 and
+            l.name == mock.sentinel.vlan_name1 and
+            l.enabled is True and
+            l.mac_address is None and
+            l.mtu == mock.sentinel.vlan_mtu1 and
+            l.vlan_link == mock.sentinel.bond_id1 and
+            l.vlan_id == mock.sentinel.vlan_id1]))
+
+        self.assertEqual(3, len(network_details.networks))
+
+        network_bond1 = [
+            n for n in network_details.networks
+            if n.address_cidr == mock.sentinel.bond_subnet_address1 and
+            n.dns_nameservers == [
+                mock.sentinel.bond_subnet_dns1,
+                mock.sentinel.bond_subnet_dns2] and
+            n.link == mock.sentinel.bond_id1 and
+            n.routes == [network_model.Route(
+                network_cidr=u'0.0.0.0/0',
+                gateway=mock.sentinel.bond_subnet_gateway1
+            )]]
+        self.assertEqual(1, len(network_bond1))
+
+        network_bond2 = [
+            n for n in network_details.networks
+            if n.address_cidr == mock.sentinel.bond_subnet_address2 and
+            n.dns_nameservers == [] and
+            n.link == mock.sentinel.bond_id1 and
+            n.routes == []]
+        self.assertEqual(1, len(network_bond2))
+
+        network_vlan1 = [
+            n for n in network_details.networks
+            if n.address_cidr == mock.sentinel.vlan_subnet_address1 and
+            n.dns_nameservers == [] and
+            n.link == mock.sentinel.vlan_link_id1 and
+            n.routes == [network_model.Route(
+                network_cidr=u'::/0',
+                gateway=mock.sentinel.vlan_subnet_gateway1
+            )]]
+        self.assertEqual(1, len(network_vlan1))
+
+        self.assertEqual(
+            [network_model.NameServerService(
+                addresses=[
+                    mock.sentinel.bond_subnet_dns1,
+                    mock.sentinel.bond_subnet_dns2],
+                search=[mock.sentinel.dns_search1])],
+            network_details.services)
+
+    def test_get_network_details_v2(self):
+        self._test_get_network_details_v2()
+
+    def test_get_network_details_v2_unsupported_version(self):
+        self._test_get_network_details_v2(unsupported_version=True)
+
+    def test_get_network_details_v2_unsupported_config_type(self):
+        self._test_get_network_details_v2(unsupported_config_type=True)
+
+    def test_get_network_details_v2_invalid_bond_type(self):
+        self._test_get_network_details_v2(invalid_bond_type=True)
+
+    def test_get_network_details_v2_invalid_bond_lb_algo(self):
+        self._test_get_network_details_v2(invalid_bond_lb_algo=True)