airship/drydock
Code Issues Proposed changes

DRYD21 - Add multirack to YAML

Browse Source 
- Add a Rack kind with information about a rack
- Add a dhcp_relay section to network definition
- Add a Rack model to the OVO objects
- Add ingester logic for Rack model
- Add unit tests for Rack model parsing

Change-Id: Ieb21350fbb7ee712a66d19c31a8df24bc013b69f
This commit is contained in:
Scott Hussey 2017-08-24 12:57:19 -05:00
parent e892df58dc
commit dc9fc231da
9 changed files with 389 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drydock_provisioner/ingester/__init__.py
View File

@ -27,6 +27,7 @@ import drydock_provisioner.objects.hwprofile as hwprofile
import drydock_provisioner.objects.node as node
import drydock_provisioner.objects.hostprofile as hostprofile
import drydock_provisioner.objects.promenade as prom
import drydock_provisioner.objects.rack as rack
from drydock_provisioner.statemgmt import DesignState
@ -131,6 +132,8 @@ class Ingester(object):
design_data.add_baremetal_node(m)
elif type(m) is prom.PromenadeConfig:
design_data.add_promenade_config(m)
elif type(m) is rack.Rack:
design_data.add_rack(m)
design_state.put_design(design_data)
return design_items
else:

67
drydock_provisioner/ingester/plugins/yaml.py
View File

@ -11,11 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# AIC YAML Ingester - This data ingester will consume a AIC YAML design
# file
#
"""YAML Ingester.
This data ingester will consume YAML site topology documents."""
import yaml
import logging
import base64
@ -34,16 +31,15 @@ class YamlIngester(IngesterPlugin):
def get_name(self):
return "yaml"
"""
AIC YAML ingester params
filenames - Array of absolute path to the YAML files to ingest
returns an array of objects from drydock_provisioner.model
"""
def ingest_data(self, **kwargs):
"""Parse and save design data.
:param filenames: Array of absolute path to the YAML files to ingest
:param content: String of valid YAML
returns an array of objects from drydock_provisioner.objects
"""
models = []
if 'filenames' in kwargs:
@ -66,11 +62,9 @@ class YamlIngester(IngesterPlugin):
return models
"""
Translate a YAML string into the internal Drydock model
"""
def parse_docs(self, yaml_string):
"""Translate a YAML string into the internal Drydock model."""
models = []
self.logger.debug(
"yamlingester:parse_docs - Parsing YAML string \n%s" %
@ -123,7 +117,40 @@ class YamlIngester(IngesterPlugin):
models.append(model)
else:
raise ValueError(
'Unknown API version %s of Region kind' %
"Unknown API version %s of Region kind" %
(api_version))
elif kind == 'Rack':
if api_version == "v1":
model = objects.Rack()
metadata = d.get('metadata', {})
spec = d.get('spec', {})
model.name = metadata.get('name', None)
model.site = metadata.get('region', None)
model.tor_switches = objects.TorSwitchList()
tors = spec.get('tor_switches', {})
for k, v in tors.items():
tor = objects.TorSwitch()
tor.switch_name = k
tor.mgmt_ip = v.get('mgmt_ip', None)
tor.sdn_api_uri = v.get('sdn_api_url', None)
model.tor_switches.append(tor)
location = spec.get('location', {})
model.location = dict()
for k, v in location.items():
model.location[k] = v
model.local_networks = [n for n in spec.get('local_networks', [])]
models.append(model)
else:
raise ValueError(
"Unknown API version %s of Rack kind" %
(api_version))
elif kind == 'NetworkLink':
if api_version == "v1":
@ -230,6 +257,12 @@ class YamlIngester(IngesterPlugin):
'metric':
r.get('metric', None),
})
dhcp_relay = spec.get('dhcp_relay', None)
if dhcp_relay is not None:
model.dhcp_relay_self_ip = dhcp_relay.get('self_ip', None)
model.dhcp_relay_upstream_target = dhcp_relay.get('upstream_target', None)
models.append(model)
elif kind == 'HardwareProfile':
if api_version == 'v1':

1
drydock_provisioner/objects/__init__.py
View File

@ -30,6 +30,7 @@ def register_all():
importlib.import_module('drydock_provisioner.objects.hwprofile')
importlib.import_module('drydock_provisioner.objects.site')
importlib.import_module('drydock_provisioner.objects.promenade')
importlib.import_module('drydock_provisioner.objects.rack')
# Utility class for calculating inheritance

2
drydock_provisioner/objects/network.py
View File

@ -103,6 +103,8 @@ class Network(base.DrydockPersistentObject, base.DrydockObject):
'ranges': ovo_fields.ListOfDictOfNullableStringsField(),
# Keys of routes are 'subnet', 'gateway', 'metric'
'routes': ovo_fields.ListOfDictOfNullableStringsField(),
'dhcp_relay_self_ip': ovo_fields.StringField(nullable=True),
'dhcp_relay_upstream_target': ovo_fields.StringField(nullable=True),
}
def __init__(self, **kwargs):

86
drydock_provisioner/objects/rack.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Model for a server rack in a site."""
import oslo_versionedobjects.fields as obj_fields
import drydock_provisioner.objects.base as base
import drydock_provisioner.objects.fields as hd_fields
@base.DrydockObjectRegistry.register
class Rack(base.DrydockPersistentObject, base.DrydockObject):
VERSION = '1.0'
fields = {
'name': obj_fields.StringField(nullable=False),
'site': obj_fields.StringField(nullable=False),
'source': hd_fields.ModelSourceField(nullable=False),
'tor_switches': obj_fields.ObjectField(
'TorSwitchList', nullable=False),
'location': obj_fields.DictOfStringsField(nullable=False),
'local_networks': obj_fields.ListOfStringsField(nullable=True),
}
def __init__(self, **kwargs):
super().__init__(**kwargs)
def get_id(self):
return self.get_name()
def get_name(self):
return self.name
@base.DrydockObjectRegistry.register
class RackList(base.DrydockObjectListBase, base.DrydockObject):
VERSION = '1.0'
fields = {'objects': obj_fields.ListOfObjectsField('Rack')}
@base.DrydockObjectRegistry.register
class TorSwitch(base.DrydockObject):
VERSION = '1.0'
fields = {
'switch_name':
obj_fields.StringField(),
'mgmt_ip':
obj_fields.StringField(nullable=True),
'sdn_api_uri':
obj_fields.StringField(nullable=True),
}
def __init__(self, **kwargs):
super().__init__(**kwargs)
# HostInterface is keyed by device_name
def get_id(self):
return self.get_name()
def get_name(self):
return self.switch_name
@base.DrydockObjectRegistry.register
class TorSwitchList(base.DrydockObjectListBase, base.DrydockObject):
VERSION = '1.0'
fields = {'objects': obj_fields.ListOfObjectsField('TorSwitch')}

18
drydock_provisioner/objects/site.py
View File

@ -148,6 +148,8 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject):
ovo_fields.ObjectField('BaremetalNodeList', nullable=True),
'prom_configs':
ovo_fields.ObjectField('PromenadeConfigList', nullable=True),
'racks':
ovo_fields.ObjectField('RackList', nullable=True),
}
def __init__(self, **kwargs):
@ -201,6 +203,22 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject):
raise errors.DesignError(
"NetworkLink %s not found in design state" % link_key)
def add_rack(self, new_rack):
if new_rack is None:
raise errors.DesignError("Invalid Rack model")
if self.racks is None:
self.racks = objects.RackList()
self.racks.append(new_rack)
def get_rack(self, rack_key):
for r in self.racks:
if r.get_id() == rack_key:
return r
raise errors.DesignError(
"Rack %s not found in design state" % rack_key)
def add_host_profile(self, new_host_profile):
if new_host_profile is None:
raise errors.DesignError("Invalid HostProfile model")

1
tests/unit/test_ingester.py
View File

@ -16,6 +16,7 @@ from drydock_provisioner.ingester import Ingester
from drydock_provisioner.statemgmt import DesignState
import drydock_provisioner.objects as objects
import logging
import pytest
import shutil
import os

90
tests/unit/test_ingester_rack_model.py Normal file
View File

@ -0,0 +1,90 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Test that rack models are properly parsed."""
from drydock_provisioner.ingester import Ingester
from drydock_provisioner.statemgmt import DesignState
import drydock_provisioner.objects as objects
import drydock_provisioner.error as errors
import logging
import pytest
import shutil
import os
import drydock_provisioner.ingester.plugins.yaml
class TestClass(object):
def test_rack_parse(self, input_files):
objects.register_all()
input_file = input_files.join("fullsite.yaml")
design_state = DesignState()
design_data = objects.SiteDesign()
design_id = design_data.assign_id()
design_state.post_design(design_data)
ingester = Ingester()
ingester.enable_plugins(
['drydock_provisioner.ingester.plugins.yaml.YamlIngester'])
ingester.ingest_data(
plugin_name='yaml',
design_state=design_state,
filenames=[str(input_file)],
design_id=design_id)
design_data = design_state.get_design(design_id)
rack = design_data.get_rack('rack1')
assert rack.location.get('grid') == 'EG12'
def test_rack_not_found(self, input_files):
objects.register_all()
input_file = input_files.join("fullsite.yaml")
design_state = DesignState()
design_data = objects.SiteDesign()
design_id = design_data.assign_id()
design_state.post_design(design_data)
ingester = Ingester()
ingester.enable_plugins(
['drydock_provisioner.ingester.plugins.yaml.YamlIngester'])
ingester.ingest_data(
plugin_name='yaml',
design_state=design_state,
filenames=[str(input_file)],
design_id=design_id)
design_data = design_state.get_design(design_id)
with pytest.raises(errors.DesignError):
rack = design_data.get_rack('foo')
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(
str(request.fspath)) + "/" + "../yaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir

251
tests/yaml_samples/fullsite.yaml
View File

@ -98,7 +98,7 @@ spec:
# If this link is a bond of physical links, how is it configured
# 802.3ad
# active-backup
# balance-rr
# balance-rr
# Can add support for others down the road
bonding:
mode: 802.3ad
@ -117,6 +117,32 @@ spec:
- mgmt
---
apiVersion: 'drydock/v1'
kind: Rack
metadata:
name: rack1
region: sitename
date: 24-AUG-2017
author: sh8121@att.com
description: A equipment rack
spec:
# List of TOR switches in this rack
tor_switches:
switch01name:
mgmt_ip: 1.1.1.1
sdn_api_uri: polo+https://polo-api.web.att.com/switchmgmt?switch=switch01name
switch02name:
mgmt_ip: 1.1.1.2
sdn_api_uri: polo+https://polo-api.web.att.com/switchmgmt?switch=switch02name
# GIS data for this rack
location:
clli: HSTNTXMOCG0
grid: EG12
# Networks wholly contained to this rack
# Nodes in a rack can only connect to local_networks of that rack
local_networks:
- pxe-rack1
---
apiVersion: 'drydock/v1'
kind: Network
metadata:
name: oob
@ -128,12 +154,12 @@ spec:
allocation: static
cidr: 172.16.100.0/24
ranges:
- type: static
start: 172.16.100.15
end: 172.16.100.254
- type: static
start: 172.16.100.15
end: 172.16.100.254
dns:
domain: ilo.sitename.att.com
servers: 172.16.100.10
domain: ilo.sitename.att.com
servers: 172.16.100.10
---
apiVersion: 'drydock/v1'
kind: Network
@ -142,26 +168,30 @@ metadata:
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
# Layer 2 VLAN segment id, could support other segmentations. Optional
# Layer 2 VLAN segment id, could support other segmentations. Optional
vlan_id: '99'
# How are addresses assigned?
allocation: dhcp
# MTU for this VLAN interface, if not specified it will be inherited from the link
# If this network utilizes a dhcp relay, where does it forward DHCPDISCOVER requests to?
dhcp_relay:
# Required if Drydock is configuring a switch with the relay
self_ip: 172.16.0.4
# Can refer to a unicast IP
upstream_target: 172.16.5.5
# MTU for this VLAN interface, if not specified it will be inherited from the link
mtu: 1500
# Network address
# Network address
cidr: 172.16.0.0/24
# Desribe IP address ranges
ranges:
- type: dhcp
start: 172.16.0.5
end: 172.16.0.254
# DNS settings for this network
# Desribe IP address ranges
ranges:
- type: dhcp
start: 172.16.0.5
end: 172.16.0.254
# DNS settings for this network
dns:
# Domain addresses on this network will be registered under
# Domain addresses on this network will be registered under
domain: admin.sitename.att.com
# DNS servers that a server using this network as its default gateway should use
# DNS servers that a server using this network as its default gateway should use
servers: 172.16.0.10
---
apiVersion: 'drydock/v1'
@ -174,24 +204,22 @@ metadata:
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '100'
# How are addresses assigned?
allocation: static
# Allow MTU to be inherited from link the network rides on
mtu: 1500
# Network address
cidr: 172.16.1.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.1.15
end: 172.16.1.254
ranges:
- type: static
start: 172.16.1.15
end: 172.16.1.254
# Static routes to be added for this network
routes:
- subnet: 0.0.0.0/0
- subnet: 0.0.0.0/0
# A blank gateway would leave to a static route specifying
# only the interface as a source
gateway: 172.16.1.1
metric: 10
gateway: 172.16.1.1
metric: 10
# DNS settings for this network
dns:
# Domain addresses on this network will be registered under
@ -209,16 +237,15 @@ metadata:
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '101'
allocation: static
mtu: 9000
cidr: 172.16.2.0/24
# Desribe IP address ranges
ranges:
ranges:
# Type can be reserved (not used for baremetal), static (all explicit
# assignments should fall here), dhcp (will be used by a DHCP server on this network)
- type: static
start: 172.16.2.15
end: 172.16.2.254
- type: static
start: 172.16.2.15
end: 172.16.2.254
dns:
domain: priv.sitename.example.com
servers: 172.16.2.9,172.16.2.10
@ -233,20 +260,18 @@ metadata:
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '102'
# How are addresses assigned?
allocation: static
# MTU size for the VLAN interface
mtu: 1500
cidr: 172.16.3.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
routes:
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 10
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 10
dns:
domain: sitename.example.com
servers: 8.8.8.8
@ -274,7 +299,7 @@ spec:
credential: admin
# Specify storage layout of base OS. Ceph out of scope
storage:
# How storage should be carved up: lvm (logical volumes), flat
# How storage should be carved up: lvm (logical volumes), flat
# (single partition)
layout: lvm
# Info specific to the boot and root disk/partitions
@ -289,18 +314,18 @@ spec:
# Info for additional partitions. Need to balance between
# flexibility and complexity
partitions:
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
# Platform (Operating System) settings
platform:
image: ubuntu_16.04
@ -328,7 +353,7 @@ spec:
# host_profile inheritance allows for deduplication of common CIs
# Inheritance is additive for CIs that are lists of multiple items
# To remove an inherited list member, prefix the primary key value
# with '!'.
# with '!'.
host_profile: defaults
# Hardware profile will map hardware specific details to the abstract
# names uses in the host profile as well as specify hardware specific
@ -342,31 +367,31 @@ spec:
interfaces:
# Keyed on device_name
# pxe is a special marker indicating which device should be used for pxe boot
- device_name: pxe
# The network link attached to this
device_link: pxe
# Slaves will specify aliases from hwdefinition.yaml
slaves:
- prim_nic01
# Which networks will be configured on this interface
networks:
- pxe
- device_name: bond0
network_link: gp
# If multiple slaves are specified, but no bonding config
# is applied to the link, design validation will fail
slaves:
- prim_nic01
- prim_nic02
# If multiple networks are specified, but no trunking
# config is applied to the link, design validation will fail
networks:
- mgmt
- private
- device_name: pxe
# The network link attached to this
device_link: pxe
# Slaves will specify aliases from hwdefinition.yaml
slaves:
- prim_nic01
# Which networks will be configured on this interface
networks:
- pxe
- device_name: bond0
network_link: gp
# If multiple slaves are specified, but no bonding config
# is applied to the link, design validation will fail
slaves:
- prim_nic01
- prim_nic02
# If multiple networks are specified, but no trunking
# config is applied to the link, design validation will fail
networks:
- mgmt
- private
metadata:
# Explicit tag assignment
tags:
- 'test'
- 'test'
---
apiVersion: 'drydock/v1'
kind: BaremetalNode
@ -381,26 +406,26 @@ spec:
# the hostname for a server, could be used in multiple DNS domains to
# represent different interfaces
interfaces:
- device_name: bond0
networks:
# '!' prefix for the value of the primary key indicates a record should be removed
- '!private'
- device_name: bond0
networks:
# '!' prefix for the value of the primary key indicates a record should be removed
- '!private'
# Addresses assigned to network interfaces
addressing:
# Which network the address applies to. If a network appears in addressing
# that isn't assigned to an interface, design validation will fail
- network: pxe
- network: pxe
# The address assigned. Either a explicit IPv4 or IPv6 address
# or dhcp or slaac
address: dhcp
- network: mgmt
address: 172.16.1.20
- network: public
address: 172.16.3.20
- network: oob
address: 172.16.100.20
address: dhcp
- network: mgmt
address: 172.16.1.20
- network: public
address: 172.16.3.20
- network: oob
address: 172.16.100.20
metadata:
rack: rack01
rack: rack1
---
apiVersion: 'drydock/v1'
kind: BaremetalNode
@ -413,16 +438,16 @@ metadata:
spec:
host_profile: k8-node
addressing:
- network: pxe
address: dhcp
- network: mgmt
address: 172.16.1.21
- network: private
address: 172.16.2.21
- network: oob
address: 172.16.100.21
- network: pxe
address: dhcp
- network: mgmt
address: 172.16.1.21
- network: private
address: 172.16.2.21
- network: oob
address: 172.16.100.21
metadata:
rack: rack02
rack: rack2
---
apiVersion: 'drydock/v1'
kind: HardwareProfile
@ -449,17 +474,17 @@ spec:
# Map hardware addresses to aliases/roles to allow a mix of hardware configs
# in a site to result in a consistent configuration
device_aliases:
- address: '0000:00:03.0'
alias: prim_nic01
- address: '0000:00:03.0'
alias: prim_nic01
# type could identify expected hardware - used for hardware manifest validation
dev_type: '82540EM Gigabit Ethernet Controller'
bus_type: 'pci'
- address: '0000:00:04.0'
alias: prim_nic02
dev_type: '82540EM Gigabit Ethernet Controller'
bus_type: 'pci'
- address: '2:0.0.0'
alias: primary_boot
dev_type: 'VBOX HARDDISK'
bus_type: 'scsi'
dev_type: '82540EM Gigabit Ethernet Controller'
bus_type: 'pci'
- address: '0000:00:04.0'
alias: prim_nic02
dev_type: '82540EM Gigabit Ethernet Controller'
bus_type: 'pci'
- address: '2:0.0.0'
alias: primary_boot
dev_type: 'VBOX HARDDISK'
bus_type: 'scsi'
...