airship/spyglass
Code Issues Proposed changes

Add gate

Browse Source 
This patch set puts in a rudimentary gate.

Change-Id: I3a2466bd7be5352b46273b385d215913eb8079ba
Signed-off-by: Tin Lam <tin@irrational.io>
This commit is contained in:
Tin Lam 2019-04-04 19:25:11 -05:00
parent dcf0735b08
commit 89dfec7b4c
12 changed files with 779 additions and 609 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.zuul.yaml
View File

@ -9,7 +9,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- project:
templates:
- noop-jobs
check:
jobs:
- openstack-tox-pep8
gate:
jobs:
- openstack-tox-pep8

91
spyglass/data_extractor/base.py
View File

@ -281,8 +281,8 @@ class BaseDataSourcePlugin(object):
# For each host list fill host profile and network IPs
for host in hosts:
host_name = host['name']
rack_name = host['rack_name']
host_name = host["name"]
rack_name = host["rack_name"]
if rack_name not in baremetal:
baremetal[rack_name] = {}
@ -290,32 +290,39 @@ class BaseDataSourcePlugin(object):
# Prepare temp dict for each host and append it to baremetal
# at a rack level
temp_host = {}
if host['host_profile'] is None:
temp_host['host_profile'] = "#CHANGE_ME"
if host["host_profile"] is None:
temp_host["host_profile"] = "#CHANGE_ME"
else:
temp_host['host_profile'] = host['host_profile']
temp_host["host_profile"] = host["host_profile"]
# Get Host IPs from plugin
temp_host_ips = self.get_ips(self.region, host_name)
# Fill network IP for this host
temp_host['ip'] = {}
temp_host['ip']['oob'] = temp_host_ips[host_name].get(
'oob', "#CHANGE_ME")
temp_host['ip']['calico'] = temp_host_ips[host_name].get(
'calico', "#CHANGE_ME")
temp_host['ip']['oam'] = temp_host_ips[host_name].get(
'oam', "#CHANGE_ME")
temp_host['ip']['storage'] = temp_host_ips[host_name].get(
'storage', "#CHANGE_ME")
temp_host['ip']['overlay'] = temp_host_ips[host_name].get(
'overlay', "#CHANGE_ME")
temp_host['ip']['pxe'] = temp_host_ips[host_name].get(
'pxe', "#CHANGE_ME")
temp_host["ip"] = {}
temp_host["ip"]["oob"] = temp_host_ips[host_name].get(
"oob", "#CHANGE_ME"
)
temp_host["ip"]["calico"] = temp_host_ips[host_name].get(
"calico", "#CHANGE_ME"
)
temp_host["ip"]["oam"] = temp_host_ips[host_name].get(
"oam", "#CHANGE_ME"
)
temp_host["ip"]["storage"] = temp_host_ips[host_name].get(
"storage", "#CHANGE_ME"
)
temp_host["ip"]["overlay"] = temp_host_ips[host_name].get(
"overlay", "#CHANGE_ME"
)
temp_host["ip"]["pxe"] = temp_host_ips[host_name].get(
"pxe", "#CHANGE_ME"
)
baremetal[rack_name][host_name] = temp_host
LOG.debug("Baremetal information:\n{}".format(
pprint.pformat(baremetal)))
LOG.debug(
"Baremetal information:\n{}".format(pprint.pformat(baremetal))
)
return baremetal
@ -348,19 +355,20 @@ class BaseDataSourcePlugin(object):
site_info = location_data
dns_data = self.get_dns_servers(self.region)
site_info['dns'] = dns_data
site_info["dns"] = dns_data
ntp_data = self.get_ntp_servers(self.region)
site_info['ntp'] = ntp_data
site_info["ntp"] = ntp_data
ldap_data = self.get_ldap_information(self.region)
site_info['ldap'] = ldap_data
site_info["ldap"] = ldap_data
domain_data = self.get_domain_name(self.region)
site_info['domain'] = domain_data
site_info["domain"] = domain_data
LOG.debug("Extracted site information:\n{}".format(
pprint.pformat(site_info)))
LOG.debug(
"Extracted site information:\n{}".format(pprint.pformat(site_info))
)
return site_info
@ -393,21 +401,28 @@ class BaseDataSourcePlugin(object):
# networks_to_scan, so look for these networks from the data
# returned by plugin
networks_to_scan = [
'calico', 'overlay', 'pxe', 'storage', 'oam', 'oob', 'ingress'
"calico",
"overlay",
"pxe",
"storage",
"oam",
"oob",
"ingress",
]
network_data['vlan_network_data'] = {}
network_data["vlan_network_data"] = {}
for net in networks:
tmp_net = {}
if net['name'] in networks_to_scan:
tmp_net['subnet'] = net.get('subnet', '#CHANGE_ME')
if ((net['name'] != 'ingress') and (net['name'] != 'oob')):
tmp_net['vlan'] = net.get('vlan', '#CHANGE_ME')
if net["name"] in networks_to_scan:
tmp_net["subnet"] = net.get("subnet", "#CHANGE_ME")
if (net["name"] != "ingress") and (net["name"] != "oob"):
tmp_net["vlan"] = net.get("vlan", "#CHANGE_ME")
network_data['vlan_network_data'][net['name']] = tmp_net
network_data["vlan_network_data"][net["name"]] = tmp_net
LOG.debug("Extracted network data:\n{}".format(
pprint.pformat(network_data)))
LOG.debug(
"Extracted network data:\n{}".format(pprint.pformat(network_data))
)
return network_data
def extract_data(self):
@ -418,9 +433,9 @@ class BaseDataSourcePlugin(object):
"""
LOG.info("Extract data from plugin")
site_data = {}
site_data['baremetal'] = self.extract_baremetal_information()
site_data['site_info'] = self.extract_site_information()
site_data['network'] = self.extract_network_information()
site_data["baremetal"] = self.extract_baremetal_information()
site_data["site_info"] = self.extract_site_information()
site_data["network"] = self.extract_network_information()
self.site_data = site_data
return site_data

7
spyglass/data_extractor/custom_exceptions.py
View File

@ -31,8 +31,11 @@ class NoSpecMatched(BaseError):
self.specs = excel_specs
def display_error(self):
print('No spec matched. Following are the available specs:\n'.format(
self.specs))
print(
"No spec matched. Following are the available specs:\n".format(
self.specs
)
)
sys.exit(1)

184
spyglass/data_extractor/plugins/formation.py
View File

@ -22,8 +22,11 @@ import urllib3
from spyglass.data_extractor.base import BaseDataSourcePlugin
from spyglass.data_extractor.custom_exceptions import (
ApiClientError, ConnectionError, MissingAttributeError,
TokenGenerationError)
ApiClientError,
ConnectionError,
MissingAttributeError,
TokenGenerationError,
)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@ -41,8 +44,8 @@ class FormationPlugin(BaseDataSourcePlugin):
LOG.info("Check spyglass --help for details")
exit()
self.source_type = 'rest'
self.source_name = 'formation'
self.source_type = "rest"
self.source_name = "formation"
# Configuration parameters
self.formation_api_url = None
@ -67,10 +70,10 @@ class FormationPlugin(BaseDataSourcePlugin):
""" Sets the config params passed by CLI"""
LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf)))
self._validate_config_options(conf)
self.formation_api_url = conf['url']
self.user = conf['user']
self.password = conf['password']
self.token = conf.get('token', None)
self.formation_api_url = conf["url"]
self.user = conf["user"]
self.password = conf["password"]
self.token = conf.get("token", None)
self._get_formation_client()
self._update_site_and_zone(self.region)
@ -78,21 +81,24 @@ class FormationPlugin(BaseDataSourcePlugin):
def get_plugin_conf(self, kwargs):
""" Validates the plugin param and return if success"""
try:
assert (kwargs['formation_url']
) is not None, "formation_url is Not Specified"
url = kwargs['formation_url']
assert (kwargs['formation_user']
) is not None, "formation_user is Not Specified"
user = kwargs['formation_user']
assert (kwargs['formation_password']
) is not None, "formation_password is Not Specified"
password = kwargs['formation_password']
assert (
kwargs["formation_url"]
) is not None, "formation_url is Not Specified"
url = kwargs["formation_url"]
assert (
kwargs["formation_user"]
) is not None, "formation_user is Not Specified"
user = kwargs["formation_user"]
assert (
kwargs["formation_password"]
) is not None, "formation_password is Not Specified"
password = kwargs["formation_password"]
except AssertionError:
LOG.error("Insufficient plugin parameter! Spyglass exited!")
raise
exit()
plugin_conf = {'url': url, 'user': user, 'password': password}
plugin_conf = {"url": url, "user": user, "password": password}
return plugin_conf
def _validate_config_options(self, conf):
@ -129,21 +135,24 @@ class FormationPlugin(BaseDataSourcePlugin):
if self.token:
return self.token
url = self.formation_api_url + '/zones'
url = self.formation_api_url + "/zones"
try:
token_response = requests.get(
url,
auth=(self.user, self.password),
verify=self.client_config.verify_ssl)
verify=self.client_config.verify_ssl,
)
except requests.exceptions.ConnectionError:
raise ConnectionError('Incorrect URL: {}'.format(url))
raise ConnectionError("Incorrect URL: {}".format(url))
if token_response.status_code == 200:
self.token = token_response.json().get('X-Subject-Token', None)
self.token = token_response.json().get("X-Subject-Token", None)
else:
raise TokenGenerationError(
'Unable to generate token because {}'.format(
token_response.reason))
"Unable to generate token because {}".format(
token_response.reason
)
)
return self.token
@ -155,9 +164,10 @@ class FormationPlugin(BaseDataSourcePlugin):
Generate the token and add it formation config object.
"""
token = self._generate_token()
self.client_config.api_key = {'X-Auth-Token': self.user + '|' + token}
self.client_config.api_key = {"X-Auth-Token": self.user + "|" + token}
self.formation_api_client = formation_client.ApiClient(
self.client_config)
self.client_config
)
def _update_site_and_zone(self, region):
"""Get Zone name and Site name from region"""
@ -169,8 +179,8 @@ class FormationPlugin(BaseDataSourcePlugin):
# site = zone[:-1]
self.region_zone_map[region] = {}
self.region_zone_map[region]['zone'] = zone
self.region_zone_map[region]['site'] = site
self.region_zone_map[region]["zone"] = zone
self.region_zone_map[region]["site"] = site
def _get_zone_by_region_name(self, region_name):
zone_api = formation_client.ZonesApi(self.formation_api_client)
@ -248,7 +258,7 @@ class FormationPlugin(BaseDataSourcePlugin):
return self.device_name_id_mapping.get(device_name, None)
def _get_racks(self, zone, rack_type='compute'):
def _get_racks(self, zone, rack_type="compute"):
zone_id = self._get_zone_id_by_name(zone)
rack_api = formation_client.RacksApi(self.formation_api_client)
racks = rack_api.zones_zone_id_racks_get(zone_id)
@ -296,35 +306,40 @@ class FormationPlugin(BaseDataSourcePlugin):
# Implement Abstract functions
def get_racks(self, region):
zone = self.region_zone_map[region]['zone']
return self._get_racks(zone, rack_type='compute')
zone = self.region_zone_map[region]["zone"]
return self._get_racks(zone, rack_type="compute")
def get_hosts(self, region, rack=None):
zone = self.region_zone_map[region]['zone']
zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
device_api = formation_client.DevicesApi(self.formation_api_client)
control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id)
compute_hosts = device_api.zones_zone_id_devices_get(
zone_id, type='KVM')
zone_id, type="KVM"
)
hosts_list = []
for host in control_hosts:
self.device_name_id_mapping[host.aic_standard_name] = host.id
hosts_list.append({
'name': host.aic_standard_name,
'type': 'controller',
'rack_name': host.rack_name,
'host_profile': host.host_profile_name
})
hosts_list.append(
{
"name": host.aic_standard_name,
"type": "controller",
"rack_name": host.rack_name,
"host_profile": host.host_profile_name,
}
)
for host in compute_hosts:
self.device_name_id_mapping[host.aic_standard_name] = host.id
hosts_list.append({
'name': host.aic_standard_name,
'type': 'compute',
'rack_name': host.rack_name,
'host_profile': host.host_profile_name
})
hosts_list.append(
{
"name": host.aic_standard_name,
"type": "compute",
"rack_name": host.rack_name,
"host_profile": host.host_profile_name,
}
)
"""
for host in itertools.chain(control_hosts, compute_hosts):
self.device_name_id_mapping[host.aic_standard_name] = host.id
@ -339,40 +354,43 @@ class FormationPlugin(BaseDataSourcePlugin):
return hosts_list
def get_networks(self, region):
zone = self.region_zone_map[region]['zone']
zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
region_id = self._get_region_id_by_name(region)
vlan_api = formation_client.VlansApi(self.formation_api_client)
vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get(
zone_id, region_id)
zone_id, region_id
)
# Case when vlans list is empty from
# zones_zone_id_regions_region_id_vlans_get
if len(vlans) is 0:
if len(vlans) == 0:
# get device-id from the first host and get the network details
hosts = self.get_hosts(self.region)
host = hosts[0]['name']
host = hosts[0]["name"]
device_id = self._get_device_id_by_name(host)
vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
zone_id, device_id)
zone_id, device_id
)
LOG.debug("Extracted region network information\n{}".format(vlans))
vlans_list = []
for vlan_ in vlans:
if len(vlan_.vlan.ipv4) is not 0:
if len(vlan_.vlan.ipv4) != 0:
tmp_vlan = {}
tmp_vlan['name'] = self._get_network_name_from_vlan_name(
vlan_.vlan.name)
tmp_vlan['vlan'] = vlan_.vlan.vlan_id
tmp_vlan['subnet'] = vlan_.vlan.subnet_range
tmp_vlan['gateway'] = vlan_.ipv4_gateway
tmp_vlan['subnet_level'] = vlan_.vlan.subnet_level
tmp_vlan["name"] = self._get_network_name_from_vlan_name(
vlan_.vlan.name
)
tmp_vlan["vlan"] = vlan_.vlan.vlan_id
tmp_vlan["subnet"] = vlan_.vlan.subnet_range
tmp_vlan["gateway"] = vlan_.ipv4_gateway
tmp_vlan["subnet_level"] = vlan_.vlan.subnet_level
vlans_list.append(tmp_vlan)
return vlans_list
def get_ips(self, region, host=None):
zone = self.region_zone_map[region]['zone']
zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
if host:
@ -381,7 +399,7 @@ class FormationPlugin(BaseDataSourcePlugin):
hosts = []
hosts_dict = self.get_hosts(zone)
for host in hosts_dict:
hosts.append(host['name'])
hosts.append(host["name"])
vlan_api = formation_client.VlansApi(self.formation_api_client)
ip_ = {}
@ -389,18 +407,23 @@ class FormationPlugin(BaseDataSourcePlugin):
for host in hosts:
device_id = self._get_device_id_by_name(host)
vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
zone_id, device_id)
zone_id, device_id
)
LOG.debug("Received VLAN Network Information\n{}".format(vlans))
ip_[host] = {}
for vlan_ in vlans:
# TODO(pg710r) We need to handle the case when incoming ipv4
# list is empty
if len(vlan_.vlan.ipv4) is not 0:
if len(vlan_.vlan.ipv4) != 0:
name = self._get_network_name_from_vlan_name(
vlan_.vlan.name)
vlan_.vlan.name
)
ipv4 = vlan_.vlan.ipv4[0].ip
LOG.debug("vlan:{},name:{},ip:{},vlan_name:{}".format(
vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name))
LOG.debug(
"vlan:{},name:{},ip:{},vlan_name:{}".format(
vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name
)
)
# TODD(pg710r) This code needs to extended to support ipv4
# and ipv6
# ip_[host][name] = {'ipv4': ipv4}
@ -419,12 +442,12 @@ class FormationPlugin(BaseDataSourcePlugin):
vlan_name contains "ILO" the network name is "oob"
"""
network_names = {
'ksn': 'calico',
'storage': 'storage',
'server': 'oam',
'ovs': 'overlay',
'ILO': 'oob',
'pxe': 'pxe'
"ksn": "calico",
"storage": "storage",
"server": "oam",
"ovs": "overlay",
"ILO": "oob",
"pxe": "pxe",
}
for name in network_names:
@ -438,7 +461,7 @@ class FormationPlugin(BaseDataSourcePlugin):
def get_dns_servers(self, region):
try:
zone = self.region_zone_map[region]['zone']
zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
zone_api = formation_client.ZonesApi(self.formation_api_client)
zone_ = zone_api.zones_zone_id_get(zone_id)
@ -463,7 +486,7 @@ class FormationPlugin(BaseDataSourcePlugin):
def get_location_information(self, region):
""" get location information for a zone and return """
site = self.region_zone_map[region]['site']
site = self.region_zone_map[region]["site"]
site_id = self._get_site_id_by_name(site)
site_api = formation_client.SitesApi(self.formation_api_client)
site_info = site_api.sites_site_id_get(site_id)
@ -471,18 +494,19 @@ class FormationPlugin(BaseDataSourcePlugin):
try:
return {
# 'corridor': site_info.corridor,
'name': site_info.city,
'state': site_info.state,
'country': site_info.country,
'physical_location_id': site_info.clli,
"name": site_info.city,
"state": site_info.state,
"country": site_info.country,
"physical_location_id": site_info.clli,
}
except AttributeError as e:
raise MissingAttributeError('Missing {} information in {}'.format(
e, site_info.city))
raise MissingAttributeError(
"Missing {} information in {}".format(e, site_info.city)
)
def get_domain_name(self, region):
try:
zone = self.region_zone_map[region]['zone']
zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
zone_api = formation_client.ZonesApi(self.formation_api_client)
zone_ = zone_api.zones_zone_id_get(zone_id)
@ -490,7 +514,7 @@ class FormationPlugin(BaseDataSourcePlugin):
raise ApiClientError(e.msg)
if not zone_.dns:
LOG.warn('Got None while running get domain name')
LOG.warn("Got None while running get domain name")
return None
return zone_.dns

9
spyglass/data_extractor/plugins/tugboat/check_exceptions.py
View File

@ -23,7 +23,7 @@ class NotEnoughIp(BaseError):
self.total_nodes = total_nodes
def display_error(self):
print('{} can not handle {} nodes'.format(self.cidr, self.total_nodes))
print("{} can not handle {} nodes".format(self.cidr, self.total_nodes))
class NoSpecMatched(BaseError):
@ -31,5 +31,8 @@ class NoSpecMatched(BaseError):
self.specs = excel_specs
def display_error(self):
print('No spec matched. Following are the available specs:\n'.format(
self.specs))
print(
"No spec matched. Following are the available specs:\n".format(
self.specs
)
)

338
spyglass/data_extractor/plugins/tugboat/excel_parser.py
View File

@ -20,17 +20,18 @@ import yaml
from openpyxl import load_workbook
from openpyxl import Workbook
from spyglass.data_extractor.custom_exceptions import NoSpecMatched
# from spyglass.data_extractor.custom_exceptions
LOG = logging.getLogger(__name__)
class ExcelParser():
class ExcelParser:
""" Parse data from excel into a dict """
def __init__(self, file_name, excel_specs):
self.file_name = file_name
with open(excel_specs, 'r') as f:
with open(excel_specs, "r") as f:
spec_raw_data = f.read()
self.excel_specs = yaml.safe_load(spec_raw_data)
# A combined design spec, returns a workbok object after combining
@ -38,12 +39,12 @@ class ExcelParser():
combined_design_spec = self.combine_excel_design_specs(file_name)
self.wb_combined = combined_design_spec
self.filenames = file_name
self.spec = 'xl_spec'
self.spec = "xl_spec"
@staticmethod
def sanitize(string):
""" Remove extra spaces and convert string to lower case """
return string.replace(' ', '').lower()
return string.replace(" ", "").lower()
def compare(self, string1, string2):
""" Compare the strings """
@ -52,19 +53,19 @@ class ExcelParser():
def validate_sheet(self, spec, sheet):
""" Check if the sheet is correct or not """
ws = self.wb_combined[sheet]
header_row = self.excel_specs['specs'][spec]['header_row']
ipmi_header = self.excel_specs['specs'][spec]['ipmi_address_header']
ipmi_column = self.excel_specs['specs'][spec]['ipmi_address_col']
header_row = self.excel_specs["specs"][spec]["header_row"]
ipmi_header = self.excel_specs["specs"][spec]["ipmi_address_header"]
ipmi_column = self.excel_specs["specs"][spec]["ipmi_address_col"]
header_value = ws.cell(row=header_row, column=ipmi_column).value
return bool(self.compare(ipmi_header, header_value))
def find_correct_spec(self):
""" Find the correct spec """
for spec in self.excel_specs['specs']:
sheet_name = self.excel_specs['specs'][spec]['ipmi_sheet_name']
for spec in self.excel_specs["specs"]:
sheet_name = self.excel_specs["specs"][spec]["ipmi_sheet_name"]
for sheet in self.wb_combined.sheetnames:
if self.compare(sheet_name, sheet):
self.excel_specs['specs'][spec]['ipmi_sheet_name'] = sheet
self.excel_specs["specs"][spec]["ipmi_sheet_name"] = sheet
if self.validate_sheet(spec, sheet):
return spec
raise NoSpecMatched(self.excel_specs)
@ -73,31 +74,37 @@ class ExcelParser():
""" Read IPMI data from the sheet """
ipmi_data = {}
hosts = []
provided_sheetname = self.excel_specs['specs'][self.
spec]['ipmi_sheet_name']
provided_sheetname = self.excel_specs["specs"][self.spec][
"ipmi_sheet_name"
]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
provided_sheetname
)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
row = self.excel_specs['specs'][self.spec]['start_row']
end_row = self.excel_specs['specs'][self.spec]['end_row']
hostname_col = self.excel_specs['specs'][self.spec]['hostname_col']
ipmi_address_col = self.excel_specs['specs'][self.
spec]['ipmi_address_col']
host_profile_col = self.excel_specs['specs'][self.
spec]['host_profile_col']
ipmi_gateway_col = self.excel_specs['specs'][self.
spec]['ipmi_gateway_col']
row = self.excel_specs["specs"][self.spec]["start_row"]
end_row = self.excel_specs["specs"][self.spec]["end_row"]
hostname_col = self.excel_specs["specs"][self.spec]["hostname_col"]
ipmi_address_col = self.excel_specs["specs"][self.spec][
"ipmi_address_col"
]
host_profile_col = self.excel_specs["specs"][self.spec][
"host_profile_col"
]
ipmi_gateway_col = self.excel_specs["specs"][self.spec][
"ipmi_gateway_col"
]
previous_server_gateway = None
while row <= end_row:
hostname = self.sanitize(
ws.cell(row=row, column=hostname_col).value)
ws.cell(row=row, column=hostname_col).value
)
hosts.append(hostname)
ipmi_address = ws.cell(row=row, column=ipmi_address_col).value
if '/' in ipmi_address:
ipmi_address = ipmi_address.split('/')[0]
if "/" in ipmi_address:
ipmi_address = ipmi_address.split("/")[0]
ipmi_gateway = ws.cell(row=row, column=ipmi_gateway_col).value
if ipmi_gateway:
previous_server_gateway = ipmi_gateway
@ -106,32 +113,39 @@ class ExcelParser():
host_profile = ws.cell(row=row, column=host_profile_col).value
try:
if host_profile is None:
raise RuntimeError("No value read from {} ".format(
self.file_name) + "sheet:{} row:{}, col:{}".format(
self.spec, row, host_profile_col))
raise RuntimeError(
"No value read from {} ".format(self.file_name)
+ "sheet:{} row:{}, col:{}".format(
self.spec, row, host_profile_col
)
)
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
ipmi_data[hostname] = {
'ipmi_address': ipmi_address,
'ipmi_gateway': ipmi_gateway,
'host_profile': host_profile,
'type': type,
"ipmi_address": ipmi_address,
"ipmi_gateway": ipmi_gateway,
"host_profile": host_profile,
"type": type,
}
row += 1
LOG.debug("ipmi data extracted from excel:\n{}".format(
pprint.pformat(ipmi_data)))
LOG.debug("host data extracted from excel:\n{}".format(
pprint.pformat(hosts)))
LOG.debug(
"ipmi data extracted from excel:\n{}".format(
pprint.pformat(ipmi_data)
)
)
LOG.debug(
"host data extracted from excel:\n{}".format(pprint.pformat(hosts))
)
return [ipmi_data, hosts]
def get_private_vlan_data(self, ws):
""" Get private vlan data from private IP sheet """
vlan_data = {}
row = self.excel_specs['specs'][self.spec]['vlan_start_row']
end_row = self.excel_specs['specs'][self.spec]['vlan_end_row']
type_col = self.excel_specs['specs'][self.spec]['net_type_col']
vlan_col = self.excel_specs['specs'][self.spec]['vlan_col']
row = self.excel_specs["specs"][self.spec]["vlan_start_row"]
end_row = self.excel_specs["specs"][self.spec]["vlan_end_row"]
type_col = self.excel_specs["specs"][self.spec]["net_type_col"]
vlan_col = self.excel_specs["specs"][self.spec]["vlan_col"]
while row <= end_row:
cell_value = ws.cell(row=row, column=type_col).value
if cell_value:
@ -140,27 +154,30 @@ class ExcelParser():
vlan = vlan.lower()
vlan_data[vlan] = cell_value
row += 1
LOG.debug("vlan data extracted from excel:\n%s",
pprint.pformat(vlan_data))
LOG.debug(
"vlan data extracted from excel:\n%s", pprint.pformat(vlan_data)
)
return vlan_data
def get_private_network_data(self):
""" Read network data from the private ip sheet """
provided_sheetname = self.excel_specs['specs'][
self.spec]['private_ip_sheet']
provided_sheetname = self.excel_specs["specs"][self.spec][
"private_ip_sheet"
]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
provided_sheetname
)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
vlan_data = self.get_private_vlan_data(ws)
network_data = {}
row = self.excel_specs['specs'][self.spec]['net_start_row']
end_row = self.excel_specs['specs'][self.spec]['net_end_row']
col = self.excel_specs['specs'][self.spec]['net_col']
vlan_col = self.excel_specs['specs'][self.spec]['net_vlan_col']
old_vlan = ''
row = self.excel_specs["specs"][self.spec]["net_start_row"]
end_row = self.excel_specs["specs"][self.spec]["net_end_row"]
col = self.excel_specs["specs"][self.spec]["net_col"]
vlan_col = self.excel_specs["specs"][self.spec]["net_vlan_col"]
old_vlan = ""
while row <= end_row:
vlan = ws.cell(row=row, column=vlan_col).value
if vlan:
@ -168,11 +185,8 @@ class ExcelParser():
network = ws.cell(row=row, column=col).value
if vlan and network:
net_type = vlan_data[vlan]
if 'vlan' not in network_data:
network_data[net_type] = {
'vlan': vlan,
'subnet': [],
}
if "vlan" not in network_data:
network_data[net_type] = {"vlan": vlan, "subnet": []}
elif not vlan and network:
# If vlan is not present then assign old vlan to vlan as vlan
# value is spread over several rows
@ -180,11 +194,11 @@ class ExcelParser():
else:
row += 1
continue
network_data[vlan_data[vlan]]['subnet'].append(network)
network_data[vlan_data[vlan]]["subnet"].append(network)
old_vlan = vlan
row += 1
for network in network_data:
network_data[network]['is_common'] = True
network_data[network]["is_common"] = True
"""
if len(network_data[network]['subnet']) > 1:
network_data[network]['is_common'] = False
@ -199,153 +213,167 @@ class ExcelParser():
def get_public_network_data(self):
""" Read public network data from public ip data """
network_data = {}
provided_sheetname = self.excel_specs['specs'][self.
spec]['public_ip_sheet']
provided_sheetname = self.excel_specs["specs"][self.spec][
"public_ip_sheet"
]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
provided_sheetname
)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
oam_row = self.excel_specs['specs'][self.spec]['oam_ip_row']
oam_col = self.excel_specs['specs'][self.spec]['oam_ip_col']
oam_vlan_col = self.excel_specs['specs'][self.spec]['oam_vlan_col']
ingress_row = self.excel_specs['specs'][self.spec]['ingress_ip_row']
oob_row = self.excel_specs['specs'][self.spec]['oob_net_row']
col = self.excel_specs['specs'][self.spec]['oob_net_start_col']
end_col = self.excel_specs['specs'][self.spec]['oob_net_end_col']
oam_row = self.excel_specs["specs"][self.spec]["oam_ip_row"]
oam_col = self.excel_specs["specs"][self.spec]["oam_ip_col"]
oam_vlan_col = self.excel_specs["specs"][self.spec]["oam_vlan_col"]
ingress_row = self.excel_specs["specs"][self.spec]["ingress_ip_row"]
oob_row = self.excel_specs["specs"][self.spec]["oob_net_row"]
col = self.excel_specs["specs"][self.spec]["oob_net_start_col"]
end_col = self.excel_specs["specs"][self.spec]["oob_net_end_col"]
network_data = {
'oam': {
'subnet': [ws.cell(row=oam_row, column=oam_col).value],
'vlan': ws.cell(row=oam_row, column=oam_vlan_col).value,
"oam": {
"subnet": [ws.cell(row=oam_row, column=oam_col).value],
"vlan": ws.cell(row=oam_row, column=oam_vlan_col).value,
},
'ingress': ws.cell(row=ingress_row, column=oam_col).value,
}
network_data['oob'] = {
'subnet': [],
"ingress": ws.cell(row=ingress_row, column=oam_col).value,
}
network_data["oob"] = {"subnet": []}
while col <= end_col:
cell_value = ws.cell(row=oob_row, column=col).value
if cell_value:
network_data['oob']['subnet'].append(self.sanitize(cell_value))
network_data["oob"]["subnet"].append(self.sanitize(cell_value))
col += 1
LOG.debug(
"public network data extracted from\
excel:\n%s", pprint.pformat(network_data))
excel:\n%s",
pprint.pformat(network_data),
)
return network_data
def get_site_info(self):
""" Read location, dns, ntp and ldap data"""
site_info = {}
provided_sheetname = self.excel_specs['specs'][
self.spec]['dns_ntp_ldap_sheet']
provided_sheetname = self.excel_specs["specs"][self.spec][
"dns_ntp_ldap_sheet"
]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
provided_sheetname
)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
dns_row = self.excel_specs['specs'][self.spec]['dns_row']
dns_col = self.excel_specs['specs'][self.spec]['dns_col']
ntp_row = self.excel_specs['specs'][self.spec]['ntp_row']
ntp_col = self.excel_specs['specs'][self.spec]['ntp_col']
domain_row = self.excel_specs['specs'][self.spec]['domain_row']
domain_col = self.excel_specs['specs'][self.spec]['domain_col']
login_domain_row = self.excel_specs['specs'][self.
spec]['login_domain_row']
ldap_col = self.excel_specs['specs'][self.spec]['ldap_col']
global_group = self.excel_specs['specs'][self.spec]['global_group']
ldap_search_url_row = self.excel_specs['specs'][
self.spec]['ldap_search_url_row']
dns_row = self.excel_specs["specs"][self.spec]["dns_row"]
dns_col = self.excel_specs["specs"][self.spec]["dns_col"]
ntp_row = self.excel_specs["specs"][self.spec]["ntp_row"]
ntp_col = self.excel_specs["specs"][self.spec]["ntp_col"]
domain_row = self.excel_specs["specs"][self.spec]["domain_row"]
domain_col = self.excel_specs["specs"][self.spec]["domain_col"]
login_domain_row = self.excel_specs["specs"][self.spec][
"login_domain_row"
]
ldap_col = self.excel_specs["specs"][self.spec]["ldap_col"]
global_group = self.excel_specs["specs"][self.spec]["global_group"]
ldap_search_url_row = self.excel_specs["specs"][self.spec][
"ldap_search_url_row"
]
dns_servers = ws.cell(row=dns_row, column=dns_col).value
ntp_servers = ws.cell(row=ntp_row, column=ntp_col).value
try:
if dns_servers is None:
raise RuntimeError(
"No value for dns_server from:{} Sheet:'{}' Row:{} Col:{}".
format(self.file_name, provided_sheetname, dns_row,
dns_col))
raise RuntimeError(
"No value for ntp_server frome:{} Sheet:'{}' Row:{} Col:{}"
.format(self.file_name, provided_sheetname, ntp_row,
ntp_col))
(
"No value for dns_server from:{} Sheet:'{}' ",
"Row:{} Col:{}",
).format(
self.file_name, provided_sheetname, dns_row, dns_col
)
)
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
dns_servers = dns_servers.replace('\n', ' ')
ntp_servers = ntp_servers.replace('\n', ' ')
if ',' in dns_servers:
dns_servers = dns_servers.split(',')
dns_servers = dns_servers.replace("\n", " ")
ntp_servers = ntp_servers.replace("\n", " ")
if "," in dns_servers:
dns_servers = dns_servers.split(",")
else:
dns_servers = dns_servers.split()
if ',' in ntp_servers:
ntp_servers = ntp_servers.split(',')
if "," in ntp_servers:
ntp_servers = ntp_servers.split(",")
else:
ntp_servers = ntp_servers.split()
site_info = {
'location': self.get_location_data(),
'dns': dns_servers,
'ntp': ntp_servers,
'domain': ws.cell(row=domain_row, column=domain_col).value,
'ldap': {
'subdomain': ws.cell(row=login_domain_row,
column=ldap_col).value,
'common_name': ws.cell(row=global_group,
column=ldap_col).value,
'url': ws.cell(row=ldap_search_url_row, column=ldap_col).value,
}
"location": self.get_location_data(),
"dns": dns_servers,
"ntp": ntp_servers,
"domain": ws.cell(row=domain_row, column=domain_col).value,
"ldap": {
"subdomain": ws.cell(
row=login_domain_row, column=ldap_col
).value,
"common_name": ws.cell(
row=global_group, column=ldap_col
).value,
"url": ws.cell(row=ldap_search_url_row, column=ldap_col).value,
},
}
LOG.debug(
"Site Info extracted from\
excel:\n%s", pprint.pformat(site_info))
excel:\n%s",
pprint.pformat(site_info),
)
return site_info
def get_location_data(self):
""" Read location data from the site and zone sheet """
provided_sheetname = self.excel_specs['specs'][self.
spec]['location_sheet']
provided_sheetname = self.excel_specs["specs"][self.spec][
"location_sheet"
]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
provided_sheetname)
provided_sheetname
)
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
corridor_row = self.excel_specs['specs'][self.spec]['corridor_row']
column = self.excel_specs['specs'][self.spec]['column']
site_name_row = self.excel_specs['specs'][self.spec]['site_name_row']
state_name_row = self.excel_specs['specs'][self.spec]['state_name_row']
country_name_row = self.excel_specs['specs'][self.
spec]['country_name_row']
clli_name_row = self.excel_specs['specs'][self.spec]['clli_name_row']
corridor_row = self.excel_specs["specs"][self.spec]["corridor_row"]
column = self.excel_specs["specs"][self.spec]["column"]
site_name_row = self.excel_specs["specs"][self.spec]["site_name_row"]
state_name_row = self.excel_specs["specs"][self.spec]["state_name_row"]
country_name_row = self.excel_specs["specs"][self.spec][
"country_name_row"
]
clli_name_row = self.excel_specs["specs"][self.spec]["clli_name_row"]
return {
'corridor': ws.cell(row=corridor_row, column=column).value,
'name': ws.cell(row=site_name_row, column=column).value,
'state': ws.cell(row=state_name_row, column=column).value,
'country': ws.cell(row=country_name_row, column=column).value,
'physical_location': ws.cell(row=clli_name_row,
column=column).value,
"corridor": ws.cell(row=corridor_row, column=column).value,
"name": ws.cell(row=site_name_row, column=column).value,
"state": ws.cell(row=state_name_row, column=column).value,
"country": ws.cell(row=country_name_row, column=column).value,
"physical_location": ws.cell(
row=clli_name_row, column=column
).value,
}
def validate_sheet_names_with_spec(self):
""" Checks is sheet name in spec file matches with excel file"""
spec = list(self.excel_specs['specs'].keys())[0]
spec_item = self.excel_specs['specs'][spec]
spec = list(self.excel_specs["specs"].keys())[0]
spec_item = self.excel_specs["specs"][spec]
sheet_name_list = []
ipmi_header_sheet_name = spec_item['ipmi_sheet_name']
ipmi_header_sheet_name = spec_item["ipmi_sheet_name"]
sheet_name_list.append(ipmi_header_sheet_name)
private_ip_sheet_name = spec_item['private_ip_sheet']
private_ip_sheet_name = spec_item["private_ip_sheet"]
sheet_name_list.append(private_ip_sheet_name)
public_ip_sheet_name = spec_item['public_ip_sheet']
public_ip_sheet_name = spec_item["public_ip_sheet"]
sheet_name_list.append(public_ip_sheet_name)
dns_ntp_ldap_sheet_name = spec_item['dns_ntp_ldap_sheet']
dns_ntp_ldap_sheet_name = spec_item["dns_ntp_ldap_sheet"]
sheet_name_list.append(dns_ntp_ldap_sheet_name)
location_sheet_name = spec_item['location_sheet']
location_sheet_name = spec_item["location_sheet"]
sheet_name_list.append(location_sheet_name)
try:
for sheetname in sheet_name_list:
workbook_object, extracted_sheetname = \
self.get_xl_obj_and_sheetname(sheetname)
workbook_object, extracted_sheetname = (
self.get_xl_obj_and_sheetname(sheetname))
if workbook_object is not None:
wb = workbook_object
sheetname = extracted_sheetname
@ -354,7 +382,8 @@ class ExcelParser():
if sheetname not in wb.sheetnames:
raise RuntimeError(
"SheetName '{}' not found ".format(sheetname))
"SheetName '{}' not found ".format(sheetname)
)
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
@ -369,16 +398,18 @@ class ExcelParser():
public_network_data = self.get_public_network_data()
site_info_data = self.get_site_info()
data = {
'ipmi_data': ipmi_data,
'network_data': {
'private': network_data,
'public': public_network_data,
"ipmi_data": ipmi_data,
"network_data": {
"private": network_data,
"public": public_network_data,
},
'site_info': site_info_data,
"site_info": site_info_data,
}
LOG.debug(
"Location data extracted from\
excel:\n%s", pprint.pformat(data))
excel:\n%s",
pprint.pformat(data),
)
return data
def combine_excel_design_specs(self, filenames):
@ -391,8 +422,9 @@ class ExcelParser():
loaded_workbook_ws = loaded_workbook[names]
for row in loaded_workbook_ws:
for cell in row:
design_spec_worksheet[cell.
coordinate].value = cell.value
design_spec_worksheet[
cell.coordinate
].value = cell.value
return design_spec
def get_xl_obj_and_sheetname(self, sheetname):
@ -400,10 +432,10 @@ class ExcelParser():
The logic confirms if the sheetname is specified for example as:
"MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs"
"""
if (re.search('.xlsx', sheetname) or re.search('.xls', sheetname)):
if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
""" Extract file name """
source_xl_file = sheetname.split(':')[0]
source_xl_file = sheetname.split(":")[0]
wb = load_workbook(source_xl_file, data_only=True)
return [wb, sheetname.split(':')[1]]
return [wb, sheetname.split(":")[1]]
else:
return [None, sheetname]

194
spyglass/data_extractor/plugins/tugboat/tugboat.py
View File

@ -25,8 +25,8 @@ LOG = logging.getLogger(__name__)
class TugboatPlugin(BaseDataSourcePlugin):
def __init__(self, region):
LOG.info("Tugboat Initializing")
self.source_type = 'excel'
self.source_name = 'tugboat'
self.source_type = "excel"
self.source_name = "tugboat"
# Configuration parameters
self.excel_path = None
@ -52,8 +52,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
Each plugin will have their own config opts.
"""
self.excel_path = conf['excel_path']
self.excel_spec = conf['excel_spec']
self.excel_path = conf["excel_path"]
self.excel_spec = conf["excel_spec"]
# Extract raw data from excel sheets
self._get_excel_obj()
@ -69,18 +69,18 @@ class TugboatPlugin(BaseDataSourcePlugin):
written as an additional safeguard.
"""
try:
assert (len(
kwargs['excel'])), "Engineering Spec file not specified"
excel_file_info = kwargs['excel']
assert (kwargs['excel_spec']
) is not None, "Excel Spec file not specified"
excel_spec_info = kwargs['excel_spec']
assert len(kwargs["excel"]), "Engineering Spec file not specified"
excel_file_info = kwargs["excel"]
assert (
kwargs["excel_spec"]
) is not None, "Excel Spec file not specified"
excel_spec_info = kwargs["excel_spec"]
except AssertionError as e:
LOG.error("{}:Spyglass exited!".format(e))
exit()
plugin_conf = {
'excel_path': excel_file_info,
'excel_spec': excel_spec_info
"excel_path": excel_file_info,
"excel_spec": excel_spec_info,
}
return plugin_conf
@ -103,19 +103,18 @@ class TugboatPlugin(BaseDataSourcePlugin):
]
"""
LOG.info("Get Host Information")
ipmi_data = self.parsed_xl_data['ipmi_data'][0]
ipmi_data = self.parsed_xl_data["ipmi_data"][0]
rackwise_hosts = self._get_rackwise_hosts()
host_list = []
for rack in rackwise_hosts.keys():
for host in rackwise_hosts[rack]:
host_list.append({
'rack_name':
rack,
'name':
host,
'host_profile':
ipmi_data[host]['host_profile']
})
host_list.append(
{
"rack_name": rack,
"name": host,
"host_profile": ipmi_data[host]["host_profile"],
}
)
return host_list
def get_networks(self, region):
@ -123,39 +122,44 @@ class TugboatPlugin(BaseDataSourcePlugin):
vlan_list = []
# Network data extracted from xl is formatted to have a predictable
# data type. For e.g VlAN 45 extracted from xl is formatted as 45
vlan_pattern = r'\d+'
private_net = self.parsed_xl_data['network_data']['private']
public_net = self.parsed_xl_data['network_data']['public']
vlan_pattern = r"\d+"
private_net = self.parsed_xl_data["network_data"]["private"]
public_net = self.parsed_xl_data["network_data"]["public"]
# Extract network information from private and public network data
for net_type, net_val in itertools.chain(private_net.items(),
public_net.items()):
for net_type, net_val in itertools.chain(
private_net.items(), public_net.items()
):
tmp_vlan = {}
# Ingress is special network that has no vlan, only a subnet string
# So treatment for ingress is different
if net_type is not 'ingress':
if net_type != "ingress":
# standardize the network name as net_type may ne different.
# For e.g insteas of pxe it may be PXE or instead of calico
# it may be ksn. Valid network names are pxe, calico, oob, oam,
# overlay, storage, ingress
tmp_vlan['name'] = self._get_network_name_from_vlan_name(
net_type)
tmp_vlan["name"] = self._get_network_name_from_vlan_name(
net_type
)
# extract vlan tag. It was extracted from xl file as 'VlAN 45'
# The code below extracts the numeric data fron net_val['vlan']
if net_val.get('vlan', "") is not "":
value = re.findall(vlan_pattern, net_val['vlan'])
tmp_vlan['vlan'] = value[0]
if net_val.get("vlan", "") != "":
value = re.findall(vlan_pattern, net_val["vlan"])
tmp_vlan["vlan"] = value[0]
else:
tmp_vlan['vlan'] = "#CHANGE_ME"
tmp_vlan["vlan"] = "#CHANGE_ME"
tmp_vlan['subnet'] = net_val.get('subnet', "#CHANGE_ME")
tmp_vlan['gateway'] = net_val.get('gateway', "#CHANGE_ME")
tmp_vlan["subnet"] = net_val.get("subnet", "#CHANGE_ME")
tmp_vlan["gateway"] = net_val.get("gateway", "#CHANGE_ME")
else:
tmp_vlan['name'] = 'ingress'
tmp_vlan['subnet'] = net_val
tmp_vlan["name"] = "ingress"
tmp_vlan["subnet"] = net_val
vlan_list.append(tmp_vlan)
LOG.debug("vlan list extracted from tugboat:\n{}".format(
pprint.pformat(vlan_list)))
LOG.debug(
"vlan list extracted from tugboat:\n{}".format(
pprint.pformat(vlan_list)
)
)
return vlan_list
def get_ips(self, region, host=None):
@ -172,33 +176,34 @@ class TugboatPlugin(BaseDataSourcePlugin):
"""
ip_ = {}
ipmi_data = self.parsed_xl_data['ipmi_data'][0]
ipmi_data = self.parsed_xl_data["ipmi_data"][0]
ip_[host] = {
'oob': ipmi_data[host].get('ipmi_address', '#CHANGE_ME'),
'oam': ipmi_data[host].get('oam', '#CHANGE_ME'),
'calico': ipmi_data[host].get('calico', '#CHANGE_ME'),
'overlay': ipmi_data[host].get('overlay', '#CHANGE_ME'),
'pxe': ipmi_data[host].get('pxe', '#CHANGE_ME'),
'storage': ipmi_data[host].get('storage', '#CHANGE_ME')
"oob": ipmi_data[host].get("ipmi_address", "#CHANGE_ME"),
"oam": ipmi_data[host].get("oam", "#CHANGE_ME"),
"calico": ipmi_data[host].get("calico", "#CHANGE_ME"),
"overlay": ipmi_data[host].get("overlay", "#CHANGE_ME"),
"pxe": ipmi_data[host].get("pxe", "#CHANGE_ME"),
"storage": ipmi_data[host].get("storage", "#CHANGE_ME"),
}
return ip_
def get_ldap_information(self, region):
""" Extract ldap information from excel"""
ldap_raw_data = self.parsed_xl_data['site_info']['ldap']
ldap_raw_data = self.parsed_xl_data["site_info"]["ldap"]
ldap_info = {}
# raw url is 'url: ldap://example.com' so we are converting to
# 'ldap://example.com'
url = ldap_raw_data.get('url', '#CHANGE_ME')
url = ldap_raw_data.get("url", "#CHANGE_ME")
try:
ldap_info['url'] = url.split(' ')[1]
ldap_info['domain'] = url.split('.')[1]
ldap_info["url"] = url.split(" ")[1]
ldap_info["domain"] = url.split(".")[1]
except IndexError as e:
LOG.error("url.split:{}".format(e))
ldap_info['common_name'] = ldap_raw_data.get('common_name',
'#CHANGE_ME')
ldap_info['subdomain'] = ldap_raw_data.get('subdomain', '#CHANGE_ME')
ldap_info["common_name"] = ldap_raw_data.get(
"common_name", "#CHANGE_ME"
)
ldap_info["subdomain"] = ldap_raw_data.get("subdomain", "#CHANGE_ME")
return ldap_info
@ -206,41 +211,44 @@ class TugboatPlugin(BaseDataSourcePlugin):
""" Returns a comma separated list of ntp ip addresses"""
ntp_server_list = self._get_formatted_server_list(
self.parsed_xl_data['site_info']['ntp'])
self.parsed_xl_data["site_info"]["ntp"]
)
return ntp_server_list
def get_dns_servers(self, region):
""" Returns a comma separated list of dns ip addresses"""
dns_server_list = self._get_formatted_server_list(
self.parsed_xl_data['site_info']['dns'])
self.parsed_xl_data["site_info"]["dns"]
)
return dns_server_list
def get_domain_name(self, region):
""" Returns domain name extracted from excel file"""
return self.parsed_xl_data['site_info']['domain']
return self.parsed_xl_data["site_info"]["domain"]
def get_location_information(self, region):
"""
Prepare location data from information extracted
by ExcelParser(i.e raw data)
"""
location_data = self.parsed_xl_data['site_info']['location']
location_data = self.parsed_xl_data["site_info"]["location"]
corridor_pattern = r'\d+'
corridor_number = re.findall(corridor_pattern,
location_data['corridor'])[0]
name = location_data.get('name', '#CHANGE_ME')
state = location_data.get('state', '#CHANGE_ME')
country = location_data.get('country', '#CHANGE_ME')
physical_location_id = location_data.get('physical_location', '')
corridor_pattern = r"\d+"
corridor_number = re.findall(
corridor_pattern, location_data["corridor"]
)[0]
name = location_data.get("name", "#CHANGE_ME")
state = location_data.get("state", "#CHANGE_ME")
country = location_data.get("country", "#CHANGE_ME")
physical_location_id = location_data.get("physical_location", "")
return {
'name': name,
'physical_location_id': physical_location_id,
'state': state,
'country': country,
'corridor': 'c{}'.format(corridor_number),
"name": name,
"physical_location_id": physical_location_id,
"state": state,
"country": country,
"corridor": "c{}".format(corridor_number),
}
def get_racks(self, region):
@ -277,29 +285,35 @@ class TugboatPlugin(BaseDataSourcePlugin):
vlan_name contains "pxe" the network name is "pxe"
"""
network_names = [
'ksn|calico', 'storage', 'oam|server', 'ovs|overlay', 'oob', 'pxe'
"ksn|calico",
"storage",
"oam|server",
"ovs|overlay",
"oob",
"pxe",
]
for name in network_names:
# Make a pattern that would ignore case.
# if name is 'ksn' pattern name is '(?i)(ksn)'
name_pattern = "(?i)({})".format(name)
if re.search(name_pattern, vlan_name):
if name is 'ksn|calico':
return 'calico'
if name is 'storage':
return 'storage'
if name is 'oam|server':
return 'oam'
if name is 'ovs|overlay':
return 'overlay'
if name is 'oob':
return 'oob'
if name is 'pxe':
return 'pxe'
if name == "ksn|calico":
return "calico"
if name == "storage":
return "storage"
if name == "oam|server":
return "oam"
if name == "ovs|overlay":
return "overlay"
if name == "oob":
return "oob"
if name == "pxe":
return "pxe"
# if nothing matches
LOG.error(
"Unable to recognize VLAN name extracted from Plugin data source")
return ("")
"Unable to recognize VLAN name extracted from Plugin data source"
)
return ""
def _get_formatted_server_list(self, server_list):
""" Format dns and ntp server list as comma separated string """
@ -309,9 +323,9 @@ class TugboatPlugin(BaseDataSourcePlugin):
# The function returns a list of comma separated dns ip addresses
servers = []
for data in server_list:
if '(' not in data:
if "(" not in data:
servers.append(data)
formatted_server_list = ','.join(servers)
formatted_server_list = ",".join(servers)
return formatted_server_list
def _get_rack(self, host):
@ -319,7 +333,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
Get rack id from the rack string extracted
from xl
"""
rack_pattern = r'\w.*(r\d+)\w.*'
rack_pattern = r"\w.*(r\d+)\w.*"
rack = re.findall(rack_pattern, host)[0]
if not self.region:
self.region = host.split(rack)[0]
@ -328,7 +342,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
def _get_rackwise_hosts(self):
""" Mapping hosts with rack ids """
rackwise_hosts = {}
hostnames = self.parsed_xl_data['ipmi_data'][1]
hostnames = self.parsed_xl_data["ipmi_data"][1]
racks = self._get_rack_data()
for rack in racks:
if rack not in rackwise_hosts:
@ -343,8 +357,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
""" Format rack name """
LOG.info("Getting rack data")
racks = {}
hostnames = self.parsed_xl_data['ipmi_data'][1]
hostnames = self.parsed_xl_data["ipmi_data"][1]
for host in hostnames:
rack = self._get_rack(host)
racks[rack] = rack.replace('r', 'rack')
racks[rack] = rack.replace("r", "rack")
return racks

217
spyglass/parser/engine.py
View File

@ -28,7 +28,7 @@ import yaml
LOG = logging.getLogger(__name__)
class ProcessDataSource():
class ProcessDataSource:
def __init__(self, sitetype):
# Initialize intermediary and save site type
self._initialize_intermediary()
@ -36,18 +36,18 @@ class ProcessDataSource():
@staticmethod
def _read_file(file_name):
with open(file_name, 'r') as f:
with open(file_name, "r") as f:
raw_data = f.read()
return raw_data
def _initialize_intermediary(self):
self.host_type = {}
self.data = {
'network': {},
'baremetal': {},
'region_name': '',
'storage': {},
'site_info': {},
"network": {},
"baremetal": {},
"region_name": "",
"storage": {},
"site_info": {},
}
self.sitetype = None
self.genesis_node = None
@ -62,37 +62,43 @@ class ProcessDataSource():
we assign only the first subnet """
LOG.info("Extracting network subnets")
network_subnets = {}
for net_type in self.data['network']['vlan_network_data']:
for net_type in self.data["network"]["vlan_network_data"]:
# One of the type is ingress and we don't want that here
if (net_type != 'ingress'):
if net_type != "ingress":
network_subnets[net_type] = netaddr.IPNetwork(
self.data['network']['vlan_network_data'][net_type]
['subnet'][0])
self.data["network"]["vlan_network_data"][net_type][
"subnet"
][0]
)
LOG.debug("Network subnets:\n{}".format(
pprint.pformat(network_subnets)))
LOG.debug(
"Network subnets:\n{}".format(pprint.pformat(network_subnets))
)
return network_subnets
def _get_genesis_node_details(self):
# Get genesis host node details from the hosts based on host type
for racks in self.data['baremetal'].keys():
rack_hosts = self.data['baremetal'][racks]
for racks in self.data["baremetal"].keys():
rack_hosts = self.data["baremetal"][racks]
for host in rack_hosts:
if rack_hosts[host]['type'] == 'genesis':
if rack_hosts[host]["type"] == "genesis":
self.genesis_node = rack_hosts[host]
self.genesis_node['name'] = host
LOG.debug("Genesis Node Details:\n{}".format(
pprint.pformat(self.genesis_node)))
self.genesis_node["name"] = host
LOG.debug(
"Genesis Node Details:\n{}".format(
pprint.pformat(self.genesis_node)
)
)
def _get_genesis_node_ip(self):
""" Returns the genesis node ip """
ip = '0.0.0.0'
ip = "0.0.0.0"
LOG.info("Getting Genesis Node IP")
if not self.genesis_node:
self._get_genesis_node_details()
ips = self.genesis_node.get('ip', '')
ips = self.genesis_node.get("ip", "")
if ips:
ip = ips.get('oam', '0.0.0.0')
ip = ips.get("oam", "0.0.0.0")
return ip
def _validate_intermediary_data(self, data):
@ -103,21 +109,21 @@ class ProcessDataSource():
The method validates this with regex pattern defined for each
data type.
"""
LOG.info('Validating Intermediary data')
LOG.info("Validating Intermediary data")
temp_data = {}
# Peforming a deep copy
temp_data = copy.deepcopy(data)
# Converting baremetal dict to list.
baremetal_list = []
for rack in temp_data['baremetal'].keys():
temp = [{k: v} for k, v in temp_data['baremetal'][rack].items()]
for rack in temp_data["baremetal"].keys():
temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()]
baremetal_list = baremetal_list + temp
temp_data['baremetal'] = baremetal_list
schema_dir = pkg_resources.resource_filename('spyglass', 'schemas/')
temp_data["baremetal"] = baremetal_list
schema_dir = pkg_resources.resource_filename("spyglass", "schemas/")
schema_file = schema_dir + "data_schema.json"
json_data = json.loads(json.dumps(temp_data))
with open(schema_file, 'r') as f:
with open(schema_file, "r") as f:
json_schema = json.load(f)
try:
# Suppressing writing of data2.json. Can use it for debugging
@ -152,14 +158,14 @@ class ProcessDataSource():
based on rule name and applies them to appropriate data objects.
"""
LOG.info("Apply design rules")
rules_dir = pkg_resources.resource_filename('spyglass', 'config/')
rules_file = rules_dir + 'rules.yaml'
rules_dir = pkg_resources.resource_filename("spyglass", "config/")
rules_file = rules_dir + "rules.yaml"
rules_data_raw = self._read_file(rules_file)
rules_yaml = yaml.safe_load(rules_data_raw)
rules_data = {}
rules_data.update(rules_yaml)
for rule in rules_data.keys():
rule_name = rules_data[rule]['name']
rule_name = rules_data[rule]["name"]
function_str = "_apply_rule_" + rule_name
rule_data_name = rules_data[rule][rule_name]
function = getattr(self, function_str)
@ -182,23 +188,25 @@ class ProcessDataSource():
compute or controller based on host_profile. For defining 'genesis'
the first controller host is defined as genesis."""
is_genesis = False
hardware_profile = rule_data[self.data['site_info']['sitetype']]
hardware_profile = rule_data[self.data["site_info"]["sitetype"]]
# Getting individual racks. The racks are sorted to ensure that the
# first controller of the first rack is assigned as 'genesis' node.
for rack in sorted(self.data['baremetal'].keys()):
for rack in sorted(self.data["baremetal"].keys()):
# Getting individual hosts in each rack. Sorting of the hosts are
# done to determine the genesis node.
for host in sorted(self.data['baremetal'][rack].keys()):
host_info = self.data['baremetal'][rack][host]
if (host_info['host_profile'] == hardware_profile[
'profile_name']['ctrl']):
for host in sorted(self.data["baremetal"][rack].keys()):
host_info = self.data["baremetal"][rack][host]
if (
host_info["host_profile"]
== hardware_profile["profile_name"]["ctrl"]
):
if not is_genesis:
host_info['type'] = 'genesis'
host_info["type"] = "genesis"
is_genesis = True
else:
host_info['type'] = 'controller'
host_info["type"] = "controller"
else:
host_info['type'] = 'compute'
host_info["type"] = "compute"
def _apply_rule_ip_alloc_offset(self, rule_data):
""" Apply offset rules to update baremetal host ip's and vlan network
@ -219,21 +227,24 @@ class ProcessDataSource():
If a particular ip exists it is overridden."""
# Ger defult ip offset
default_ip_offset = rule_data['default']
default_ip_offset = rule_data["default"]
host_idx = 0
LOG.info("Update baremetal host ip's")
for racks in self.data['baremetal'].keys():
rack_hosts = self.data['baremetal'][racks]
for racks in self.data["baremetal"].keys():
rack_hosts = self.data["baremetal"][racks]
for host in rack_hosts:
host_networks = rack_hosts[host]['ip']
host_networks = rack_hosts[host]["ip"]
for net in host_networks:
ips = list(self.network_subnets[net])
host_networks[net] = str(ips[host_idx + default_ip_offset])
host_idx = host_idx + 1
LOG.debug("Updated baremetal host:\n{}".format(
pprint.pformat(self.data['baremetal'])))
LOG.debug(
"Updated baremetal host:\n{}".format(
pprint.pformat(self.data["baremetal"])
)
)
def _update_vlan_net_data(self, rule_data):
""" Offset allocation rules to determine ip address range(s)
@ -245,31 +256,37 @@ class ProcessDataSource():
LOG.info("Apply network design rules")
# Collect Rules
default_ip_offset = rule_data['default']
oob_ip_offset = rule_data['oob']
gateway_ip_offset = rule_data['gateway']
ingress_vip_offset = rule_data['ingress_vip']
default_ip_offset = rule_data["default"]
oob_ip_offset = rule_data["oob"]
gateway_ip_offset = rule_data["gateway"]
ingress_vip_offset = rule_data["ingress_vip"]
# static_ip_end_offset for non pxe network
static_ip_end_offset = rule_data['static_ip_end']
static_ip_end_offset = rule_data["static_ip_end"]
# dhcp_ip_end_offset for pxe network
dhcp_ip_end_offset = rule_data['dhcp_ip_end']
dhcp_ip_end_offset = rule_data["dhcp_ip_end"]
# Set ingress vip and CIDR for bgp
LOG.info("Apply network design rules:bgp")
subnet = netaddr.IPNetwork(
self.data['network']['vlan_network_data']['ingress']['subnet'][0])
self.data["network"]["vlan_network_data"]["ingress"]["subnet"][0]
)
ips = list(subnet)
self.data['network']['bgp']['ingress_vip'] = str(
ips[ingress_vip_offset])
self.data['network']['bgp']['public_service_cidr'] = self.data[
'network']['vlan_network_data']['ingress']['subnet'][0]
LOG.debug("Updated network bgp data:\n{}".format(
pprint.pformat(self.data['network']['bgp'])))
self.data["network"]["bgp"]["ingress_vip"] = str(
ips[ingress_vip_offset]
)
self.data["network"]["bgp"]["public_service_cidr"] = self.data[
"network"
]["vlan_network_data"]["ingress"]["subnet"][0]
LOG.debug(
"Updated network bgp data:\n{}".format(
pprint.pformat(self.data["network"]["bgp"])
)
)
LOG.info("Apply network design rules:vlan")
# Apply rules to vlan networks
for net_type in self.network_subnets:
if net_type == 'oob':
if net_type == "oob":
ip_offset = oob_ip_offset
else:
ip_offset = default_ip_offset
@ -277,49 +294,60 @@ class ProcessDataSource():
subnet = self.network_subnets[net_type]
ips = list(subnet)
self.data['network']['vlan_network_data'][net_type][
'gateway'] = str(ips[gateway_ip_offset])
self.data["network"]["vlan_network_data"][net_type][
"gateway"
] = str(ips[gateway_ip_offset])
self.data['network']['vlan_network_data'][net_type][
'reserved_start'] = str(ips[1])
self.data['network']['vlan_network_data'][net_type][
'reserved_end'] = str(ips[ip_offset])
self.data["network"]["vlan_network_data"][net_type][
"reserved_start"
] = str(ips[1])
self.data["network"]["vlan_network_data"][net_type][
"reserved_end"
] = str(ips[ip_offset])
static_start = str(ips[ip_offset + 1])
static_end = str(ips[static_ip_end_offset])
if net_type == 'pxe':
if net_type == "pxe":
mid = len(ips) // 2
static_end = str(ips[mid - 1])
dhcp_start = str(ips[mid])
dhcp_end = str(ips[dhcp_ip_end_offset])
self.data['network']['vlan_network_data'][net_type][
'dhcp_start'] = dhcp_start
self.data['network']['vlan_network_data'][net_type][
'dhcp_end'] = dhcp_end
self.data["network"]["vlan_network_data"][net_type][
"dhcp_start"
] = dhcp_start
self.data["network"]["vlan_network_data"][net_type][
"dhcp_end"
] = dhcp_end
self.data['network']['vlan_network_data'][net_type][
'static_start'] = static_start
self.data['network']['vlan_network_data'][net_type][
'static_end'] = static_end
self.data["network"]["vlan_network_data"][net_type][
"static_start"
] = static_start
self.data["network"]["vlan_network_data"][net_type][
"static_end"
] = static_end
# There is no vlan for oob network
if (net_type != 'oob'):
self.data['network']['vlan_network_data'][net_type][
'vlan'] = self.data['network']['vlan_network_data'][
net_type]['vlan']
if net_type != "oob":
self.data["network"]["vlan_network_data"][net_type][
"vlan"
] = self.data["network"]["vlan_network_data"][net_type]["vlan"]
# OAM have default routes. Only for cruiser. TBD
if (net_type == 'oam'):
if net_type == "oam":
routes = ["0.0.0.0/0"]
else:
routes = []
self.data['network']['vlan_network_data'][net_type][
'routes'] = routes
self.data["network"]["vlan_network_data"][net_type][
"routes"
] = routes
LOG.debug("Updated vlan network data:\n{}".format(
pprint.pformat(self.data['network']['vlan_network_data'])))
LOG.debug(
"Updated vlan network data:\n{}".format(
pprint.pformat(self.data["network"]["vlan_network_data"])
)
)
def load_extracted_data_from_data_source(self, extracted_data):
"""
@ -334,8 +362,11 @@ class ProcessDataSource():
LOG.info("Loading plugin data source")
self.data = extracted_data
LOG.debug("Extracted data from plugin:\n{}".format(
pprint.pformat(extracted_data)))
LOG.debug(
"Extracted data from plugin:\n{}".format(
pprint.pformat(extracted_data)
)
)
# Uncommeent following segment for debugging purpose.
# extracted_file = "extracted_file.yaml"
# yaml_file = yaml.dump(extracted_data, default_flow_style=False)
@ -344,13 +375,14 @@ class ProcessDataSource():
# f.close()
# Append region_data supplied from CLI to self.data
self.data['region_name'] = self.region_name
self.data["region_name"] = self.region_name
def dump_intermediary_file(self, intermediary_dir):
""" Writing intermediary yaml """
LOG.info("Writing intermediary yaml")
intermediary_file = "{}_intermediary.yaml".format(
self.data['region_name'])
self.data["region_name"]
)
# Check of if output dir = intermediary_dir exists
if intermediary_dir is not None:
outfile = "{}/{}".format(intermediary_dir, intermediary_file)
@ -358,7 +390,7 @@ class ProcessDataSource():
outfile = intermediary_file
LOG.info("Intermediary file:{}".format(outfile))
yaml_file = yaml.dump(self.data, default_flow_style=False)
with open(outfile, 'w') as f:
with open(outfile, "w") as f:
f.write(yaml_file)
f.close()
@ -379,10 +411,11 @@ class ProcessDataSource():
def edit_intermediary_yaml(self):
""" Edit generated data using on browser """
LOG.info(
"edit_intermediary_yaml: Invoking web server for yaml editing")
with tempfile.NamedTemporaryFile(mode='r+') as file_obj:
"edit_intermediary_yaml: Invoking web server for yaml editing"
)
with tempfile.NamedTemporaryFile(mode="r+") as file_obj:
yaml.safe_dump(self.data, file_obj, default_flow_style=False)
host = self._get_genesis_node_ip()
os.system('yaml-editor -f {0} -h {1}'.format(file_obj.name, host))
os.system("yaml-editor -f {0} -h {1}".format(file_obj.name, host))
file_obj.seek(0)
self.data = yaml.safe_load(file_obj)

29
spyglass/site_processors/base.py
View File

@ -22,23 +22,20 @@ class BaseProcessor:
@staticmethod
def get_role_wise_nodes(yaml_data):
hosts = {
'genesis': {},
'masters': [],
'workers': [],
}
hosts = {"genesis": {}, "masters": [], "workers": []}
for rack in yaml_data['baremetal']:
for host in yaml_data['baremetal'][rack]:
if yaml_data['baremetal'][rack][host]['type'] == 'genesis':
hosts['genesis'] = {
'name': host,
'pxe': yaml_data['baremetal'][rack][host]['ip']['pxe'],
'oam': yaml_data['baremetal'][rack][host]['ip']['oam'],
for rack in yaml_data["baremetal"]:
for host in yaml_data["baremetal"][rack]:
if yaml_data["baremetal"][rack][host]["type"] == "genesis":
hosts["genesis"] = {
"name": host,
"pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"],
"oam": yaml_data["baremetal"][rack][host]["ip"]["oam"],
}
elif yaml_data['baremetal'][rack][host][
'type'] == 'controller':
hosts['masters'].append(host)
elif (
yaml_data["baremetal"][rack][host]["type"] == "controller"
):
hosts["masters"].append(host)
else:
hosts['workers'].append(host)
hosts["workers"].append(host)
return hosts

30
spyglass/site_processors/site_processor.py
View File

@ -36,12 +36,12 @@ class SiteProcessor(BaseProcessor):
"""
# Check of manifest_dir exists
if self.manifest_dir is not None:
site_manifest_dir = self.manifest_dir + '/pegleg_manifests/site/'
site_manifest_dir = self.manifest_dir + "/pegleg_manifests/site/"
else:
site_manifest_dir = 'pegleg_manifests/site/'
site_manifest_dir = "pegleg_manifests/site/"
LOG.info("Site manifest output dir:{}".format(site_manifest_dir))
template_software_dir = template_dir + '/'
template_software_dir = template_dir + "/"
template_dir_abspath = os.path.dirname(template_software_dir)
LOG.debug("Template Path:%s", template_dir_abspath)
@ -50,16 +50,19 @@ class SiteProcessor(BaseProcessor):
j2_env = Environment(
autoescape=False,
loader=FileSystemLoader(dirpath),
trim_blocks=True)
trim_blocks=True,
)
j2_env.filters[
'get_role_wise_nodes'] = self.get_role_wise_nodes
"get_role_wise_nodes"
] = self.get_role_wise_nodes
templatefile = os.path.join(dirpath, filename)
outdirs = dirpath.split('templates')[1]
outdirs = dirpath.split("templates")[1]
outfile_path = '{}{}{}'.format(
site_manifest_dir, self.yaml_data['region_name'], outdirs)
outfile_yaml = templatefile.split('.j2')[0].split('/')[-1]
outfile = outfile_path + '/' + outfile_yaml
outfile_path = "{}{}{}".format(
site_manifest_dir, self.yaml_data["region_name"], outdirs
)
outfile_yaml = templatefile.split(".j2")[0].split("/")[-1]
outfile = outfile_path + "/" + outfile_yaml
outfile_dir = os.path.dirname(outfile)
if not os.path.exists(outfile_dir):
os.makedirs(outfile_dir)
@ -71,7 +74,10 @@ class SiteProcessor(BaseProcessor):
out.close()
except IOError as ioe:
LOG.error(
"IOError during rendering:{}".format(outfile_yaml))
"IOError during rendering:{}".format(outfile_yaml)
)
raise SystemExit(
"Error when generating {:s}:\n{:s}".format(
outfile, ioe.strerror))
outfile, ioe.strerror
)
)

161
spyglass/spyglass.py
View File

@ -22,98 +22,115 @@ import yaml
from spyglass.parser.engine import ProcessDataSource
from spyglass.site_processors.site_processor import SiteProcessor
LOG = logging.getLogger('spyglass')
LOG = logging.getLogger("spyglass")
@click.command()
@click.option(
'--site',
'-s',
help='Specify the site for which manifests to be generated')
"--site", "-s", help="Specify the site for which manifests to be generated"
)
@click.option(
'--type', '-t', help='Specify the plugin type formation or tugboat')
@click.option('--formation_url', '-f', help='Specify the formation url')
@click.option('--formation_user', '-u', help='Specify the formation user id')
"--type", "-t", help="Specify the plugin type formation or tugboat"
)
@click.option("--formation_url", "-f", help="Specify the formation url")
@click.option("--formation_user", "-u", help="Specify the formation user id")
@click.option(
'--formation_password', '-p', help='Specify the formation user password')
"--formation_password", "-p", help="Specify the formation user password"
)
@click.option(
'--intermediary',
'-i',
"--intermediary",
"-i",
type=click.Path(exists=True),
help=
'Intermediary file path generate manifests, use -m also with this option')
help=(
"Intermediary file path generate manifests, "
"use -m also with this option"
),
)
@click.option(
'--additional_config',
'-d',
"--additional_config",
"-d",
type=click.Path(exists=True),
help='Site specific configuraton details')
help="Site specific configuraton details",
)
@click.option(
'--generate_intermediary',
'-g',
"--generate_intermediary",
"-g",
is_flag=True,
help='Dump intermediary file from passed excel and excel spec')
help="Dump intermediary file from passed excel and excel spec",
)
@click.option(
'--intermediary_dir',
'-idir',
"--intermediary_dir",
"-idir",
type=click.Path(exists=True),
help='The path where intermediary file needs to be generated')
help="The path where intermediary file needs to be generated",
)
@click.option(
'--edit_intermediary/--no_edit_intermediary',
'-e/-nedit',
"--edit_intermediary/--no_edit_intermediary",
"-e/-nedit",
default=True,
help='Flag to let user edit intermediary')
help="Flag to let user edit intermediary",
)
@click.option(
'--generate_manifests',
'-m',
"--generate_manifests",
"-m",
is_flag=True,
help='Generate manifests from the generated intermediary file')
help="Generate manifests from the generated intermediary file",
)
@click.option(
'--manifest_dir',
'-mdir',
"--manifest_dir",
"-mdir",
type=click.Path(exists=True),
help='The path where manifest files needs to be generated')
help="The path where manifest files needs to be generated",
)
@click.option(
'--template_dir',
'-tdir',
"--template_dir",
"-tdir",
type=click.Path(exists=True),
help='The path where J2 templates are available')
help="The path where J2 templates are available",
)
@click.option(
'--excel',
'-x',
"--excel",
"-x",
multiple=True,
type=click.Path(exists=True),
help=
'Path to engineering excel file, to be passed with generate_intermediary')
help=(
"Path to engineering excel file, to be passed with "
"generate_intermediary"
),
)
@click.option(
'--excel_spec',
'-e',
"--excel_spec",
"-e",
type=click.Path(exists=True),
help='Path to excel spec, to be passed with generate_intermediary')
help="Path to excel spec, to be passed with generate_intermediary",
)
@click.option(
'--loglevel',
'-l',
"--loglevel",
"-l",
default=20,
multiple=False,
show_default=True,
help='Loglevel NOTSET:0 ,DEBUG:10, \
INFO:20, WARNING:30, ERROR:40, CRITICAL:50')
help="Loglevel NOTSET:0 ,DEBUG:10, \
INFO:20, WARNING:30, ERROR:40, CRITICAL:50",
)
def main(*args, **kwargs):
# Extract user provided inputs
generate_intermediary = kwargs['generate_intermediary']
intermediary_dir = kwargs['intermediary_dir']
edit_intermediary = kwargs['edit_intermediary']
generate_manifests = kwargs['generate_manifests']
manifest_dir = kwargs['manifest_dir']
intermediary = kwargs['intermediary']
site = kwargs['site']
template_dir = kwargs['template_dir']
loglevel = kwargs['loglevel']
generate_intermediary = kwargs["generate_intermediary"]
intermediary_dir = kwargs["intermediary_dir"]
edit_intermediary = kwargs["edit_intermediary"]
generate_manifests = kwargs["generate_manifests"]
manifest_dir = kwargs["manifest_dir"]
intermediary = kwargs["intermediary"]
site = kwargs["site"]
template_dir = kwargs["template_dir"]
loglevel = kwargs["loglevel"]
# Set Logging format
LOG.setLevel(loglevel)
stream_handle = logging.StreamHandler()
formatter = logging.Formatter(
'(%(name)s): %(asctime)s %(levelname)s %(message)s')
"(%(name)s): %(asctime)s %(levelname)s %(message)s"
)
stream_handle.setFormatter(formatter)
LOG.addHandler(stream_handle)
@ -139,19 +156,21 @@ def main(*args, **kwargs):
intermediary_yaml = {}
if intermediary is None:
LOG.info("Generating Intermediary yaml")
plugin_type = kwargs.get('type', None)
plugin_type = kwargs.get("type", None)
plugin_class = None
# Discover the plugin and load the plugin class
LOG.info("Load the plugin class")
for entry_point in pkg_resources.iter_entry_points(
'data_extractor_plugins'):
"data_extractor_plugins"
):
if entry_point.name == plugin_type:
plugin_class = entry_point.load()
if plugin_class is None:
LOG.error(
"Unsupported Plugin type. Plugin type:{}".format(plugin_type))
"Unsupported Plugin type. Plugin type:{}".format(plugin_type)
)
exit()
# Extract data from plugin data source
@ -162,16 +181,22 @@ def main(*args, **kwargs):
data_extractor.extract_data()
# Apply any additional_config provided by user
additional_config = kwargs.get('additional_config', None)
additional_config = kwargs.get("additional_config", None)
if additional_config is not None:
with open(additional_config, 'r') as config:
with open(additional_config, "r") as config:
raw_data = config.read()
additional_config_data = yaml.safe_load(raw_data)
LOG.debug("Additional config data:\n{}".format(
pprint.pformat(additional_config_data)))
LOG.debug(
"Additional config data:\n{}".format(
pprint.pformat(additional_config_data)
)
)
LOG.info("Apply additional configuration from:{}".format(
additional_config))
LOG.info(
"Apply additional configuration from:{}".format(
additional_config
)
)
data_extractor.apply_additional_data(additional_config_data)
LOG.debug(pprint.pformat(data_extractor.site_data))
@ -179,14 +204,16 @@ def main(*args, **kwargs):
LOG.info("Apply design rules to the extracted data")
process_input_ob = ProcessDataSource(site)
process_input_ob.load_extracted_data_from_data_source(
data_extractor.site_data)
data_extractor.site_data
)
LOG.info("Generate intermediary yaml")
intermediary_yaml = process_input_ob.generate_intermediary_yaml(
edit_intermediary)
edit_intermediary
)
else:
LOG.info("Loading intermediary from user provided input")
with open(intermediary, 'r') as intermediary_file:
with open(intermediary, "r") as intermediary_file:
raw_data = intermediary_file.read()
intermediary_yaml = yaml.safe_load(raw_data)
@ -201,5 +228,5 @@ def main(*args, **kwargs):
LOG.info("Spyglass Execution Completed")
if __name__ == '__main__':
if __name__ == "__main__":
main()

119
spyglass/utils/editor/editor.py
View File

@ -26,49 +26,55 @@ from flask_bootstrap import Bootstrap
app_path = os.path.dirname(os.path.abspath(__file__))
app = Flask('Yaml Editor!',
template_folder=os.path.join(app_path, 'templates'),
static_folder=os.path.join(app_path, 'static'))
app = Flask(
"Yaml Editor!",
template_folder=os.path.join(app_path, "templates"),
static_folder=os.path.join(app_path, "static"),
)
Bootstrap(app)
logging.getLogger('werkzeug').setLevel(logging.ERROR)
logging.getLogger("werkzeug").setLevel(logging.ERROR)
LOG = app.logger
@app.route('/favicon.ico')
@app.route("/favicon.ico")
def favicon():
return send_from_directory(app.static_folder, 'favicon.ico')
return send_from_directory(app.static_folder, "favicon.ico")
@app.route('/', methods=['GET', 'POST'])
@app.route("/", methods=["GET", "POST"])
def index():
"""Renders index page to edit provided yaml file."""
LOG.info('Rendering yaml file for editing')
with open(app.config['YAML_FILE']) as file_obj:
LOG.info("Rendering yaml file for editing")
with open(app.config["YAML_FILE"]) as file_obj:
data = yaml.safe_load(file_obj)
return render_template('yaml.html',
data=json.dumps(data),
change_str=app.config['STRING_TO_CHANGE'])
return render_template(
"yaml.html",
data=json.dumps(data),
change_str=app.config["STRING_TO_CHANGE"],
)
@app.route('/save', methods=['POST'])
@app.route("/save", methods=["POST"])
def save():
"""Save current progress on file."""
LOG.info('Saving edited inputs from user to yaml file')
out = request.json.get('yaml_data')
with open(app.config['YAML_FILE'], 'w') as file_obj:
LOG.info("Saving edited inputs from user to yaml file")
out = request.json.get("yaml_data")
with open(app.config["YAML_FILE"], "w") as file_obj:
yaml.safe_dump(out, file_obj, default_flow_style=False)
return "Data saved successfully!"
@app.route('/saveExit', methods=['POST'])
@app.route("/saveExit", methods=["POST"])
def save_exit():
"""Save current progress on file and shuts down the server."""
LOG.info('Saving edited inputs from user to yaml file and shutting'
' down server')
out = request.json.get('yaml_data')
with open(app.config['YAML_FILE'], 'w') as file_obj:
LOG.info(
"Saving edited inputs from user to yaml file and shutting"
" down server"
)
out = request.json.get("yaml_data")
with open(app.config["YAML_FILE"], "w") as file_obj:
yaml.safe_dump(out, file_obj, default_flow_style=False)
func = request.environ.get('werkzeug.server.shutdown')
func = request.environ.get("werkzeug.server.shutdown")
if func:
func()
return "Saved successfully, Shutting down app! You may close the tab!"
@ -77,68 +83,72 @@ def save_exit():
@app.errorhandler(404)
def page_not_found(e):
"""Serves 404 error."""
LOG.info('User tried to access unavailable page.')
return '<h1>404: Page not Found!</h1>'
LOG.info("User tried to access unavailable page.")
return "<h1>404: Page not Found!</h1>"
def run(*args, **kwargs):
"""Starts the server."""
LOG.info('Initiating web server for yaml editing')
port = kwargs.get('port', None)
LOG.info("Initiating web server for yaml editing")
port = kwargs.get("port", None)
if not port:
port = 8161
app.run(host='0.0.0.0', port=port, debug=False)
app.run(host="0.0.0.0", port=port, debug=False)
@click.command()
@click.option(
'--file',
'-f',
"--file",
"-f",
required=True,
type=click.File(),
multiple=False,
help="Path with file name to the intermediary yaml file."
help="Path with file name to the intermediary yaml file.",
)
@click.option(
'--host',
'-h',
default='0.0.0.0',
"--host",
"-h",
default="0.0.0.0",
type=click.STRING,
multiple=False,
help="Optional host parameter to run Flask on."
help="Optional host parameter to run Flask on.",
)
@click.option(
'--port',
'-p',
"--port",
"-p",
default=8161,
type=click.INT,
multiple=False,
help="Optional port parameter to run Flask on."
help="Optional port parameter to run Flask on.",
)
@click.option(
'--string',
'-s',
default='#CHANGE_ME',
"--string",
"-s",
default="#CHANGE_ME",
type=click.STRING,
multiple=False,
help="Text which is required to be changed on yaml file."
help="Text which is required to be changed on yaml file.",
)
def main(*args, **kwargs):
LOG.setLevel(logging.INFO)
LOG.info('Initiating yaml-editor')
LOG.info("Initiating yaml-editor")
try:
yaml.safe_load(kwargs['file'])
yaml.safe_load(kwargs["file"])
except yaml.YAMLError as e:
LOG.error('EXITTING - Please provide a valid yaml file.')
if hasattr(e, 'problem_mark'):
LOG.error("EXITTING - Please provide a valid yaml file.")
if hasattr(e, "problem_mark"):
mark = e.problem_mark
LOG.error("Error position: ({0}:{1})".format(
mark.line + 1, mark.column + 1))
LOG.error(
"Error position: ({0}:{1})".format(
mark.line + 1, mark.column + 1
)
)
sys.exit(2)
except Exception:
LOG.error('EXITTING - Please provide a valid yaml file.')
LOG.error("EXITTING - Please provide a valid yaml file.")
sys.exit(2)
LOG.info("""
LOG.info(
"""
##############################################################################
@ -146,12 +156,15 @@ Please go to http://{0}:{1}/ to edit your yaml file.
##############################################################################
""".format(kwargs['host'], kwargs['port']))
app.config['YAML_FILE'] = kwargs['file'].name
app.config['STRING_TO_CHANGE'] = kwargs['string']
""".format(
kwargs["host"], kwargs["port"]
)
)
app.config["YAML_FILE"] = kwargs["file"].name
app.config["STRING_TO_CHANGE"] = kwargs["string"]
run(*args, **kwargs)
if __name__ == '__main__':
if __name__ == "__main__":
"""Invoked when used as a script."""
main()