diff --git a/doc/source/index.rst b/doc/source/index.rst
index c7e7824..2aa9c51 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -25,9 +25,9 @@ What is the Spyglass XLS Plugin?
The Spyglass XLS plugin is used to generate airship-seaworthy site manifest
files from an excel based engineering spec. The plugin is configured with an
-Excel sheet and its corresponding excel specification as inputs. Spyglass uses
-this plugin to construct an intermediary yaml which is processed further using
-J2 templates to generate site manifests.
+Excel sheet or a CSV file and its corresponding excel specification as inputs.
+Spyglass uses this plugin to construct an intermediary yaml which is processed
+further using Jinja2 templates to generate site manifests.
Excel Specification
-------------------
@@ -38,7 +38,8 @@ need to be filled by the Deployment Engineer.
Below is the definition for each key in the Excel spec
* ipmi_sheet_name - name of the sheet from where IPMI and host profile
- information is to be read
+ information is to be read (File name, excluding the file extension, when
+ using a CSV file)
* start_row - row number from where the IPMI and host profile information
starts
* end_row - row number from where the IPMI and host profile information ends
@@ -88,21 +89,34 @@ Below is the definition for each key in the Excel spec
* country_name_row - row number which has the country name
* clli_name_row - row number which has CLLI information
-Example: Tugboat Plugin Usage
------------------------------
+Example: Spyglass XLS Plugin Usage
+----------------------------------
-1. Required Input (Refer to 'spyglass/examples' folder to get these inputs)
+1. Required Input (Refer to 'spyglass_plugin_xls/examples' folder to get these
+inputs)
a) Excel File: SiteDesignSpec_v0.1.xlsx
b) Excel Spec: excel_spec_upstream.yaml
c) Site Config: site_config.yaml
- d) Template_dir: '../examples/templates'
+ d) Template_dir: 'spyglass/examples/templates'
e) Site name: airship-seaworthy
-2. Spyglass CLI Command:
+2. Spyglass CLI Command using an Excel File:
.. code-block:: bash
- spyglass m -i -p tugboat -x SiteDesignSpec_v0.1.xlsx \
- -e excel_spec_upstream.yaml -c site_config.yaml \
- -s airship-seaworthy -t <relative path to J2 templates dir>
\ No newline at end of file
+ spyglass excel documents -i \
+ -x ../spyglass-plugin-xls/spyglass_plugin_xls/examples/SiteDesignSpec_v0.1.xlsx \
+ -e ../spyglass-plugin-xls/spyglass_plugin_xls/examples/excel_spec.yaml \
+ -c ../spyglass-plugin-xls/spyglass_plugin_xls/examples/site_config.yaml \
+ -s airship-seaworthy -t spyglass/examples/templates/
+
+3. Spyglass CLI Command using a CSV File:
+
+.. code-block:: bash
+
+ spyglass excel documents -i \
+ -x ../spyglass-plugin-xls/spyglass_plugin_xls/examples/Site-Information.csv \
+ -e ../spyglass-plugin-xls/spyglass_plugin_xls/examples/excel_spec.yaml \
+ -c ../spyglass-plugin-xls/spyglass_plugin_xls/examples/site_config.yaml \
+ -s airship-seaworthy -t spyglass/examples/templates/
\ No newline at end of file
diff --git a/spyglass_plugin_xls/examples/Site-Information.csv b/spyglass_plugin_xls/examples/Site-Information.csv
new file mode 100644
index 0000000..41accd6
--- /dev/null
+++ b/spyglass_plugin_xls/examples/Site-Information.csv
@@ -0,0 +1,63 @@
+Site-Information,,,,
+IPMI Host Details,,,,
+,Server Name,IPMI Address,IPMI Gateway,Host Profile
+,cab2r72c12,10.0.220.138,10.0.220.129,dp-r720
+,cab2r72c13,10.0.220.139,10.0.220.129,dp-r720
+,cab2r72c14,10.0.220.140,10.0.220.129,dp-r720
+,cab2r72c15,10.0.220.141,10.0.220.129,dp-r720
+,cab2r72c16,10.0.220.142,10.0.220.129,cp-r720
+,cab2r72c17,10.0.220.143,10.0.220.129,cp-r720
+,cab2r73c12,10.0.220.170,10.0.220.161,dp-r720
+,cab2r73c13,10.0.220.171,10.0.220.161,dp-r720
+,cab2r73c14,10.0.220.172,10.0.220.161,dp-r720
+,cab2r73c15,10.0.220.173,10.0.220.161,dp-r720
+,cab2r73c16,10.0.220.174,10.0.220.161,cp-r720
+,cab2r73c17,10.0.220.175,10.0.220.161,cp-r720
+,,,,
+Network Details,,,,
+Usage,vLAN,,,
+iSCSI/Storage,vLAN 23,,,
+PXE,vLAN 21,,,
+Calico BGP peering addresses,vLAN 22,,,
+Overlay,vLAN 24,,,
+CNI Pod addresses,N/A,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+VLAN,POD,,,
+vLAN 23,30.31.1.0/25,,,
+vLAN 21,30.30.4.0/25,,,
+vLAN 21,30.30.4.128/25,,,
+vLAN 21,30.30.5.0/25,,,
+vLAN 21,30.30.5.128/25,,,
+,,,,
+vLAN 22,30.29.1.0/25,,,
+vLAN 24,30.19.0.0/25,,,
+,,,,
+VLAN,Rack1,Rack2,Rack3,Rack4
+VLAN-21,10.0.220.0/26,,,
+VLAN-49,10.0.220.64/29,,,
+Calico Ingress,10.0.220.72/29,,,
+Loopback 3 ,10.0.220.80/27,,,
+Spare,10.0.220.112-10.0.220.127,,,
+VLAN 1023,10.0.220.128/27,10.0.220.160/27,10.0.220.192/27,10.0.220.224/27
+SiteDetails,,,,
+,,,,
+Domain,dmy00.example.com,,,
+Subdomain,testitservices,,,
+Group,AA-AAA-dmy00,,,
+ldap_url,url: ldap://ldap.example.com,,,
+NTP,150.234.210.5 (ns1.example.com),,,
+DNS,"40.40.40.40 (ntp1.example.com),
+41.41.41.41 (ntp2.example.com)",,,
+,,,,
+Sitename,SampleSiteName,,,
+Corridor,Corridor 1,,,
+State,New Jersey,,,
+Country,SampleCountry,,,
+Clli,XXXXXX21,,,
diff --git a/spyglass_plugin_xls/excel_parser.py b/spyglass_plugin_xls/excel_parser.py
index 045af24..5bbe932 100644
--- a/spyglass_plugin_xls/excel_parser.py
+++ b/spyglass_plugin_xls/excel_parser.py
@@ -13,7 +13,9 @@
# limitations under the License.
from copy import deepcopy
+import csv
import logging
+import os
import pprint
import re
@@ -316,10 +318,19 @@ class ExcelParser(object):
@staticmethod
def load_excel_data(filename):
- """Combines multiple excel file to a single design spec"""
+ """Combines multiple excel or csv files to a single design spec"""
design_spec = Workbook()
- loaded_workbook = load_workbook(filename, data_only=True)
+ if os.path.splitext(filename)[1] == '.csv':
+ loaded_workbook = Workbook()
+ ws = loaded_workbook.active
+ with open(filename) as f:
+ reader = csv.reader(f, delimiter=',')
+ for row in reader:
+ ws.append(row)
+ ws.title = os.path.splitext(os.path.basename(filename))[0]
+ else:
+ loaded_workbook = load_workbook(filename, data_only=True)
for names in loaded_workbook.sheetnames:
design_spec_worksheet = design_spec.create_sheet(names)
loaded_workbook_ws = loaded_workbook[names]
@@ -334,11 +345,21 @@ class ExcelParser(object):
'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs'
"""
-
- if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
+ file_type = os.path.splitext(sheetname.split(':')[0])[1]
+ if file_type == '.xlsx' or file_type == '.xls':
# Extract file name
source_xl_file = sheetname.split(":")[0]
wb = load_workbook(source_xl_file, data_only=True)
return [wb, sheetname.split(":")[1]]
+ elif file_type == ".csv":
+ source_csv_file = sheetname.split(":")[0]
+ wb = Workbook()
+ ws = wb.active
+ with open(source_csv_file) as f:
+ reader = csv.reader(f, delimiter=',')
+ for row in reader:
+ ws.append(row)
+ ws.title = os.path.splitext(os.path.basename(source_csv_file))[0]
+ return [wb, sheetname.split(":")[1]]
else:
return [None, sheetname]
diff --git a/tests/shared/Site-Information.csv b/tests/shared/Site-Information.csv
new file mode 100644
index 0000000..41accd6
--- /dev/null
+++ b/tests/shared/Site-Information.csv
@@ -0,0 +1,63 @@
+Site-Information,,,,
+IPMI Host Details,,,,
+,Server Name,IPMI Address,IPMI Gateway,Host Profile
+,cab2r72c12,10.0.220.138,10.0.220.129,dp-r720
+,cab2r72c13,10.0.220.139,10.0.220.129,dp-r720
+,cab2r72c14,10.0.220.140,10.0.220.129,dp-r720
+,cab2r72c15,10.0.220.141,10.0.220.129,dp-r720
+,cab2r72c16,10.0.220.142,10.0.220.129,cp-r720
+,cab2r72c17,10.0.220.143,10.0.220.129,cp-r720
+,cab2r73c12,10.0.220.170,10.0.220.161,dp-r720
+,cab2r73c13,10.0.220.171,10.0.220.161,dp-r720
+,cab2r73c14,10.0.220.172,10.0.220.161,dp-r720
+,cab2r73c15,10.0.220.173,10.0.220.161,dp-r720
+,cab2r73c16,10.0.220.174,10.0.220.161,cp-r720
+,cab2r73c17,10.0.220.175,10.0.220.161,cp-r720
+,,,,
+Network Details,,,,
+Usage,vLAN,,,
+iSCSI/Storage,vLAN 23,,,
+PXE,vLAN 21,,,
+Calico BGP peering addresses,vLAN 22,,,
+Overlay,vLAN 24,,,
+CNI Pod addresses,N/A,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+,,,,
+VLAN,POD,,,
+vLAN 23,30.31.1.0/25,,,
+vLAN 21,30.30.4.0/25,,,
+vLAN 21,30.30.4.128/25,,,
+vLAN 21,30.30.5.0/25,,,
+vLAN 21,30.30.5.128/25,,,
+,,,,
+vLAN 22,30.29.1.0/25,,,
+vLAN 24,30.19.0.0/25,,,
+,,,,
+VLAN,Rack1,Rack2,Rack3,Rack4
+VLAN-21,10.0.220.0/26,,,
+VLAN-49,10.0.220.64/29,,,
+Calico Ingress,10.0.220.72/29,,,
+Loopback 3 ,10.0.220.80/27,,,
+Spare,10.0.220.112-10.0.220.127,,,
+VLAN 1023,10.0.220.128/27,10.0.220.160/27,10.0.220.192/27,10.0.220.224/27
+SiteDetails,,,,
+,,,,
+Domain,dmy00.example.com,,,
+Subdomain,testitservices,,,
+Group,AA-AAA-dmy00,,,
+ldap_url,url: ldap://ldap.example.com,,,
+NTP,150.234.210.5 (ns1.example.com),,,
+DNS,"40.40.40.40 (ntp1.example.com),
+41.41.41.41 (ntp2.example.com)",,,
+,,,,
+Sitename,SampleSiteName,,,
+Corridor,Corridor 1,,,
+State,New Jersey,,,
+Country,SampleCountry,,,
+Clli,XXXXXX21,,,
diff --git a/tests/unit/test_excel_parser.py b/tests/unit/test_excel_parser.py
index 5088c23..51e68ca 100644
--- a/tests/unit/test_excel_parser.py
+++ b/tests/unit/test_excel_parser.py
@@ -32,6 +32,8 @@ INVALID_EXCEL_SPEC_PATH = os.path.join(FIXTURE_DIR, 'invalid_excel_spec.yaml')
EXCEL_FILE_PATH = os.path.join(FIXTURE_DIR, 'SiteDesignSpec_v0.1.xlsx')
+CSV_FILE_PATH = os.path.join(FIXTURE_DIR, "Site-Information.csv")
+
SITE_CONFIG_PATH = os.path.join(FIXTURE_DIR, 'site_config.yaml')
@@ -55,6 +57,18 @@ class TestExcelParser(unittest.TestCase):
self.assertIsInstance(result.wb_combined, Workbook)
self.assertEqual('xl_spec', result.spec)
+ def test___init__csv(self):
+ with open(EXCEL_SPEC_PATH, 'r') as f:
+ loaded_spec = yaml.safe_load(f)
+ result = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+ self.assertEqual(CSV_FILE_PATH, result.file_name)
+ self.assertDictEqual(loaded_spec, result.excel_specs)
+ self.assertDictEqual(
+ loaded_spec['specs'][self.SPEC], result.loaded_spec)
+ self.assertDictEqual(self.raw_excel_data, result.loaded_data)
+ self.assertIsInstance(result.wb_combined, Workbook)
+ self.assertEqual('xl_spec', result.spec)
+
def test_sanitize(self):
test_string = 'Hello THIS is A TeSt'
expected_output = 'hellothisisatest'
@@ -68,6 +82,13 @@ class TestExcelParser(unittest.TestCase):
result = obj.compare(test_string1, test_string2)
self.assertTrue(result)
+ def test_compare_csv(self):
+ test_string1 = 'These strings are equal.'
+ test_string2 = 'These strIngs are Equal .'
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.compare(test_string1, test_string2)
+ self.assertTrue(result)
+
def test_compare_false(self):
test_string1 = 'These strings are not equal.'
test_string2 = 'These strIngs are Equal.'
@@ -75,11 +96,23 @@ class TestExcelParser(unittest.TestCase):
result = obj.compare(test_string1, test_string2)
self.assertFalse(result)
+ def test_compare_false_csv(self):
+ test_string1 = 'These strings are not equal.'
+ test_string2 = 'These strIngs are Equal.'
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.compare(test_string1, test_string2)
+ self.assertFalse(result)
+
def test__get_workbook(self):
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj._get_workbook('Site-Information')
self.assertIsInstance(result, Worksheet)
+ def test__get_workbook_csv(self):
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj._get_workbook('Site-Information')
+ self.assertIsInstance(result, Worksheet)
+
def test__check_sanitize_settings(self):
test_data_sanitize_only = {'sanitize': False}
sanitize, no_sanitize_keys = ExcelParser._check_sanitize_settings(
@@ -102,30 +135,61 @@ class TestExcelParser(unittest.TestCase):
obj.loaded_spec['public']['data']['oam'], 'Site-Information')
self.assertDictEqual(expected_data, result)
+ def test_extract_data_points_csv(self):
+ expected_data = self.raw_excel_data['public']['oam']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+ result = obj.extract_data_points(
+ obj.loaded_spec['public']['data']['oam'], 'Site-Information')
+ self.assertDictEqual(expected_data, result)
+
def test_extract_data_points_unsanitized(self):
expected_data = self.raw_excel_data['location']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
result = obj.extract_data_points(obj.loaded_spec['location'])
self.assertDictEqual(expected_data, result)
+ def test_extract_data_points_unsanitized_csv(self):
+ expected_data = self.raw_excel_data['location']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+ result = obj.extract_data_points(obj.loaded_spec['location'])
+ self.assertDictEqual(expected_data, result)
+
def test_extract_data_series(self):
expected_data = self.raw_excel_data['ipmi']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
result = obj.extract_data_series(obj.loaded_spec['ipmi'])
self.assertEqual(expected_data, result)
+ def test_extract_data_series_csv(self):
+ expected_data = self.raw_excel_data['ipmi']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+ result = obj.extract_data_series(obj.loaded_spec['ipmi'])
+ self.assertEqual(expected_data, result)
+
def test_extract_data_series_no_sanitize(self):
expected_data = self.raw_excel_data['private_vlan']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
result = obj.extract_data_series(obj.loaded_spec['private_vlan'])
self.assertEqual(expected_data, result)
+ def test_extract_data_series_no_sanitize_csv(self):
+ expected_data = self.raw_excel_data['private_vlan']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+ result = obj.extract_data_series(obj.loaded_spec['private_vlan'])
+ self.assertEqual(expected_data, result)
+
def test_extract_data_using_spec(self):
expected_data = self.raw_excel_data
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
result = obj.extract_data_using_spec(obj.loaded_spec)
self.assertDictEqual(expected_data, result)
+ def test_extract_data_using_spec_csv(self):
+ expected_data = self.raw_excel_data
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH, self.SPEC)
+ result = obj.extract_data_using_spec(obj.loaded_spec)
+ self.assertDictEqual(expected_data, result)
+
def test_get_ipmi_data(self):
expected_hosts = self.site_data['ipmi_data'][1]
expected_ipmi_data = self.site_data['ipmi_data'][0]
@@ -134,6 +198,14 @@ class TestExcelParser(unittest.TestCase):
self.assertDictEqual(result[0], expected_ipmi_data)
self.assertEqual(result[1], expected_hosts)
+ def test_get_ipmi_data_csv(self):
+ expected_hosts = self.site_data['ipmi_data'][1]
+ expected_ipmi_data = self.site_data['ipmi_data'][0]
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_ipmi_data()
+ self.assertDictEqual(result[0], expected_ipmi_data)
+ self.assertEqual(result[1], expected_hosts)
+
def test_get_private_vlan_data(self):
expected_vlan_data = {
'vlan23': 'iSCSI/Storage',
@@ -146,48 +218,102 @@ class TestExcelParser(unittest.TestCase):
result = obj.get_private_vlan_data()
self.assertDictEqual(expected_vlan_data, result)
+ def test_get_private_vlan_data_csv(self):
+ expected_vlan_data = {
+ 'vlan23': 'iSCSI/Storage',
+ 'vlan21': 'PXE',
+ 'vlan22': 'Calico BGP peering addresses',
+ 'vlan24': 'Overlay',
+ 'na': 'CNI Pod addresses'
+ }
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_private_vlan_data()
+ self.assertDictEqual(expected_vlan_data, result)
+
def test_get_private_network_data(self):
expected_network_data = self.site_data['network_data']['private']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_private_network_data()
self.assertDictEqual(expected_network_data, result)
+ def test_get_private_network_data_csv(self):
+ expected_network_data = self.site_data['network_data']['private']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_private_network_data()
+ self.assertDictEqual(expected_network_data, result)
+
def test_get_public_network_data(self):
expected_network_data = self.site_data['network_data']['public']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_public_network_data()
self.assertEqual(expected_network_data, result)
+ def test_get_public_network_data_csv(self):
+ expected_network_data = self.site_data['network_data']['public']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_public_network_data()
+ self.assertEqual(expected_network_data, result)
+
def test_get_site_info(self):
expected_site_info = self.site_data['site_info']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_site_info()
self.assertDictEqual(expected_site_info, result)
+ def test_get_site_info_csv(self):
+ expected_site_info = self.site_data['site_info']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_site_info()
+ self.assertDictEqual(expected_site_info, result)
+
def test_get_location_data(self):
expected_location_data = self.site_data['site_info']['location']
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_location_data()
self.assertEqual(expected_location_data, result)
+ def test_get_location_data_csv(self):
+ expected_location_data = self.site_data['site_info']['location']
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_location_data()
+ self.assertEqual(expected_location_data, result)
+
def test_validate_sheet_names_with_spec(self):
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
self.assertIsNone(obj.validate_sheet_names_with_spec())
+ def test_validate_sheet_names_with_spec_csv(self):
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ self.assertIsNone(obj.validate_sheet_names_with_spec())
+
def test_validate_sheet_names_with_spec_invalid(self):
with self.assertRaises(ExcelSheetNotFound):
ExcelParser(EXCEL_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
+ def test_validate_sheet_names_with_spec_invalid_csv(self):
+ with self.assertRaises(ExcelSheetNotFound):
+ ExcelParser(CSV_FILE_PATH, INVALID_EXCEL_SPEC_PATH)
+
def test_get_data(self):
expected_data = self.site_data
obj = ExcelParser(EXCEL_FILE_PATH, EXCEL_SPEC_PATH)
result = obj.get_data()
self.assertDictEqual(expected_data, result)
+ def test_get_data_csv(self):
+ expected_data = self.site_data
+ obj = ExcelParser(CSV_FILE_PATH, EXCEL_SPEC_PATH)
+ result = obj.get_data()
+ self.assertDictEqual(expected_data, result)
+
def test_load_excel_data(self):
result = ExcelParser.load_excel_data(EXCEL_FILE_PATH)
self.assertIsInstance(result, Workbook)
+ def test_load_csv_data(self):
+ result = ExcelParser.load_excel_data(CSV_FILE_PATH)
+ self.assertIsInstance(result, Workbook)
+
def test_get_xl_obj_and_sheetname(self):
result = ExcelParser.get_xl_obj_and_sheetname('Site-Information')
self.assertEqual([None, 'Site-Information'], result)
@@ -198,3 +324,10 @@ class TestExcelParser(unittest.TestCase):
self.assertIsInstance(result, list)
self.assertIsInstance(result[0], Workbook)
self.assertEqual(result[1], 'Site-Information')
+
+ def test_get_xl_obj_and_sheetname_file_specified_csv(self):
+ sheet = CSV_FILE_PATH + ':Site-Information'
+ result = ExcelParser.get_xl_obj_and_sheetname(sheet)
+ self.assertIsInstance(result, list)
+ self.assertIsInstance(result[0], Workbook)
+ self.assertEqual(result[1], 'Site-Information')