From bc3b4e08af09096395559fec03b6f15bc292b029 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel Date: Thu, 26 Mar 2015 15:59:38 +0100 Subject: [PATCH] Allow to upload all data from nodes.csv file There is possibility to upload CSV file, but only partial data is fetched from it. Changed it to fetch everything what should be included in form for adding new nodes. Change-Id: Ic71d10c1911e034dd213212e74c1aa0d7a8bd252 --- doc/source/index.rst | 1 + doc/source/user_guide.rst | 16 +++++ tuskar_ui/infrastructure/nodes/forms.py | 40 ++---------- tuskar_ui/utils/tests.py | 87 +++++++++++++++++++++++++ tuskar_ui/utils/utils.py | 46 +++++++++++++ 5 files changed, 157 insertions(+), 33 deletions(-) create mode 100644 doc/source/user_guide.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index 809c3dfe9..d6176cb05 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -8,6 +8,7 @@ Contents: README install + user_guide HACKING Indices and tables diff --git a/doc/source/user_guide.rst b/doc/source/user_guide.rst new file mode 100644 index 000000000..782c9e050 --- /dev/null +++ b/doc/source/user_guide.rst @@ -0,0 +1,16 @@ +========== +User Guide +========== + +Nodes List File +--------------- + +To allow users to load a bunch of nodes at once, there is possibility to +upload CSV file with given list of nodes. This file should be formatted as + +:: + + driver,address,username,password/ssh key,mac addresses,cpu architecture,number of CPUs,available memory,available storage + +Even if there is no all data available, we assume empty values for missing +keys and try to parse everything, what is possible. diff --git a/tuskar_ui/infrastructure/nodes/forms.py b/tuskar_ui/infrastructure/nodes/forms.py index 12d365cd1..6ec6b6e80 100644 --- a/tuskar_ui/infrastructure/nodes/forms.py +++ b/tuskar_ui/infrastructure/nodes/forms.py @@ -11,8 +11,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import csv - import django.forms from django.utils.translation import ugettext_lazy as _ from horizon import exceptions @@ -21,6 +19,7 @@ from horizon import messages from tuskar_ui import api import tuskar_ui.forms +from tuskar_ui.utils import utils DEFAULT_KERNEL_IMAGE_NAME = 'bm-deploy-kernel' @@ -307,38 +306,13 @@ class UploadNodeForm(forms.SelfHandlingForm): return True def get_data(self): - data = [] + try: + output = utils.parse_csv_file(self.cleaned_data['csv_file']) + except ValueError as e: + messages.error(self.request, e.message) + output = [] - for row in csv.reader(self.cleaned_data['csv_file']): - try: - driver = row[0].strip() - except IndexError: - messages.error(self.request, - _("Unable to parse the CSV file.")) - return [] - - if driver == 'pxe_ssh': - node = dict( - ssh_address=row[1], - ssh_username=row[2], - ssh_key_contents=row[3], - mac_addresses=row[4], - driver=driver, - ) - data.append(node) - elif driver == 'pxe_ipmitool': - node = dict( - ipmi_address=row[1], - ipmi_username=row[2], - ipmi_password=row[3], - driver=driver, - ) - data.append(node) - else: - messages.error(self.request, - _("Unknown driver: %s.") % driver) - return [] - return data + return output RegisterNodeFormset = django.forms.formsets.formset_factory( diff --git a/tuskar_ui/utils/tests.py b/tuskar_ui/utils/tests.py index 607296698..408f916c0 100644 --- a/tuskar_ui/utils/tests.py +++ b/tuskar_ui/utils/tests.py @@ -15,6 +15,7 @@ import collections import datetime +from django.utils.translation import ugettext_lazy as _ import mock from tuskar_ui.test import helpers @@ -77,6 +78,92 @@ class UtilsTests(helpers.TestCase): ret = utils.safe_int_cast(object()) self.assertEqual(ret, 0) + def test_parse_correct_csv_file(self): + correct_file = [ + 'pxe_ipmitool,ipmi_address,ipmi_username,ipmi_password,' + 'mac_addresses,cpu_arch,cpus,memory_mb,local_gb', + 'pxe_ipmitool,,,,MAC_ADDRESS,,CPUS,,LOCAL_GB', + 'pxe_ssh,ssh_address,ssh_username,ssh_key_contents,mac_addresses' + ',cpu_arch,cpus,memory_mb,local_gb', + 'pxe_ssh,SSH,USER,KEY', + 'pxe_ssh,SSH,USER,,,CPU_ARCH', + ] + + correct_data = utils.parse_csv_file(correct_file) + + self.assertSequenceEqual( + correct_data, [ + { + 'driver': 'pxe_ipmitool', + 'ipmi_address': 'ipmi_address', + 'ipmi_username': 'ipmi_username', + 'ipmi_password': 'ipmi_password', + 'mac_addresses': 'mac_addresses', + 'cpu_arch': 'cpu_arch', + 'cpus': 'cpus', + 'memory_mb': 'memory_mb', + 'local_gb': 'local_gb', + }, { + 'driver': 'pxe_ipmitool', + 'ipmi_address': '', + 'ipmi_username': '', + 'ipmi_password': '', + 'mac_addresses': 'MAC_ADDRESS', + 'cpu_arch': '', + 'cpus': 'CPUS', + 'memory_mb': '', + 'local_gb': 'LOCAL_GB', + }, { + 'driver': 'pxe_ssh', + 'ssh_address': 'ssh_address', + 'ssh_username': 'ssh_username', + 'ssh_key_contents': 'ssh_key_contents', + 'mac_addresses': 'mac_addresses', + 'cpu_arch': 'cpu_arch', + 'cpus': 'cpus', + 'memory_mb': 'memory_mb', + 'local_gb': 'local_gb', + }, + { + 'driver': 'pxe_ssh', + 'ssh_address': 'SSH', + 'ssh_username': 'USER', + 'ssh_key_contents': 'KEY', + }, + { + 'driver': 'pxe_ssh', + 'ssh_address': 'SSH', + 'ssh_username': 'USER', + 'ssh_key_contents': '', + 'mac_addresses': '', + 'cpu_arch': 'CPU_ARCH', + }, + ] + ) + + def test_parse_csv_file_wrong(self): + no_csv_file = [ + '', + 'File with first empty line -- it\'s not a CSV file.', + ] + + with self.assertRaises(ValueError) as raised: + utils.parse_csv_file(no_csv_file) + + self.assertEqual(unicode(raised.exception.message), + unicode(_("Unable to parse the CSV file."))) + + def test_parse_wrong_driver_file(self): + wrong_driver_file = [ + 'wrong_driver,ssh_address,ssh_user', + ] + + with self.assertRaises(ValueError) as raised: + utils.parse_csv_file(wrong_driver_file) + + self.assertEqual(unicode(raised.exception.message), + unicode(_("Unknown driver: %s.") % 'wrong_driver')) + class MeteringTests(helpers.TestCase): def test_query_data(self): diff --git a/tuskar_ui/utils/utils.py b/tuskar_ui/utils/utils.py index a75eb24c9..f99d9fdad 100644 --- a/tuskar_ui/utils/utils.py +++ b/tuskar_ui/utils/utils.py @@ -11,8 +11,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import csv +from itertools import izip import re +from django.utils.translation import ugettext_lazy as _ + CAMEL_RE = re.compile(r'([A-Z][a-z]+|[A-Z]+(?=[A-Z\s]|$))') @@ -92,3 +96,45 @@ def safe_int_cast(value): return int(value) except (TypeError, ValueError): return 0 + + +def parse_csv_file(csv_file): + """Parses given CSV file. + + If there is no error, it returns list of dicts. When something went wrong, + list is empty, but warning contains appropriate information about + possible problems. + """ + + parsed_data = [] + + for row in csv.reader(csv_file): + try: + driver = row[0].strip() + except IndexError: + raise ValueError(_("Unable to parse the CSV file.")) + + if driver in ('pxe_ssh', 'pxe_ipmitool'): + node_keys = ( + 'mac_addresses', 'cpu_arch', 'cpus', 'memory_mb', 'local_gb') + + if driver == 'pxe_ssh': + driver_keys = ( + 'driver', 'ssh_address', 'ssh_username', + 'ssh_key_contents' + ) + + elif driver == 'pxe_ipmitool': + driver_keys = ( + 'driver', 'ipmi_address', 'ipmi_username', + 'ipmi_password' + ) + + node = dict(izip(driver_keys+node_keys, row)) + + parsed_data.append(node) + + else: + raise ValueError(_("Unknown driver: %s.") % driver) + + return parsed_data