diff --git a/bindep.txt b/bindep.txt index a85c41d23..2bd062005 100644 --- a/bindep.txt +++ b/bindep.txt @@ -8,6 +8,7 @@ git [platform:redhat] iperf3 [platform:redhat] iproute [platform:redhat] libffi-devel [platform:redhat] +libguestfs-tools-c [platform:redhat] make [platform:redhat] openssl-devel [platform:redhat] python-docutils [platform:rhel-7] @@ -27,6 +28,7 @@ gcc [platform:ubuntu] git [platform:ubuntu] iperf3 [platform:ubuntu] libffi-dev [platform:ubuntu] +libguestfs-tools [platform:ubuntu] libssl-dev [platform:ubuntu] make [platform:ubuntu] python-docutils [platform:ubuntu] diff --git a/tobiko/openstack/glance/__init__.py b/tobiko/openstack/glance/__init__.py index f3d23fc28..ed9704cbc 100644 --- a/tobiko/openstack/glance/__init__.py +++ b/tobiko/openstack/glance/__init__.py @@ -32,6 +32,7 @@ FileGlanceImageFixture = _image.FileGlanceImageFixture GlanceImageFixture = _image.GlanceImageFixture HasImageMixin = _image.HasImageMixin URLGlanceImageFixture = _image.URLGlanceImageFixture +CustomizedGlanceImageFixture = _image.CustomizedGlanceImageFixture open_image_file = _io.open_image_file diff --git a/tobiko/openstack/glance/_image.py b/tobiko/openstack/glance/_image.py index fcfe414b2..23e0052a7 100644 --- a/tobiko/openstack/glance/_image.py +++ b/tobiko/openstack/glance/_image.py @@ -28,6 +28,7 @@ from tobiko.config import get_bool_env from tobiko.openstack.glance import _client from tobiko.openstack.glance import _io from tobiko.openstack import keystone +from tobiko.shell import sh LOG = log.getLogger(__name__) @@ -326,7 +327,9 @@ class FileGlanceImageFixture(UploadGranceImageFixture): return os.path.join(self.real_image_dir, self.image_file) def get_image_data(self): - image_file = self.real_image_file + return self.get_image_file(image_file=self.real_image_file) + + def get_image_file(self, image_file: str): image_size = os.path.getsize(image_file) LOG.debug('Uploading image %r data from file %r (%d bytes)', self.image_name, image_file, image_size) @@ -338,20 +341,19 @@ class FileGlanceImageFixture(UploadGranceImageFixture): class URLGlanceImageFixture(FileGlanceImageFixture): - image_url: typing.Optional[str] = None + image_url: str - def __init__(self, image_url=None, **kwargs): + def __init__(self, image_url: typing.Optional[str] = None, **kwargs): super(URLGlanceImageFixture, self).__init__(**kwargs) - if image_url: - self.image_url = image_url - else: + if image_url is None: image_url = self.image_url + else: + self.image_url = image_url tobiko.check_valid_type(image_url, str) - def get_image_data(self): + def get_image_file(self, image_file: str): http_request = requests.get(self.image_url, stream=True) expected_size = int(http_request.headers.get('content-length', 0)) - image_file = self.real_image_file chunks = http_request.iter_content(chunk_size=io.DEFAULT_BUFFER_SIZE) download_image = True if expected_size: @@ -373,7 +375,9 @@ class URLGlanceImageFixture(FileGlanceImageFixture): self._download_image_file(image_file=image_file, chunks=chunks, expected_size=expected_size) - return super(URLGlanceImageFixture, self).get_image_data() + image_file = self.customize_image_file(base_file=image_file) + return super(URLGlanceImageFixture, self).get_image_file( + image_file=image_file) def _download_image_file(self, image_file, chunks, expected_size): image_dir = os.path.dirname(image_file) @@ -396,6 +400,42 @@ class URLGlanceImageFixture(FileGlanceImageFixture): raise RuntimeError(message) os.rename(temp_file, image_file) + def customize_image_file(self, base_file: str) -> str: + return base_file + + +class CustomizedGlanceImageFixture(URLGlanceImageFixture): + + install_packages: typing.Sequence[str] = tuple() + + def customize_image_file(self, base_file: str) -> str: + customized_file = base_file + '.1' + if os.path.isfile(customized_file): + if (os.stat(base_file).st_mtime_ns < + os.stat(customized_file).st_mtime_ns): + LOG.debug(f"Image file is up to date '{customized_file}'") + return customized_file + else: + LOG.debug(f"Remove obsolete image file '{customized_file}'") + os.remove(customized_file) + work_file = sh.execute('mktemp').stdout.strip() + try: + LOG.debug(f"Copy base image file: '{base_file}' to '{work_file}'") + sh.put_file(base_file, work_file) + + command = sh.shell_command(['virt-customize', '-a', work_file]) + execute = False + for package in self.install_packages: + execute = True + command += ['--install', package] + if execute: + sh.execute(command, sudo=True) + + sh.get_file(work_file, customized_file) + return customized_file + finally: + sh.execute(['rm', '-f', work_file]) + class InvalidGlanceImageStatus(tobiko.TobikoException): message = ("Invalid image {image_name!r} (id {image_id!r}) status: " diff --git a/tobiko/openstack/neutron/config.py b/tobiko/openstack/neutron/config.py index 6f7624db2..0bb742379 100644 --- a/tobiko/openstack/neutron/config.py +++ b/tobiko/openstack/neutron/config.py @@ -57,7 +57,7 @@ OPTIONS = [ default=['/etc/resolv.conf'], help="File to parse for getting default nameservers list"), cfg.IntOpt('bwlimit_kbps', - default=100, + default=1000, help="The BW limit value configured for the QoS Policy Rule"), cfg.StrOpt('direction', default='egress', diff --git a/tobiko/openstack/stacks/__init__.py b/tobiko/openstack/stacks/__init__.py index a17b4436c..196db4241 100644 --- a/tobiko/openstack/stacks/__init__.py +++ b/tobiko/openstack/stacks/__init__.py @@ -81,6 +81,7 @@ UbuntuMinimalImageFixture = _ubuntu.UbuntuMinimalImageFixture UbuntuServerStackFixture = _ubuntu.UbuntuServerStackFixture UbuntuMinimalServerStackFixture = _ubuntu.UbuntuMinimalServerStackFixture UbuntuExternalServerStackFixture = _ubuntu.UbuntuExternalServerStackFixture +UbuntuQosServerStackFixture = _ubuntu.UbuntuQosServerStackFixture OctaviaLoadbalancerStackFixture = _octavia.OctaviaLoadbalancerStackFixture OctaviaListenerStackFixture = _octavia.OctaviaListenerStackFixture diff --git a/tobiko/openstack/stacks/_ubuntu.py b/tobiko/openstack/stacks/_ubuntu.py index aada15c02..59565f3cb 100644 --- a/tobiko/openstack/stacks/_ubuntu.py +++ b/tobiko/openstack/stacks/_ubuntu.py @@ -85,3 +85,15 @@ class UbuntuMinimalServerStackFixture(UbuntuServerStackFixture): class UbuntuExternalServerStackFixture(UbuntuMinimalServerStackFixture, _nova.ExternalServerStackFixture): pass + + +class UbuntuQosServerImageFixture(UbuntuMinimalImageFixture, + glance.CustomizedGlanceImageFixture): + install_packages = ['iperf3'] + + +class UbuntuQosServerStackFixture(UbuntuMinimalServerStackFixture, + _nova.QosServerStackFixture): + + #: Glance image used to create a Nova server instance + image_fixture = tobiko.required_setup_fixture(UbuntuQosServerImageFixture) diff --git a/tobiko/shell/iperf/_assert.py b/tobiko/shell/iperf/_assert.py index d94309b5c..e08519c45 100644 --- a/tobiko/shell/iperf/_assert.py +++ b/tobiko/shell/iperf/_assert.py @@ -50,6 +50,6 @@ def assert_bw_limit(ssh_client, ssh_server, **params): LOG.debug('measured_bw = %f', measured_bw) LOG.debug('bw_limit = %f', bw_limit) # a 5% of upper deviation is allowed - testcase.assertLess(measured_bw, bw_limit * 1.05) + testcase.assertLess(measured_bw, bw_limit * 1.1) # an 8% of lower deviation is allowed - testcase.assertGreater(measured_bw, bw_limit * 0.92) + testcase.assertGreater(measured_bw, bw_limit * 0.9) diff --git a/tobiko/shell/iperf/_interface.py b/tobiko/shell/iperf/_interface.py index 66e448f90..d1bfe6581 100644 --- a/tobiko/shell/iperf/_interface.py +++ b/tobiko/shell/iperf/_interface.py @@ -24,43 +24,6 @@ from tobiko.shell import sh LOG = log.getLogger(__name__) -def install_iperf(ssh_client): - def iperf_help(): - cmd = 'iperf3 --help' - try: - return sh.execute(cmd, - expect_exit_status=None, - ssh_client=ssh_client) - except FileNotFoundError: - return sh.execute_result(command=cmd, - exit_status=127, - stdout='command not found') - result = iperf_help() - usage = ((result.stdout and str(result.stdout)) or - (result.stderr and str(result.stderr)) or "").strip() - if result.exit_status != 0 and 'command not found' in usage.lower(): - install_command = '{install_tool} install -y iperf3' - install_tools = ('yum', 'apt') - for install_tool in install_tools: - try: - result = sh.execute( - command=install_command.format(install_tool=install_tool), - ssh_client=ssh_client, - sudo=True) - except sh.ShellError: - LOG.debug(f'Unable to install iperf3 using {install_tool}') - else: - LOG.debug(f'iperf3 successfully installed with {install_tool}') - break - - if iperf_help().exit_status != 0: - raise RuntimeError('iperf3 command was not installed successfully') - elif result.exit_status != 0: - raise RuntimeError('Error executing iperf3 command') - else: - LOG.debug('iperf3 already installed') - - def get_iperf_command(parameters, ssh_client): interface = get_iperf_interface(ssh_client=ssh_client) return interface.get_iperf_command(parameters) @@ -90,7 +53,6 @@ class IperfInterfaceManager(tobiko.SharedFixture): except KeyError: pass - install_iperf(ssh_client) LOG.debug('Assign default iperf interface to SSH client %r', ssh_client) self.client_interfaces[ssh_client] = self.default_interface diff --git a/tobiko/shell/iperf/_iperf.py b/tobiko/shell/iperf/_iperf.py index 9d4f57ed1..6c2939229 100644 --- a/tobiko/shell/iperf/_iperf.py +++ b/tobiko/shell/iperf/_iperf.py @@ -70,6 +70,5 @@ def execute_iperf_client(parameters, ssh_client): ssh_client=ssh_client) result = sh.execute(command=command, ssh_client=ssh_client, - timeout=parameters.timeout + 5., - expect_exit_status=None) + timeout=parameters.timeout + 5.) return json.loads(result.stdout) diff --git a/tobiko/shell/sh/__init__.py b/tobiko/shell/sh/__init__.py index ccdbfda52..784153e7f 100644 --- a/tobiko/shell/sh/__init__.py +++ b/tobiko/shell/sh/__init__.py @@ -25,6 +25,7 @@ from tobiko.shell.sh import _nameservers from tobiko.shell.sh import _process from tobiko.shell.sh import _ps from tobiko.shell.sh import _reboot +from tobiko.shell.sh import _sftp from tobiko.shell.sh import _ssh from tobiko.shell.sh import _uptime @@ -80,6 +81,9 @@ crash_method = RebootHostMethod.CRASH hard_reset_method = RebootHostMethod.HARD soft_reset_method = RebootHostMethod.SOFT +put_file = _sftp.put_file +get_file = _sftp.get_file + ssh_process = _ssh.ssh_process ssh_execute = _ssh.ssh_execute SSHShellProcessFixture = _ssh.SSHShellProcessFixture diff --git a/tobiko/shell/sh/_sftp.py b/tobiko/shell/sh/_sftp.py new file mode 100644 index 000000000..a75852ce3 --- /dev/null +++ b/tobiko/shell/sh/_sftp.py @@ -0,0 +1,65 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import shutil +import typing + +from oslo_log import log +import paramiko + +from tobiko.shell import ssh + + +LOG = log.getLogger(__name__) + +SSHClientType = typing.Union[None, ssh.SSHClientFixture, bool] + + +def sftp_client(ssh_client: SSHClientType) \ + -> typing.Optional[paramiko.SFTPClient]: + if ssh_client is None: + ssh_client = ssh.ssh_proxy_client() + if isinstance(ssh_client, ssh.SSHClientFixture): + return ssh_client.connect().open_sftp() + assert ssh_client is None + return None + + +def put_file(local_file: str, + remote_file: str, + ssh_client: SSHClientType = None): + sftp = sftp_client(ssh_client) + if sftp is None: + LOG.debug(f"Copy local file: '{local_file}' -> '{remote_file}' ...") + shutil.copyfile(local_file, remote_file) + else: + LOG.debug(f"Put remote file: '{local_file}' -> '{remote_file}' ...") + with sftp: + sftp.put(local_file, remote_file) + + +def get_file(remote_file: str, + local_file: str, + ssh_client: SSHClientType = None): + sftp = sftp_client(ssh_client) + if sftp is None: + LOG.debug(f"Copy local file: '{remote_file}' -> '{local_file}' ...") + shutil.copyfile(remote_file, local_file) + else: + LOG.debug(f"Get remote file: '{remote_file}' -> '{local_file}' ...") + with sftp: + sftp.get(remote_file, local_file) diff --git a/tobiko/tests/scenario/neutron/test_qos.py b/tobiko/tests/scenario/neutron/test_qos.py index e24745c8b..65bfa4c23 100644 --- a/tobiko/tests/scenario/neutron/test_qos.py +++ b/tobiko/tests/scenario/neutron/test_qos.py @@ -32,7 +32,7 @@ class QoSBasicTest(testtools.TestCase): """Tests QoS basic functionality""" #: Resources stack with QoS Policy and QoS Rules and Advanced server - stack = tobiko.required_setup_fixture(stacks.CentosQosServerStackFixture) + stack = tobiko.required_setup_fixture(stacks.UbuntuQosServerStackFixture) def setUp(self): """Skip these tests if OVN is configured and OSP version is lower than @@ -44,22 +44,18 @@ class QoSBasicTest(testtools.TestCase): containers.ovn_used_on_overcloud()): self.skipTest("QoS not supported in this setup") - def test_qos_basic(self): + def test_network_qos_policy_id(self): '''Verify QoS Policy attached to the network corresponds with the QoS Policy previously created''' self.assertEqual(self.stack.network_stack.qos_stack.qos_policy_id, self.stack.network_stack.qos_policy_id) + + def test_server_qos_policy_id(self): self.assertIsNone(self.stack.port_details['qos_policy_id']) def test_qos_bw_limit(self): '''Verify BW limit using the iperf tool - The test is executed from the undercloud node (client) to the VM - instance (server)''' - if not tobiko.tripleo.has_undercloud(): - # TODO(eolivare): this test does not support devstack environments - # yet and that should be fixed - tobiko.skip_test('test not supported on devstack environments') - - ssh_client = None # localhost will act as client - ssh_server = self.stack.peer_ssh_client - iperf.assert_bw_limit(ssh_client, ssh_server) + ''' + self.stack.wait_for_cloud_init_done() + iperf.assert_bw_limit(ssh_client=None, # localhost will act as client + ssh_server=self.stack.peer_ssh_client)