From 1c3b60a8c850a5940d72ce99641591cac4b53c28 Mon Sep 17 00:00:00 2001 From: akrzos Date: Tue, 28 Nov 2017 14:10:48 -0500 Subject: [PATCH] Mix and Match Workloads * Mix and Match Workloads * rerun_type - ['iteration', 'complete'] - allows you to rerun complete browbeat workload or iteratively * browbeat/config.py for loading and validating config schema * tests/test_config.py to test config.py Change-Id: I99ea87c01c17d5d06cf7f8d1eec3299aa8d483a5 --- .gitignore | 6 + .../templates/browbeat-api-ci.yaml.j2 | 236 +++--- .../templates/browbeat-basic.yaml.j2 | 181 +++-- .../templates/browbeat-yoda-ci.yaml.j2 | 116 +-- .../templates/browbeat-yoda-scale.yaml.j2 | 334 +++++---- browbeat-complete.yaml | 690 +++++++++--------- browbeat-config.yaml | 457 ++++++------ browbeat.py | 159 ++-- browbeat/config.py | 57 ++ browbeat/grafana.py | 6 +- browbeat/path.py | 2 + browbeat/perfkit.py | 105 ++- browbeat/rally.py | 250 +++---- browbeat/schema/browbeat.yml | 194 +++++ browbeat/schema/perfkit.yml | 48 ++ browbeat/schema/rally.yml | 38 + browbeat/schema/shaker.yml | 19 + browbeat/schema/yoda.yml | 71 ++ browbeat/shaker.py | 107 ++- browbeat/tools.py | 68 +- browbeat/validate.yaml | 331 --------- browbeat/workloadbase.py | 27 +- browbeat/yoda.py | 62 +- ci-scripts/config/browbeat-ci.yaml | 237 ------ conf/browbeat-glance-complete.yaml | 102 +-- conf/browbeat-keystone-complete.yaml | 404 +++++----- conf/browbeat-keystone-minimal.yaml | 390 +++++----- conf/browbeat-workloads.yml | 120 +-- conf/ceilometer-minimal.yaml | 316 ++++---- conf/ceilometer-persist-response-timing.yaml | 452 ++---------- conf/gnocchi-minimal.yaml | 236 +++--- conf/perfkit-benchmarks.yaml | 219 +++--- ...nova-neutron-gnocchi-10k-1000-per-30m.yaml | 436 ++--------- ...y-nova-neutron-gnocchi-1k-100-per-30m.yaml | 436 ++--------- ...y-nova-neutron-gnocchi-1k-500-per-30m.yaml | 172 ++--- ...y-nova-neutron-gnocchi-5k-500-per-30m.yaml | 436 ++--------- ...va-persist-instances-with-network-fip.yaml | 228 +++--- ...ist-instances-with-network-volume-fip.yaml | 248 ++++--- ...persist-instances-with-network-volume.yaml | 228 +++--- ...y-nova-persist-instances-with-network.yaml | 208 +++--- conf/telemetry-nova-persist-instances.yaml | 189 ++--- test-requirements.txt | 1 + tests/__init__.py | 0 tests/data/invalid_browbeat.yml | 89 +++ tests/data/invalid_browbeat_workload.yml | 68 ++ tests/data/valid_browbeat.yml | 88 +++ tests/data/workloads.yml | 87 +++ tests/test_config.py | 54 ++ tox.ini | 10 +- 49 files changed, 4180 insertions(+), 4838 deletions(-) create mode 100644 browbeat/config.py create mode 100644 browbeat/schema/browbeat.yml create mode 100644 browbeat/schema/perfkit.yml create mode 100644 browbeat/schema/rally.yml create mode 100644 browbeat/schema/shaker.yml create mode 100644 browbeat/schema/yoda.yml delete mode 100644 browbeat/validate.yaml delete mode 100644 ci-scripts/config/browbeat-ci.yaml create mode 100644 tests/__init__.py create mode 100644 tests/data/invalid_browbeat.yml create mode 100644 tests/data/invalid_browbeat_workload.yml create mode 100644 tests/data/valid_browbeat.yml create mode 100644 tests/data/workloads.yml create mode 100644 tests/test_config.py diff --git a/.gitignore b/.gitignore index fed58d862..0749548e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ *.py[cod] + +# Distribution / packaging +.eggs + +# Testing +.cache .tox # Virtual Environments: diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 index fab96a06a..c38c07814 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 @@ -1,7 +1,11 @@ browbeat: - results: results/ - rerun: 3 cloud_name: {{ browbeat_cloud_name }} + rerun: 3 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: {{ elastic_enabled }} host: {{ elastic_host }} @@ -16,116 +20,136 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: {{ grafana_enabled }} - grafana_ip: {{ grafana_host }} - grafana_port: 3000 + host: {{ grafana_host }} + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv plugins: - netcreate-boot: rally/rally-plugins/netcreate-boot - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 16 - - 24 - times: 2500 - scenarios: - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - - name: keystonebasic - enabled: true - concurrency: - - 16 - times: 1000 - scenarios: - - name: create-and-list-tenants - enabled: true - file: rally/keystonebasic/create_and_list_tenants-cc.yml - - name: neutron - enabled: true - concurrency: - - 8 - - 12 - times: 128 - scenarios: - - name: create-list-router - enabled: true - file: rally/neutron/neutron-create-list-router-cc.yml - - name: create-list-network - enabled: true - file: rally/neutron/neutron-create-list-network-cc.yml - - name: create-list-subnet - enabled: true - file: rally/neutron/neutron-create-list-subnet-cc.yml - - name: create-list-port - enabled: true - file: rally/neutron/neutron-create-list-port-cc.yml - - name: create-list-security-group - enabled: true - file: rally/neutron/neutron-create-list-security-group-cc.yml - - name: nova - enabled: true - concurrency: - - 8 - - 12 - times: 64 - scenarios: - - name: boot-snapshot-delete - enabled: true - file: rally/nova/nova-boot-snapshot-cc.yml - image_name: cirros - flavor_name: m1.tiny - - name: boot-list - enabled: true - file: rally/nova/nova-boot-list-cc.yml - image_name: cirros - flavor_name: m1.tiny - - name: glance - enabled: true - concurrency: - - 8 - - 12 - times: 64 - scenarios: - - name: create-and-delete-image - enabled: true - image_location: http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2.xz - flavor_name: m1.tiny - file: rally/glance/create-and-delete-image-cc.yml - - name: plugins - enabled: true - concurrency: - - 8 - times: 64 - scenarios: - - name: netcreate-1-boot - enabled: true - enable_dhcp: true - image_name: cirros - flavor_name: m1.tiny - num_networks: 1 - file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml - - name: netcreate-5-boot - enabled: true - enable_dhcp: true - image_name: cirros - flavor_name: m1.tiny - num_networks: 5 - file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: authenticate + enabled: true + type: rally + concurrency: + - 16 + - 24 + times: 2500 + scenarios: + - name: authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + - name: keystonebasic + enabled: true + type: rally + concurrency: + - 16 + times: 1000 + scenarios: + - name: create-and-list-tenants + enabled: true + file: rally/keystonebasic/create_and_list_tenants-cc.yml + - name: neutron + enabled: true + type: rally + concurrency: + - 8 + - 12 + times: 128 + scenarios: + - name: create-list-router + enabled: true + file: rally/neutron/neutron-create-list-router-cc.yml + - name: create-list-network + enabled: true + file: rally/neutron/neutron-create-list-network-cc.yml + - name: create-list-subnet + enabled: true + file: rally/neutron/neutron-create-list-subnet-cc.yml + - name: create-list-port + enabled: true + file: rally/neutron/neutron-create-list-port-cc.yml + - name: create-list-security-group + enabled: true + file: rally/neutron/neutron-create-list-security-group-cc.yml + - name: nova + enabled: true + type: rally + concurrency: + - 8 + - 12 + times: 64 + scenarios: + - name: boot-snapshot-delete + enabled: true + file: rally/nova/nova-boot-snapshot-cc.yml + image_name: cirros + flavor_name: m1.tiny + - name: boot-list + enabled: true + file: rally/nova/nova-boot-list-cc.yml + image_name: cirros + flavor_name: m1.tiny + - name: glance + enabled: true + type: rally + concurrency: + - 8 + - 12 + times: 64 + scenarios: + - name: create-and-delete-image + enabled: true + image_location: http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2.xz + flavor_name: m1.tiny + file: rally/glance/create-and-delete-image-cc.yml + - name: plugins + enabled: true + type: rally + concurrency: + - 8 + times: 64 + scenarios: + - name: netcreate-1-boot + enabled: true + enable_dhcp: true + image_name: cirros + flavor_name: m1.tiny + num_networks: 1 + file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml + - name: netcreate-5-boot + enabled: true + enable_dhcp: true + image_name: cirros + flavor_name: m1.tiny + num_networks: 5 + file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 index 9ae2d5a35..6c76f53a4 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 @@ -1,9 +1,13 @@ # Tests to be completed for the install-and-check.sh script minimal and short workloads are performed # to confirm functionality. browbeat: - results: results/ - rerun: 1 cloud_name: {{ browbeat_cloud_name }} + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: {{ elastic_enabled }} host: {{ elastic_host }} @@ -18,86 +22,111 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: {{ grafana_enabled }} - grafana_ip: {{ grafana_host }} - grafana_port: 3000 + host: {{ grafana_host }} + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv plugins: - netcreate-boot: rally/rally-plugins/netcreate-boot - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - - name: cinder - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: create-attach-volume-centos - enabled: true - file: rally/cinder/cinder-create-and-attach-volume-cc.yml - - name: keystonebasic - enabled: false - concurrency: - - 1 - times: 1 - scenarios: - - name: create-and-list-tenants - enabled: true - file: rally/keystonebasic/create_and_list_tenants-cc.yml - - name: neutron - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: create-list-network - enabled: true - file: rally/neutron/neutron-create-list-network-cc.yml - - name: nova - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: boot-snapshot-delete - enabled: true - file: rally/nova/nova-boot-snapshot-cc.yml - image_name: cirros - flavor_name: m1.tiny - - name: plugins - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: netcreate-1-boot - enabled: true - enable_dhcp: true - image_name: cirros - flavor_name: m1.tiny - num_networks: 1 - file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: authenticate + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + + - name: cinder + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: create-attach-volume-centos + enabled: true + file: rally/cinder/cinder-create-and-attach-volume-cc.yml + + - name: keystonebasic + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: create-and-list-tenants + enabled: true + file: rally/keystonebasic/create_and_list_tenants-cc.yml + + - name: neutron + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: create-list-network + enabled: true + file: rally/neutron/neutron-create-list-network-cc.yml + + - name: nova + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: boot-snapshot-delete + enabled: true + file: rally/nova/nova-boot-snapshot-cc.yml + image_name: cirros + flavor_name: m1.xtiny + + - name: plugins + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: netcreate-1-boot + enabled: true + enable_dhcp: true + image_name: cirros + flavor_name: m1.tiny + num_networks: 1 + file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 index a19e7632c..c2578f3eb 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-yoda-ci.yaml.j2 @@ -1,9 +1,13 @@ -# Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed +# Tests to be completed for the install-and-check.sh script minimal and short workloads are performed # to confirm functionality. browbeat: - results: results/ - rerun: 1 cloud_name: {{ browbeat_cloud_name }} + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: {{ elastic_enabled }} host: {{ elastic_host }} @@ -18,53 +22,73 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: {{ grafana_enabled }} - grafana_ip: {{ grafana_host }} - grafana_port: 3000 + host: {{ grafana_host }} + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true +rally: + sleep_before: 0 + sleep_after: 0 + plugins: + - netcreate-boot: rally/rally-plugins/netcreate-boot +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 yoda: - enabled: true instackenv: "/home/stack/instackenv.json" stackrc: "/home/stack/stackrc" - benchmarks: - - name: introspect-{{ overcloud_size }}-batch-5 - type: introspection - enabled: true - method: individual - times: 3 - timeout: 1800 - batch_size: 5 - - name: introspect-{{ overcloud_size }}-bulk - type: introspection - enabled: true - method: bulk - times: 3 - timeout: 1800 - - name: HA-Max-Compute-{{ overcloud_size }}-stack-update - type: overcloud - ntp_server: pool.ntp.org - timeout: 600 - templates: - - "" - enabled: true - step: 1 - keep_stack: true - times: 1 - cloud: - - node: "compute" - start_scale: {{ overcloud_size | int - 3 }} - end_scale: {{ overcloud_size | int - 3 }} - - node: "control" - start_scale: 3 - end_scale: 3 + +workloads: + - name: introspect-{{ overcloud_size }}-batch-5 + enabled: true + type: yoda + yoda_type: introspection + method: individual + times: 3 + timeout: 1800 + batch_size: 5 + - name: introspect-{{ overcloud_size }}-bulk + enabled: true + type: yoda + yoda_type: introspection + method: bulk + times: 3 + timeout: 1800 + - name: HA-Max-Compute-{{ overcloud_size }}-stack-update + enabled: true + type: yoda + yoda_type: overcloud + ntp_server: pool.ntp.org + timeout: 600 + templates: + - "" + step: 1 + keep_stack: true + times: 1 + cloud: + - node: "compute" + start_scale: {{ overcloud_size | int - 3 }} + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-yoda-scale.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-yoda-scale.yaml.j2 index 3bd328aca..4adc44220 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-yoda-scale.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-yoda-scale.yaml.j2 @@ -1,9 +1,11 @@ -# Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed -# to confirm functionality. browbeat: - results: results/ - rerun: 1 cloud_name: {{ browbeat_cloud_name }} + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: {{ elastic_enabled }} host: {{ elastic_host }} @@ -18,157 +20,187 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: {{ grafana_enabled }} - grafana_ip: {{ grafana_host }} - grafana_port: 3000 + host: {{ grafana_host }} + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true +rally: + sleep_before: 0 + sleep_after: 0 + plugins: + - netcreate-boot: rally/rally-plugins/netcreate-boot +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 yoda: - enabled: true instackenv: "/home/stack/instackenv.json" stackrc: "/home/stack/stackrc" - benchmarks: - - name: introspect-{{ overcloud_size }}-10-individual-batch-2 - type: introspection - enabled: true - method: individual - times: 10 - timeout: 900 - batch_size: 2 - - name: introspect-{{ overcloud_size }}-10-individual-batch-4 - type: introspection - enabled: true - method: individual - times: 10 - timeout: 900 - batch_size: 4 - - name: introspect-{{ overcloud_size }}-10-individual-batch-8 - type: introspection - enabled: true - method: individual - times: 10 - timeout: 900 - batch_size: 8 - - name: introspect-{{ overcloud_size }}-10-individual-batch-16 - type: introspection - enabled: true - method: individual - times: 10 - timeout: 900 - batch_size: 16 - - name: introspect-{{ overcloud_size }}-10-individual-batch-32 - type: introspection - enabled: true - method: individual - times: 10 - timeout: 900 - batch_size: 32 - - name: introspect-{{ overcloud_size }}-10-individual-batch-{{ overcloud_size }} - type: introspection - enabled: true - method: individual - times: 10 - timeout: 900 - batch_size: {{ overcloud_size }} - - name: introspect-{{ overcloud_size }}-50-bulk - type: introspection - enabled: true - method: bulk - times: 50 - timeout: 900 - - name: No-HA-Max-Compute-{{ overcloud_size }}-full-deploy - type: overcloud - ntp_server: clock01.util.phx2.redhat.com - timeout: 600 - templates: - - "" - enabled: true - step: 5 - keep_stack: false - times: 2 - cloud: - - node: "compute" - start_scale: 1 - end_scale: {{ overcloud_size | int - 1 }} - - node: "control" - start_scale: 1 - end_scale: 1 - - name: No-HA-Max-Compute-{{ overcloud_size }}-stack-update - type: overcloud - ntp_server: clock01.util.phx2.redhat.com - timeout: 600 - templates: - - "" - instackenv: "/home/stack/instackenv.json" - enabled: true - step: 5 - keep_stack: true - times: 2 - cloud: - - node: "compute" - start_scale: 1 - end_scale: {{ overcloud_size | int - 1 }} - - node: "control" - start_scale: 1 - end_scale: 1 - - name: HA-Max-Compute-{{ overcloud_size }}-full-deploy - type: overcloud - ntp_server: clock01.util.phx2.redhat.com - timeout: 600 - templates: - - "" - enabled: true - step: 5 - keep_stack: false - times: 2 - cloud: - - node: "compute" - start_scale: 1 - end_scale: {{ overcloud_size | int - 3 }} - - node: "control" - start_scale: 3 - end_scale: 3 - - name: HA-Max-Compute-{{ overcloud_size }}-stack-update - type: overcloud - ntp_server: clock01.util.phx2.redhat.com - timeout: 600 - templates: - - "" - enabled: true - step: 5 - keep_stack: true - times: 2 - cloud: - - node: "compute" - start_scale: 1 - end_scale: {{ overcloud_size | int - 3 }} - - node: "control" - start_scale: 3 - end_scale: 3 - - name: HA-Max-Compute-{{ overcloud_size }}-stack-update - type: overcloud - ntp_server: clock01.util.phx2.redhat.com - timeout: 600 - templates: - - "" - enabled: true - step: 5 - keep_stack: true - times: 2 - cloud: - - node: "compute" - start_scale: 1 - end_scale: {{ overcloud_size | int - 3 }} - - node: "control" - start_scale: 3 - end_scale: 3 + +workloads: + + - name: introspect-{{ overcloud_size }}-10-individual-batch-2 + type: yoda + yoda_type: introspection + enabled: true + method: individual + times: 10 + timeout: 900 + batch_size: 2 + - name: introspect-{{ overcloud_size }}-10-individual-batch-4 + type: yoda + yoda_type: introspection + enabled: true + method: individual + times: 10 + timeout: 900 + batch_size: 4 + - name: introspect-{{ overcloud_size }}-10-individual-batch-8 + type: yoda + yoda_type: introspection + enabled: true + method: individual + times: 10 + timeout: 900 + batch_size: 8 + - name: introspect-{{ overcloud_size }}-10-individual-batch-16 + type: yoda + yoda_type: introspection + enabled: true + method: individual + times: 10 + timeout: 900 + batch_size: 16 + - name: introspect-{{ overcloud_size }}-10-individual-batch-32 + type: yoda + yoda_type: introspection + enabled: true + method: individual + times: 10 + timeout: 900 + batch_size: 32 + - name: introspect-{{ overcloud_size }}-10-individual-batch-{{ overcloud_size }} + type: yoda + yoda_type: introspection + enabled: true + method: individual + times: 10 + timeout: 900 + batch_size: {{ overcloud_size }} + - name: introspect-{{ overcloud_size }}-50-bulk + type: yoda + yoda_type: introspection + enabled: true + method: bulk + times: 50 + timeout: 900 + - name: No-HA-Max-Compute-{{ overcloud_size }}-full-deploy + type: yoda + yoda_type: overcloud + ntp_server: clock01.util.phx2.redhat.com + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: false + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 1 }} + - node: "control" + start_scale: 1 + end_scale: 1 + - name: No-HA-Max-Compute-{{ overcloud_size }}-stack-update + type: yoda + yoda_type: overcloud + ntp_server: clock01.util.phx2.redhat.com + timeout: 600 + templates: + - "" + instackenv: "/home/stack/instackenv.json" + enabled: true + step: 5 + keep_stack: true + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 1 }} + - node: "control" + start_scale: 1 + end_scale: 1 + - name: HA-Max-Compute-{{ overcloud_size }}-full-deploy + type: yoda + yoda_type: overcloud + ntp_server: clock01.util.phx2.redhat.com + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: false + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 + - name: HA-Max-Compute-{{ overcloud_size }}-stack-update + type: yoda + yoda_type: overcloud + ntp_server: clock01.util.phx2.redhat.com + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: true + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 + - name: HA-Max-Compute-{{ overcloud_size }}-stack-update + type: yoda + yoda_type: overcloud + ntp_server: clock01.util.phx2.redhat.com + timeout: 600 + templates: + - "" + enabled: true + step: 5 + keep_stack: true + times: 2 + cloud: + - node: "compute" + start_scale: 1 + end_scale: {{ overcloud_size | int - 3 }} + - node: "control" + start_scale: 3 + end_scale: 3 diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index 5234561a5..c566338d4 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -1,13 +1,17 @@ # Complete set of Stress Tests, this can take a long time (day(s)) browbeat: - results: results/ - rerun: 3 cloud_name: openstack + rerun: 3 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: - enabled: false - regather: false + enabled: true host: 1.1.1.1 port: 9200 + regather: true metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -17,23 +21,13 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: true - grafana_ip: 1.1.1.1 - grafana_port: 3000 + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance perfkit: - enabled: true sleep_before: 0 sleep_after: 0 default: @@ -45,13 +39,7 @@ perfkit: openstack_network: browbeat_private timing_measurements: runtimes ignore_package_requirements: true - benchmarks: - - name: fio-centos-m1-small - enabled: false - benchmarks: fio - openstack_volume_size: 1 rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: @@ -59,207 +47,7 @@ rally: - neutron: rally/rally-plugins/neutron - netcreate-boot: rally/rally-plugins/netcreate-boot - workloads: rally/rally-plugins/workloads - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 64 - - 128 - - 192 - - 256 - - 320 - - 384 - - 448 - - 512 - times: 10000 - scenarios: - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - sla_max_avg_duration: 6 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-neutron - enabled: true - file: rally/authenticate/validate_neutron-cc.yml - sla_max_avg_duration: 18 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-nova - enabled: true - file: rally/authenticate/validate_nova-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: cinder - enabled: true - concurrency: - - 16 - - 32 - - 48 - - 64 - times: 128 - scenarios: - - name: create-attach-volume-centos - enabled: true - file: rally/cinder/cinder-create-and-attach-volume-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-attach-volume-cirros - enabled: true - image_name: cirros - file: rally/cinder/cinder-create-and-attach-volume-cc.yml - flavor_name: m1.tiny - sla_max_seconds: 30 - sla_max_failure: 0 - - name: keystonebasic - enabled: true - concurrency: - - 256 - times: 5000 - scenarios: - - name: create-and-list-tenants - enabled: true - file: rally/keystonebasic/create_and_list_tenants-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-users - enabled: true - file: rally/keystonebasic/create_and_list_users-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: neutron - enabled: true - concurrency: - - 16 - - 32 - - 48 - - 64 - times: 500 - scenarios: - - name: create-list-network - enabled: true - file: rally/neutron/neutron-create-list-network-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-list-port - enabled: true - file: rally/neutron/neutron-create-list-port-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-list-router - enabled: true - file: rally/neutron/neutron-create-list-router-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-list-security-group - enabled: true - file: rally/neutron/neutron-create-list-security-group-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-list-subnet - enabled: true - file: rally/neutron/neutron-create-list-subnet-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: nova - enabled: true - concurrency: - - 16 - - 32 - - 48 - - 64 - - 80 - - 96 - times: 128 - scenarios: - - name: boot-snapshot-delete-cirros - enabled: true - file: rally/nova/nova-boot-snapshot-cc.yml - image_name: cirros - flavor_name: m1.xtiny - - name: boot-list-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-boot-list-cc.yml - flavor_name: m1.xtiny - - name: boot-show-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-boot-show-cc.yml - flavor_name: m1.xtiny - - name: boot-bounce-delete-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-boot-bounce-delete-cc.yml - flavor_name: m1.xtiny - - name: boot-lock-unlock-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-lock-unlock-cc.yml - flavor_name: m1.xtiny - - name: glance - enabled: true - concurrency: - - 8 - times: 10 - scenarios: - - name: create-image-boot-instances - enabled: true - image_location: /home/stack/cirros.qcow2 - instances: 2 - flavor_name: m1.small - file: rally/glance/create-image-and-boot-instances-cc.yml - - name: create-and-delete-image - enabled: true - image_location: /home/stack/cirros.qcow2 - flavor_name: m1.small - file: rally/glance/create-and-delete-image-cc.yml - - name: create-and-list-image - enabled: true - image_location: /home/stack/cirros.qcow2 - file: rally/glance/create-and-list-image-cc.yml - - name: list-images - enabled: true - file: rally/glance/list-images-cc.yml - - name: plugins - enabled: true - concurrency: - - 16 - - 32 - - 48 - - 64 - times: 128 - scenarios: - - name: netcreate-boot - enabled: true - enable_dhcp: true - image_name: cirros - flavor_name: m1.tiny - file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: router-subnet-create-delete - enabled: true - num_networks: 50 - file: rally/rally-plugins/neutron/router_subnet_create_delete.yml - - name: glance-create-boot-delete - enabled: true - image_location: /home/stack/cirros.qcow2 - flavor_name: m1.xtiny - file: rally/rally-plugins/glance/glance_create_boot_delete.yml - - name: rally-port-create-associate - file: rally/rally-plugins/neutron/port_create_assoicate.yml - enabled: true - num_networks: 1 - num_ports: 100 - hypervisor: overcloud-compute-0.localdomain - browbeat_ssh_config: ansible/ssh-config - username: heat-admin -#shaker scenarios require atleast 2 compute nodes shaker: - enabled: true server: 1.1.1.1 port: 5555 flavor: m1.small @@ -268,121 +56,349 @@ shaker: sleep_after: 5 shaker_region: regionOne external_host: 2.2.2.2 - scenarios: - - name: l2-4-1 - enabled: true - density: 4 - compute: 1 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l2-8-1 - enabled: true - density: 8 - compute: 1 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l2-4-2 - enabled: true - density: 4 - compute: 2 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l2-8-2 - enabled: true - density: 8 - compute: 2 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l3-north-south-4-1 - enabled: true - placement: single_room - density: 4 - compute: 1 - progression: null - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-north-south-8-1 - enabled: false - placement: single_room - density: 8 - compute: 1 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-north-south-4-2 - enabled: true - placement: single_room - density: 4 - compute: 2 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-north-south-8-2 - enabled: true - placement: single_room - density: 8 - compute: 2 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-east-west-4-1 - enabled: true - density: 4 - compute: 1 - placement: single_room - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml - - name: l3-east-west-8-1 - enabled: true - density: 8 - compute: 1 - placement: single_room - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml - - name: l3-east-west-4-2 - enabled: true - density: 4 - compute: 2 - placement: single_room - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml - - name: l3-east-west-8-2 - enabled: true - density: 8 - compute: 2 - time: 60 - placement: single_room - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml -#yoda scenarios WILL redeploy your overcloud yoda: - enabled: false instackenv: "/home/stack/instackenv.json" stackrc: "/home/stack/stackrc" - benchmarks: - - name: scale-deploy - type: overcloud - ntp_server: pool.ntp.org - enabled: true - templates: - - "" - timeout: 600 #deploy timeout in minutes - step: 1 - keep_stack: false - times: 3 - cloud: - - node: "compute" - start_scale: 1 - end_scale: 1 - - node: "control" - start_scale: 1 - end_scale: 3 - - name: introspect-batch - type: introspection - enabled: true - method: individual #other option is bulk - times: 3 - timeout: 900 #introspection timeout in seconds - batch_size: 2 + +workloads: +# PerfKitBenchmarker + - name: fio-centos-m1-small + enabled: false + type: perfkit + benchmarks: fio + openstack_volume_size: 1 + +# Rally + - name: authenticate + enabled: true + type: rally + concurrency: + - 64 + - 128 + - 192 + - 256 + - 320 + - 384 + - 448 + - 512 + times: 10000 + scenarios: + - name: authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + sla_max_avg_duration: 6 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-neutron + enabled: true + file: rally/authenticate/validate_neutron-cc.yml + sla_max_avg_duration: 18 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-nova + enabled: true + file: rally/authenticate/validate_nova-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: cinder + enabled: true + type: rally + concurrency: + - 16 + - 32 + - 48 + - 64 + times: 128 + scenarios: + - name: create-attach-volume-centos + enabled: true + file: rally/cinder/cinder-create-and-attach-volume-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-attach-volume-cirros + enabled: true + image_name: cirros + file: rally/cinder/cinder-create-and-attach-volume-cc.yml + flavor_name: m1.tiny + sla_max_seconds: 30 + sla_max_failure: 0 + - name: keystonebasic + enabled: true + type: rally + concurrency: + - 256 + times: 5000 + scenarios: + - name: create-and-list-tenants + enabled: true + file: rally/keystonebasic/create_and_list_tenants-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-users + enabled: true + file: rally/keystonebasic/create_and_list_users-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: neutron + enabled: true + type: rally + concurrency: + - 16 + - 32 + - 48 + - 64 + times: 500 + scenarios: + - name: create-list-network + enabled: true + file: rally/neutron/neutron-create-list-network-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-list-port + enabled: true + file: rally/neutron/neutron-create-list-port-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-list-router + enabled: true + file: rally/neutron/neutron-create-list-router-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-list-security-group + enabled: true + file: rally/neutron/neutron-create-list-security-group-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-list-subnet + enabled: true + file: rally/neutron/neutron-create-list-subnet-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: nova + enabled: true + type: rally + concurrency: + - 16 + - 32 + - 48 + - 64 + - 80 + - 96 + times: 128 + scenarios: + - name: boot-snapshot-delete-cirros + enabled: true + file: rally/nova/nova-boot-snapshot-cc.yml + image_name: cirros + flavor_name: m1.xtiny + - name: boot-list-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-boot-list-cc.yml + flavor_name: m1.xtiny + - name: boot-show-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-boot-show-cc.yml + flavor_name: m1.xtiny + - name: boot-bounce-delete-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-boot-bounce-delete-cc.yml + flavor_name: m1.xtiny + - name: boot-lock-unlock-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-lock-unlock-cc.yml + flavor_name: m1.xtiny + - name: glance + enabled: true + type: rally + concurrency: + - 8 + times: 10 + scenarios: + - name: create-image-boot-instances + enabled: true + image_location: /home/stack/cirros.qcow2 + instances: 2 + flavor_name: m1.small + file: rally/glance/create-image-and-boot-instances-cc.yml + - name: create-and-delete-image + enabled: true + image_location: /home/stack/cirros.qcow2 + flavor_name: m1.small + file: rally/glance/create-and-delete-image-cc.yml + - name: create-and-list-image + enabled: true + image_location: /home/stack/cirros.qcow2 + file: rally/glance/create-and-list-image-cc.yml + - name: list-images + enabled: true + file: rally/glance/list-images-cc.yml + - name: plugins + enabled: true + type: rally + concurrency: + - 16 + - 32 + - 48 + - 64 + times: 128 + scenarios: + - name: netcreate-boot + enabled: true + enable_dhcp: true + image_name: cirros + flavor_name: m1.tiny + file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: router-subnet-create-delete + enabled: true + num_networks: 50 + file: rally/rally-plugins/neutron/router_subnet_create_delete.yml + - name: glance-create-boot-delete + enabled: true + image_location: /home/stack/cirros.qcow2 + flavor_name: m1.xtiny + file: rally/rally-plugins/glance/glance_create_boot_delete.yml + - name: rally-port-create-associate + file: rally/rally-plugins/neutron/port_create_assoicate.yml + enabled: true + num_networks: 1 + num_ports: 100 + hypervisor: overcloud-compute-0.localdomain + browbeat_ssh_config: ansible/ssh-config + username: heat-admin + +# Shaker scenarios require atleast 2 compute nodes + - name: l2-4-1 + enabled: true + type: shaker + density: 4 + compute: 1 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: l2-8-1 + enabled: true + type: shaker + density: 8 + compute: 1 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: l2-4-2 + enabled: true + type: shaker + density: 4 + compute: 2 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: l2-8-2 + enabled: true + type: shaker + density: 8 + compute: 2 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: l3-north-south-4-1 + enabled: true + type: shaker + placement: single_room + density: 4 + compute: 1 + progression: null + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml + - name: l3-north-south-8-1 + enabled: false + type: shaker + placement: single_room + density: 8 + compute: 1 + progression: null + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml + - name: l3-north-south-4-2 + enabled: true + type: shaker + placement: single_room + density: 4 + compute: 2 + progression: null + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml + - name: l3-north-south-8-2 + enabled: true + type: shaker + placement: single_room + density: 8 + compute: 2 + progression: null + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml + - name: l3-east-west-4-1 + enabled: true + type: shaker + density: 4 + compute: 1 + placement: single_room + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml + - name: l3-east-west-8-1 + enabled: true + type: shaker + density: 8 + compute: 1 + placement: single_room + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml + - name: l3-east-west-4-2 + enabled: true + type: shaker + density: 4 + compute: 2 + placement: single_room + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml + - name: l3-east-west-8-2 + enabled: true + type: shaker + density: 8 + compute: 2 + time: 60 + placement: single_room + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml + +# Yoda scenarios WILL redeploy your overcloud + - name: scale-deploy + enabled: true + type: yoda + yoda_type: overcloud + ntp_server: pool.ntp.org + templates: + - "" + timeout: 600 #deploy timeout in minutes + step: 1 + keep_stack: false + times: 3 + cloud: + - node: "compute" + start_scale: 1 + end_scale: 1 + - node: "control" + start_scale: 1 + end_scale: 3 + - name: introspect-batch + enabled: true + type: yoda + yoda_type: introspection + method: individual #other option is bulk + times: 3 + timeout: 900 #introspection timeout in seconds + batch_size: 2 diff --git a/browbeat-config.yaml b/browbeat-config.yaml index 283a8f74a..61c7aaacf 100644 --- a/browbeat-config.yaml +++ b/browbeat-config.yaml @@ -1,13 +1,21 @@ # Basic set of initial stress tests to test overcloud before running complete set of benchmarks. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + # Two types of rerun: + # iteration reruns on the iteration + # complete reruns after all workloads complete + # rerun_type: complete + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: - enabled: true - regather: true + enabled: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -17,23 +25,13 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: true - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: false + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance perfkit: - enabled: false sleep_before: 0 sleep_after: 0 default: @@ -45,45 +43,7 @@ perfkit: openstack_network: browbeat_private timing_measurements: runtimes ignore_package_requirements: true - benchmarks: - - name: fio-centos-m1-small - enabled: false - benchmarks: fio - openstack_volume_size: 1 -shaker: - enabled: true - server: 1.1.1.1 - port: 5555 - flavor: m1.small - join_timeout: 600 - sleep_before: 5 - sleep_after: 5 - shaker_region: regionOne - external_host: 2.2.2.2 - scenarios: - - name: l2 - enabled: true - density: 1 - compute: 1 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l3-north-south - enabled: true - placement: double_room - density: 1 - compute: 1 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-east-west - enabled: true - density: 1 - compute: 1 - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: @@ -91,178 +51,231 @@ rally: - neutron: rally/rally-plugins/neutron - netcreate-boot: rally/rally-plugins/netcreate-boot - workloads: rally/rally-plugins/workloads - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 8 - times: 100 - scenarios: - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - - name: authentic-neutron - enabled: true - file: rally/authenticate/validate_neutron-cc.yml - - name: authentic-nova - enabled: true - file: rally/authenticate/validate_nova-cc.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - - name: cinder - enabled: true - concurrency: - - 2 - times: 10 - scenarios: - - name: create-attach-volume-cirros - enabled: true - image_name: cirros - flavor_name: m1.tiny - file: rally/cinder/cinder-create-and-attach-volume-cc.yml +workloads: +# PerfKitBenchmarker + - name: fio-centos-m1-small + enabled: false + type: perfkit + benchmarks: fio + openstack_volume_size: 1 +# Shaker + - name: shaker-l2 + enabled: true + type: shaker + density: 1 + compute: 1 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: shaker-l3-north-south + enabled: true + type: shaker + placement: double_room + density: 1 + compute: 1 + progression: null + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml + - name: shaker-l3-east-west + enabled: true + type: shaker + density: 1 + compute: 1 + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml +# Rally + - name: authenticate + enabled: true + type: rally + concurrency: + - 8 + times: 100 + scenarios: + - name: authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + - name: authentic-neutron + enabled: true + file: rally/authenticate/validate_neutron-cc.yml + - name: authentic-nova + enabled: true + file: rally/authenticate/validate_nova-cc.yml - - name: keystonebasic - enabled: true - concurrency: - - 8 - times: 100 - scenarios: - - name: create-and-list-tenants - enabled: false - file: rally/keystonebasic/create_and_list_tenants-cc.yml - - name: create-and-list-users - enabled: true - file: rally/keystonebasic/create_and_list_users-cc.yml + - name: cinder + enabled: true + type: rally + concurrency: + - 2 + times: 10 + scenarios: + - name: create-attach-volume-cirros + enabled: true + image_name: cirros + flavor_name: m1.tiny + file: rally/cinder/cinder-create-and-attach-volume-cc.yml - - name: neutron - enabled: true - concurrency: - - 2 - times: 10 - scenarios: - - name: create-list-network - enabled: true - file: rally/neutron/neutron-create-list-network-cc.yml - - name: create-list-port - enabled: true - file: rally/neutron/neutron-create-list-port-cc.yml - - name: create-list-router - enabled: true - file: rally/neutron/neutron-create-list-router-cc.yml - - name: create-list-security-group - enabled: true - file: rally/neutron/neutron-create-list-security-group-cc.yml - - name: create-list-subnet - enabled: true - file: rally/neutron/neutron-create-list-subnet-cc.yml + - name: keystonebasic + enabled: true + type: rally + concurrency: + - 8 + times: 100 + scenarios: + - name: create-and-list-tenants + enabled: false + file: rally/keystonebasic/create_and_list_tenants-cc.yml + - name: create-and-list-users + enabled: true + file: rally/keystonebasic/create_and_list_users-cc.yml - - name: nova - enabled: true - concurrency: - - 2 - times: 10 - scenarios: - - name: boot-snapshot-delete-cirros - enabled: true - file: rally/nova/nova-boot-snapshot-cc.yml - image_name: cirros - flavor_name: m1.xtiny - - name: boot-list-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-boot-list-cc.yml - flavor_name: m1.xtiny - - name: boot-show-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-boot-show-cc.yml - flavor_name: m1.xtiny - - name: boot-bounce-delete-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-boot-bounce-delete-cc.yml - flavor_name: m1.xtiny - - name: boot-lock-unlock-cirros - enabled: true - image_name: cirros - file: rally/nova/nova-lock-unlock-cc.yml - flavor_name: m1.xtiny + - name: neutron + enabled: true + type: rally + concurrency: + - 2 + times: 10 + scenarios: + - name: create-list-network + enabled: true + file: rally/neutron/neutron-create-list-network-cc.yml + - name: create-list-port + enabled: true + file: rally/neutron/neutron-create-list-port-cc.yml + - name: create-list-router + enabled: true + file: rally/neutron/neutron-create-list-router-cc.yml + - name: create-list-security-group + enabled: true + file: rally/neutron/neutron-create-list-security-group-cc.yml + - name: create-list-subnet + enabled: true + file: rally/neutron/neutron-create-list-subnet-cc.yml - - name: glance - enabled: true - concurrency: - - 2 - times: 10 - scenarios: - - name: create-and-delete-image - enabled: true - image_location: /home/stack/cirros.qcow2 - container_format: bare - disk_format: qcow2 - file: rally/glance/create-and-delete-image-cc.yml + - name: nova + enabled: true + type: rally + concurrency: + - 2 + times: 10 + scenarios: + - name: boot-snapshot-delete-cirros + enabled: true + file: rally/nova/nova-boot-snapshot-cc.yml + image_name: cirros + flavor_name: m1.xtiny + - name: boot-list-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-boot-list-cc.yml + flavor_name: m1.xtiny + - name: boot-show-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-boot-show-cc.yml + flavor_name: m1.xtiny + - name: boot-bounce-delete-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-boot-bounce-delete-cc.yml + flavor_name: m1.xtiny + - name: boot-lock-unlock-cirros + enabled: true + image_name: cirros + file: rally/nova/nova-lock-unlock-cc.yml + flavor_name: m1.xtiny - - name: simple-plugins - enabled: true - concurrency: - - 8 - times: 10 - scenarios: - - name: netcreate-boot - enabled: true - enable_dhcp: true - image_name: cirros - flavor_name: m1.tiny - file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml - - name: netcreate-boot-ping - enabled: true - image_name: cirros - flavor_name: m1.xtiny - ext_net_id: - file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot_fip_ping.yml - - name: router-subnet-create-delete - enabled: true - num_networks: 5 - file: rally/rally-plugins/neutron/router_subnet_create_delete.yml - - name: glance-create-boot-delete - enabled: true - image_location: /home/stack/cirros.qcow2 - flavor_name: m1.xtiny - file: rally/rally-plugins/glance/glance_create_boot_delete.yml - - name: neutron-securitygroup-port - enabled: true - file: rally/rally-plugins/neutron/securitygroup_port.yml + - name: glance + enabled: true + type: rally + concurrency: + - 2 + times: 10 + scenarios: + - name: create-and-delete-image + enabled: true + image_location: /home/stack/cirros.qcow2 + container_format: bare + disk_format: qcow2 + file: rally/glance/create-and-delete-image-cc.yml - - name: plugin-workloads - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: linpack - enabled: true - image_name: browbeat-linpack - flavor_name: m1.small - external_network: - net_id: - file: rally/rally-plugins/workloads/linpack.yml + - name: simple-plugins + enabled: true + type: rally + concurrency: + - 8 + times: 10 + scenarios: + - name: netcreate-boot + enabled: true + enable_dhcp: true + image_name: cirros + flavor_name: m1.tiny + file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot.yml + - name: netcreate-boot-ping + enabled: true + image_name: cirros + flavor_name: m1.xtiny + ext_net_id: + file: rally/rally-plugins/netcreate-boot/netcreate_nova_boot_fip_ping.yml + - name: router-subnet-create-delete + enabled: true + num_networks: 5 + file: rally/rally-plugins/neutron/router_subnet_create_delete.yml + - name: glance-create-boot-delete + enabled: true + image_location: /home/stack/cirros.qcow2 + flavor_name: m1.xtiny + file: rally/rally-plugins/glance/glance_create_boot_delete.yml + - name: neutron-securitygroup-port + enabled: true + file: rally/rally-plugins/neutron/securitygroup_port.yml - - name: browbeat-pbench-uperf - enabled: true - user: root - image_name: browbeat-uperf - flavor_name: m1.small - # hypervisor_server: "nova:overcloud-compute-1.localdomain" - # hypervisor_client: "nova:overcloud-compute-0.localdomain" - external_network: - net_id: - protocols: tcp,udp - instances: 1 - num_pairs: 1 - samples: 1 - test_types: stream,rr - message_sizes: 64,1024,16384 - test_name: "browbeat-pbench-uperf" - send_results: true - cloudname: - elastic_host: - elastic_port: 9200 - file: rally/rally-plugins/workloads/pbench-uperf.yml + - name: plugin-workloads + enabled: false + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: linpack + enabled: true + image_name: browbeat-linpack + flavor_name: m1.small + external_network: + net_id: + file: rally/rally-plugins/workloads/linpack.yml + + - name: browbeat-pbench-uperf + enabled: true + user: root + image_name: browbeat-uperf + flavor_name: m1.small + # hypervisor_server: "nova:overcloud-compute-1.localdomain" + # hypervisor_client: "nova:overcloud-compute-0.localdomain" + external_network: + net_id: + protocols: tcp,udp + instances: 1 + num_pairs: 1 + samples: 1 + test_types: stream,rr + message_sizes: 64,1024,16384 + test_name: "browbeat-pbench-uperf" + send_results: true + cloudname: + elastic_host: + elastic_port: 9200 + file: rally/rally-plugins/workloads/pbench-uperf.yml diff --git a/browbeat.py b/browbeat.py index c7ab596ad..880921e4a 100755 --- a/browbeat.py +++ b/browbeat.py @@ -21,6 +21,8 @@ import time import browbeat.elastic import browbeat.tools import browbeat.workloadbase +from browbeat.config import load_browbeat_config +from browbeat.path import results_path _workload_opts = ['perfkit', 'rally', 'shaker', 'yoda'] _config_file = 'browbeat-config.yaml' @@ -28,26 +30,20 @@ debug_log_file = 'log/debug.log' def main(): - tools = browbeat.tools.Tools() parser = argparse.ArgumentParser( description="Browbeat Performance and Scale testing for Openstack") parser.add_argument( - '-s', - '--setup', - nargs='?', - default=_config_file, + '-s', '--setup', nargs='?', default=_config_file, help='Provide Browbeat YAML configuration file. Default is ./{}'.format(_config_file)) - parser.add_argument('workloads', nargs='*', help='Browbeat workload(s). Takes a space separated' - ' list of workloads ({}) or \"all\"'.format(', '.join(_workload_opts))) - parser.add_argument('--debug', action='store_true', - help='Enable Debug messages') - parser.add_argument('-p', '--postprocess', - dest="path", help="Path to process, ie results/20170101/") - parser.add_argument('-c', '--compare', - help="Compare metadata", dest="compare", - choices=['software-metadata']) - parser.add_argument('-u', '--uuid', - help="UUIDs to pass", dest="uuids", nargs=2) + parser.add_argument( + 'workloads', nargs='*', help='Browbeat workload(s). Takes a space separated' + ' list of workloads ({}) or \"all\"'.format(', '.join(_workload_opts))) + parser.add_argument('--debug', action='store_true', help='Enable Debug messages') + parser.add_argument( + '-p', '--postprocess', dest="path", help="Path to process, ie results/20171130-191420/") + parser.add_argument( + '-c', '--compare', help="Compare metadata", dest="compare", choices=['software-metadata']) + parser.add_argument('-u', '--uuid', help="UUIDs to pass", dest="uuids", nargs=2) _cli_args = parser.parse_args() _logger = logging.getLogger('browbeat') @@ -70,8 +66,10 @@ def main(): _logger.debug("CLI Args: {}".format(_cli_args)) # Load Browbeat yaml config file: - _config = tools._load_config(_cli_args.setup) + _config = load_browbeat_config(_cli_args.setup) + tools = browbeat.tools.Tools(_config) + # Browbeat compare if _cli_args.compare == "software-metadata": es = browbeat.elastic.Elastic(_config, "BrowbeatCLI") es.compare_metadata("_all", 'controller', _cli_args.uuids) @@ -81,74 +79,93 @@ def main(): parser.print_help() exit(1) - # Default to all workloads + # Browbeat postprocess + if _cli_args.path: + _logger.info("Browbeat Postprocessing {}".format(_cli_args.path)) + return tools.post_process(_cli_args) + + # Browbeat workload - "browbeat run" if _cli_args.workloads == []: _cli_args.workloads.append('all') - if _cli_args.path: - return tools.post_process(_cli_args) if len(_cli_args.workloads) == 1 and 'all' in _cli_args.workloads: _cli_args.workloads = _workload_opts - invalid_wkld = [ - wkld for wkld in _cli_args.workloads if wkld not in _workload_opts] + invalid_wkld = [wkld for wkld in _cli_args.workloads if wkld not in _workload_opts] if invalid_wkld: _logger.error("Invalid workload(s) specified: {}".format(invalid_wkld)) if 'all' in _cli_args.workloads: _logger.error( "If you meant 'all' use: './browbeat.py all' or './browbeat.py'") exit(1) - else: - time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") - _logger.info("Browbeat test suite kicked off") - _logger.info("Browbeat UUID: {}".format(browbeat.elastic.browbeat_uuid)) - if _config['elasticsearch']['enabled']: - _logger.info("Checking for Metadata") - metadata_exists = tools.check_metadata() - if not metadata_exists: - _logger.error("Elasticsearch has been enabled but" - " metadata files do not exist") - _logger.info("Gathering Metadata") - tools.gather_metadata() - elif _config['elasticsearch']['regather']: - _logger.info("Regathering Metadata") - tools.gather_metadata() - _logger.info("Running workload(s): {}".format( - ','.join(_cli_args.workloads))) - for wkld_provider in _cli_args.workloads: - if wkld_provider in _config: - if _config[wkld_provider]['enabled']: - tools._run_workload_provider(wkld_provider) - else: - _logger.warning("{} is not enabled in {}".format(wkld_provider, - _cli_args.setup)) - else: - _logger.error("{} is missing in {}".format( - wkld_provider, _cli_args.setup)) - result_dir = _config['browbeat']['results'] - browbeat.workloadbase.WorkloadBase.print_report(result_dir, time_stamp) - _logger.info("Saved browbeat result summary to {}".format( - os.path.join(result_dir, time_stamp + '.' + 'report'))) - browbeat.workloadbase.WorkloadBase.print_summary() + result_dir_ts = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") + _logger.info("Browbeat test suite kicked off") + _logger.info("Browbeat UUID: {}".format(browbeat.elastic.browbeat_uuid)) + if _config['elasticsearch']['enabled']: + _logger.info("Checking for Metadata") + metadata_exists = tools.check_metadata() + if not metadata_exists: + _logger.error("Elasticsearch has been enabled but" + " metadata files do not exist") + _logger.info("Gathering Metadata") + tools.gather_metadata() + elif _config['elasticsearch']['regather']: + _logger.info("Regathering Metadata") + tools.gather_metadata() - browbeat_rc = 0 - if browbeat.workloadbase.WorkloadBase.failure > 0: - browbeat_rc = 1 - if browbeat.workloadbase.WorkloadBase.index_failures > 0: - browbeat_rc = 2 + _logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads))) + # Iteration rerun_type pushes rerun logic down to the workload itself. This allows the workload + # to run multiple times before moving to the next workload + if _config["browbeat"]["rerun_type"] == "iteration": + for workload in _config["workloads"]: + if not workload["enabled"]: + _logger.info("{} workload {} disabled in browbeat config".format(workload["type"], + workload["name"])) + continue - if browbeat_rc == 1: - _logger.info("Browbeat finished with test failures, UUID: {}".format( - browbeat.elastic.browbeat_uuid)) - sys.exit(browbeat_rc) - elif browbeat_rc == 2: - _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}" - .format(browbeat.elastic.browbeat_uuid)) - sys.exit(browbeat_rc) - else: - _logger.info("Browbeat finished successfully, UUID: {}".format( - browbeat.elastic.browbeat_uuid)) - sys.exit(0) + if not workload["type"] in _cli_args.workloads: + _logger.info( + "{} workload {} disabled via cli".format(workload["type"], workload["name"])) + continue + + _logger.info("{} workload {} is enabled".format(workload["type"], workload["name"])) + tools.run_workload(workload, result_dir_ts, 0) + browbeat.workloadbase.WorkloadBase.print_summary() + + elif _config["browbeat"]["rerun_type"] == "complete": + # Complete rerun_type, reruns after all workloads have been run. + for run_iteration in range(0, _config["browbeat"]["rerun"]): + for workload in _config["workloads"]: + if not workload["enabled"]: + _logger.info("{} workload {} disabled in browbeat config" + .format(workload["type"], workload["name"])) + continue + + if not workload["type"] in _cli_args.workloads: + _logger.info("{} workload {} disabled via cli".format(workload["type"], + workload["name"])) + continue + + _logger.info("{} workload {} is enabled".format(workload["type"], workload["name"])) + tools.run_workload(workload, result_dir_ts, run_iteration) + browbeat.workloadbase.WorkloadBase.print_summary() + + browbeat.workloadbase.WorkloadBase.print_report(results_path, result_dir_ts) + _logger.info("Saved browbeat result summary to {}" + .format(os.path.join(results_path, "{}.report".format(result_dir_ts)))) + + if browbeat.workloadbase.WorkloadBase.failure > 0: + _logger.info( + "Browbeat finished with test failures, UUID: {}".format(browbeat.elastic.browbeat_uuid)) + sys.exit(1) + + if browbeat.workloadbase.WorkloadBase.index_failures > 0: + _logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}" + .format(browbeat.elastic.browbeat_uuid)) + sys.exit(2) + + _logger.info("Browbeat finished successfully, UUID: {}".format(browbeat.elastic.browbeat_uuid)) + sys.exit(0) if __name__ == '__main__': sys.exit(main()) diff --git a/browbeat/config.py b/browbeat/config.py new file mode 100644 index 000000000..a7e9343bc --- /dev/null +++ b/browbeat/config.py @@ -0,0 +1,57 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import yaml + +from pykwalify import core as pykwalify_core +from pykwalify import errors as pykwalify_errors + +from browbeat.path import conf_schema_path + + +_logger = logging.getLogger("browbeat.config") + +def load_browbeat_config(path): + """Loads and validates an entire Browbeat config per the expected schema. + + :param path: The path to the Browbeat Config file + """ + with open(path, "r") as config_file: + browbeat_config = yaml.safe_load(config_file) + _logger.debug("Browbeat config {} yaml loaded".format(path)) + + # Validate base config for Browbeat format + _validate_yaml("browbeat", browbeat_config) + _logger.info("Config {} validated".format(path)) + + # Validate per-workloads + for workload in browbeat_config["workloads"]: + _validate_yaml(workload["type"], workload) + _logger.info("Workload {} validated as {}".format(workload["name"], workload["type"])) + + return browbeat_config + +def _validate_yaml(schema, config): + """Raises exception if config is invalid. + + :param schema: The schema to validate with (browbeat, perfkit, rally...) + :param config: Loaded yaml to validate + """ + with open("{}/{}.yml".format(conf_schema_path, schema), "r") as schema_file: + schema = yaml.safe_load(schema_file) + check = pykwalify_core.Core(source_data=config, schema_data=schema) + try: + check.validate(raise_exception=True) + except pykwalify_errors.SchemaError as e: + _logger.error("Schema validation failed") + raise Exception("File does not conform to {} schema: {}".format(schema, e)) diff --git a/browbeat/grafana.py b/browbeat/grafana.py index 487334fbb..b7e3a203e 100644 --- a/browbeat/grafana.py +++ b/browbeat/grafana.py @@ -19,8 +19,8 @@ class Grafana(object): self.logger = logging.getLogger('browbeat.grafana') self.config = config self.cloud_name = self.config['browbeat']['cloud_name'] - self.grafana_ip = self.config['grafana']['grafana_ip'] - self.grafana_port = self.config['grafana']['grafana_port'] + self.host = self.config['grafana']['host'] + self.port = self.config['grafana']['port'] self.grafana_url = {} def grafana_urls(self): @@ -31,7 +31,7 @@ class Grafana(object): from_ts = time['from_ts'] to_ts = time['to_ts'] url = 'http://{}:{}/dashboard/db/'.format( - self.grafana_ip, self.grafana_port) + self.host, self.port) for dashboard in self.config['grafana']['dashboards']: self.grafana_url[dashboard] = '{}{}?from={}&to={}&var-Cloud={}'.format( url, diff --git a/browbeat/path.py b/browbeat/path.py index 8a5ad653a..3f91fb670 100644 --- a/browbeat/path.py +++ b/browbeat/path.py @@ -27,6 +27,8 @@ base_path = os.path.abspath(os.path.join(browbeat_path, os.pardir)) conf_path = os.path.join(browbeat_path, 'conf') +conf_schema_path = os.path.join(browbeat_project_path, 'schema') + log_path = os.path.join(browbeat_path, 'log') results_path = os.path.join(browbeat_path, 'results') diff --git a/browbeat/perfkit.py b/browbeat/perfkit.py index 00a771201..458d7634e 100644 --- a/browbeat/perfkit.py +++ b/browbeat/perfkit.py @@ -23,15 +23,17 @@ import elastic import grafana from path import get_overcloudrc from path import get_workload_venv +from path import results_path import tools import workloadbase class PerfKit(workloadbase.WorkloadBase): - def __init__(self, config): + def __init__(self, config, result_dir_ts): self.logger = logging.getLogger('browbeat.perfkit') self.overcloudrc = get_overcloudrc() self.config = config + self.result_dir_ts = result_dir_ts self.error_count = 0 self.tools = tools.Tools(self.config) self.grafana = grafana.Grafana(self.config) @@ -40,16 +42,6 @@ class PerfKit(workloadbase.WorkloadBase): self.scenario_count = 0 self.pass_count = 0 - def _log_details(self): - self.logger.info( - "Current number of Perkit scenarios executed: {}".format(self.scenario_count)) - self.logger.info( - "Current number of Perfkit test(s) executed: {}".format(self.test_count)) - self.logger.info( - "Current number of Perfkit test(s) succeeded: {}".format(self.pass_count)) - self.logger.info( - "Current number of Perfkit test failures: {}".format(self.error_count)) - def string_to_dict(self, string): """Function for converting "|" quoted hash data into python dictionary.""" dict_data = {} @@ -126,6 +118,8 @@ class PerfKit(workloadbase.WorkloadBase): # Build command to run if 'enabled' in benchmark_config: del benchmark_config['enabled'] + if 'type' in benchmark_config: + del benchmark_config['type'] cmd = ("source {0}; source {1}; " "{2}/PerfKitBenchmarker/pkb.py " "--cloud={3} --run_uri=browbeat".format( @@ -182,54 +176,47 @@ class PerfKit(workloadbase.WorkloadBase): return success, to_ts, from_ts - def run_workloads(self): + def run_workload(self, workload, run_iteration): self.logger.info("Starting PerfKitBenchmarker Workloads.") time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp)) - benchmarks = self.config.get('perfkit')['benchmarks'] - if (benchmarks is not None and len(benchmarks) > 0): - for benchmark in benchmarks: - if benchmark['enabled']: - self.logger.info("Benchmark: {}".format(benchmark['name'])) - self.update_scenarios() - self.update_total_scenarios() - # Add default parameters as necessary - for default_item, value in self.config['perfkit']['default'].iteritems(): - if default_item not in benchmark: - benchmark[default_item] = value - for run in range(self.config['browbeat']['rerun']): - self.update_tests() - self.update_total_tests() - result_dir = self.tools.create_results_dir( - self.config['browbeat']['results'], time_stamp, benchmark['name'], - str(run)) - test_name = "{}-{}-{}".format(time_stamp, benchmark['name'], run) - workload = self.__class__.__name__ - self.workload_logger(result_dir, workload) - success, to_ts, from_ts = self.run_benchmark(benchmark, result_dir, - test_name) - index_success = 'disabled' - if self.config['elasticsearch']['enabled']: - index_success = self.index_results(success, result_dir, test_name, run, - benchmark) - new_test_name = test_name.split('-') - new_test_name = new_test_name[2:] - new_test_name = '-'.join(new_test_name) - if success: - self.update_pass_tests() - self.update_total_pass_tests() - self.get_time_dict(to_ts, from_ts, benchmark['benchmarks'], - new_test_name, self.__class__.__name__, "pass", - index_success) - else: - self.update_fail_tests() - self.update_total_fail_tests() - self.get_time_dict(to_ts, from_ts, benchmark['benchmarks'], - new_test_name, self.__class__.__name__, "fail", - index_success) - self._log_details() - else: - self.logger.info( - "Skipping {} benchmark, enabled: false".format(benchmark['name'])) - else: - self.logger.error("Config file contains no perfkit benchmarks.") + + self.logger.info("Benchmark: {}".format(workload['name'])) + self.update_scenarios() + self.update_total_scenarios() + # Add default parameters as necessary + for default_item, value in self.config['perfkit']['default'].iteritems(): + if default_item not in workload: + workload[default_item] = value + + # Correct iteration/rerun + rerun_range = range(self.config["browbeat"]["rerun"]) + if self.config["browbeat"]["rerun_type"] == "complete": + rerun_range = range(run_iteration, run_iteration + 1) + + for run in rerun_range: + self.update_tests() + self.update_total_tests() + result_dir = self.tools.create_results_dir( + results_path, self.result_dir_ts, workload['name'], str(run)) + test_name = "{}-{}-{}".format(time_stamp, workload['name'], run) + self.workload_logger(self.__class__.__name__) + success, to_ts, from_ts = self.run_benchmark(workload, result_dir, test_name) + index_success = 'disabled' + if self.config['elasticsearch']['enabled']: + index_success = self.index_results(success, result_dir, test_name, run, workload) + new_test_name = test_name.split('-') + new_test_name = new_test_name[2:] + new_test_name = '-'.join(new_test_name) + if success: + self.update_pass_tests() + self.update_total_pass_tests() + self.get_time_dict(to_ts, from_ts, workload['benchmarks'], + new_test_name, self.__class__.__name__, "pass", + index_success) + else: + self.update_fail_tests() + self.update_total_fail_tests() + self.get_time_dict(to_ts, from_ts, workload['benchmarks'], + new_test_name, self.__class__.__name__, "fail", + index_success) diff --git a/browbeat/rally.py b/browbeat/rally.py index f0340a9f7..a13f24721 100644 --- a/browbeat/rally.py +++ b/browbeat/rally.py @@ -23,15 +23,17 @@ import time import elastic import grafana from path import get_workload_venv +from path import results_path import workloadbase import tools class Rally(workloadbase.WorkloadBase): - def __init__(self, config, hosts=None): + def __init__(self, config, result_dir_ts): self.logger = logging.getLogger('browbeat.rally') self.config = config + self.result_dir_ts = result_dir_ts self.tools = tools.Tools(self.config) self.grafana = grafana.Grafana(self.config) self.elastic = elastic.Elastic(self.config, self.__class__.__name__.lower()) @@ -43,7 +45,7 @@ class Rally(workloadbase.WorkloadBase): def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark): self.logger.debug("--------------------------------") self.logger.debug("task_file: {}".format(task_file)) - self.logger.debug("scenario_args: {}".format(scenario_args)) + self.logger.info("Running with scenario_args: {}".format(scenario_args)) self.logger.debug("result_dir: {}".format(result_dir)) self.logger.debug("test_name: {}".format(test_name)) self.logger.debug("--------------------------------") @@ -91,17 +93,6 @@ class Rally(workloadbase.WorkloadBase): test_name) return self.tools.run_cmd(cmd)['stdout'] - def _get_details(self): - self.logger.info( - "Current number of Rally scenarios executed:{}".format( - self.scenario_count)) - self.logger.info( - "Current number of Rally tests executed:{}".format(self.test_count)) - self.logger.info( - "Current number of Rally tests passed:{}".format(self.pass_count)) - self.logger.info( - "Current number of Rally test failures:{}".format(self.error_count)) - def gen_scenario_html(self, task_ids, test_name): all_task_ids = ' '.join(task_ids) cmd = "source {}; ".format(get_workload_venv('rally', True)) @@ -223,152 +214,113 @@ class Rally(workloadbase.WorkloadBase): success = False return success - def run_workloads(self): - """Iterates through all rally scenarios in browbeat yaml config file""" - results = collections.OrderedDict() - self.logger.info("Starting Rally workloads") + def run_workload(self, workload, run_iteration): + """Runs a Browbeat Rally workload""" + results = [] + self.logger.info("Running Rally workload: {}".format(workload["name"])) es_ts = datetime.datetime.utcnow() - dir_ts = es_ts.strftime("%Y%m%d-%H%M%S") - self.logger.debug("Time Stamp (Prefix): {}".format(dir_ts)) - benchmarks = self.config.get('rally')['benchmarks'] - if (benchmarks is not None and len(benchmarks) > 0): - for benchmark in benchmarks: - if benchmark['enabled']: - self.logger.info("Benchmark: {}".format(benchmark['name'])) - scenarios = benchmark['scenarios'] - def_concurrencies = benchmark['concurrency'] - def_times = benchmark['times'] - self.logger.debug( - "Default Concurrencies: {}".format(def_concurrencies)) - self.logger.debug("Default Times: {}".format(def_times)) - for scenario in scenarios: - if scenario['enabled']: - self.update_scenarios() - self.update_total_scenarios() - scenario_name = scenario['name'] - scenario_file = scenario['file'] - self.logger.info( - "Running Scenario: {}".format(scenario_name)) - self.logger.debug( - "Scenario File: {}".format(scenario_file)) + def_concurrencies = workload["concurrency"] + def_times = workload["times"] + self.logger.debug("Default Concurrencies: {}".format(def_concurrencies)) + self.logger.debug("Default Times: {}".format(def_times)) + for scenario in workload["scenarios"]: + if not scenario["enabled"]: + self.logger.info("{} scenario is disabled".format(scenario['name'])) + continue - del scenario['enabled'] - del scenario['file'] - del scenario['name'] - if len(scenario) > 0: - self.logger.debug( - "Overriding Scenario Args: {}".format(scenario)) + self.logger.info("Running Scenario: {}".format(scenario["name"])) + self.logger.debug("Scenario File: {}".format(scenario["file"])) + self.update_scenarios() + self.update_total_scenarios() + scenario_name = scenario["name"] + scenario_file = scenario["file"] + del scenario["enabled"] + del scenario["file"] + del scenario["name"] + if len(scenario) > 0: + self.logger.debug("Overriding Scenario Args: {}".format(scenario)) - result_dir = self.tools.create_results_dir( - self.config['browbeat']['results'], dir_ts, - self.__class__.__name__.lower(), - benchmark['name'], scenario_name - ) - self.logger.debug( - "Created result directory: {}".format(result_dir)) - workload = self.__class__.__name__ - self.workload_logger(result_dir, workload) + result_dir = self.tools.create_results_dir( + results_path, self.result_dir_ts, + self.__class__.__name__.lower(), + workload["name"], scenario_name + ) - # Override concurrency/times - if 'concurrency' in scenario: - concurrencies = scenario['concurrency'] - del scenario['concurrency'] - else: - concurrencies = def_concurrencies - concurrency_count_dict = collections.Counter( - concurrencies) - if 'times' not in scenario: - scenario['times'] = def_times + self.logger.debug("Created result directory: {}".format(result_dir)) + self.workload_logger(self.__class__.__name__) - for concurrency in concurrencies: - scenario['concurrency'] = concurrency - for run in range(self.config['browbeat']['rerun']): - if run not in results: - results[run] = [] - self.update_tests() - self.update_total_tests() - if concurrency_count_dict[concurrency] == 1: - test_name = "{}-browbeat-{}-{}-iteration-{}".format( - dir_ts, scenario_name, concurrency, run) - else: - test_name = "{}-browbeat-{}-{}-{}-iteration-{}".format( - dir_ts, scenario_name, concurrency, - concurrency_count_dict[concurrency], run) - self.logger.debug("Duplicate concurrency {} found," - " setting test name" - " to {}".format(concurrency, test_name)) - concurrency_count_dict[ - concurrency] -= 1 + # Override concurrency/times + if "concurrency" in scenario: + concurrencies = scenario["concurrency"] + del scenario["concurrency"] + else: + concurrencies = def_concurrencies + if "times" not in scenario: + scenario["times"] = def_times - if not result_dir: - self.logger.error( - "Failed to create result directory") - exit(1) + concurrency_count_dict = collections.Counter() + for concurrency in concurrencies: + scenario["concurrency"] = concurrency - from_time, to_time = self.run_scenario( - scenario_file, scenario, result_dir, test_name, - benchmark['name']) + # Correct iteration/rerun + rerun_range = range(self.config["browbeat"]["rerun"]) + if self.config["browbeat"]["rerun_type"] == "complete": + rerun_range = range(run_iteration, run_iteration + 1) - new_test_name = test_name.split('-') - new_test_name = new_test_name[3:] - new_test_name = "-".join(new_test_name) + for run in rerun_range: + self.update_tests() + self.update_total_tests() + concurrency_count_dict[concurrency] += 1 + test_name = "{}-browbeat-{}-{}-{}-iteration-{}".format( + es_ts.strftime("%Y%m%d-%H%M%S"), scenario_name, concurrency, + concurrency_count_dict[concurrency], run) - # Find task id (if task succeeded in - # running) - task_id = self.get_task_id(test_name) - if task_id: - self.logger.info( - "Generating Rally HTML for task_id : {}". - format(task_id)) - self.gen_scenario_html( - [task_id], test_name) - self.gen_scenario_json_file( - task_id, test_name) - results[run].append(task_id) - self.update_pass_tests() - self.update_total_pass_tests() - if self.config['elasticsearch']['enabled']: - # Start indexing - index_status = self.json_result( - task_id, scenario_name, run, test_name, result_dir) - if not index_status: - self.update_index_failures() - self.get_time_dict(to_time, from_time, - benchmark[ - 'name'], new_test_name, - workload, "pass", index_status) - else: - self.get_time_dict(to_time, from_time, benchmark[ - 'name'], new_test_name, - workload, "pass", ) + if not result_dir: + self.logger.error( + "Failed to create result directory") + exit(1) - else: - self.logger.error( - "Cannot find task_id") - self.update_fail_tests() - self.update_total_fail_tests() - self.get_time_dict( - to_time, from_time, benchmark[ - 'name'], new_test_name, - workload, "fail") + from_time, to_time = self.run_scenario( + scenario_file, scenario, result_dir, test_name, workload["name"]) - for data in glob.glob("./{}*".format(test_name)): - shutil.move(data, result_dir) - - self._get_details() + new_test_name = test_name.split("-") + new_test_name = new_test_name[3:] + new_test_name = "-".join(new_test_name) + # Find task id (if task succeeded in running) + task_id = self.get_task_id(test_name) + if task_id: + self.logger.info("Generating Rally HTML for task_id : {}".format(task_id)) + self.gen_scenario_html([task_id], test_name) + self.gen_scenario_json_file(task_id, test_name) + results.append(task_id) + self.update_pass_tests() + self.update_total_pass_tests() + if self.config["elasticsearch"]["enabled"]: + # Start indexing + index_status = self.json_result( + task_id, scenario_name, run, test_name, result_dir) + if not index_status: + self.update_index_failures() + self.get_time_dict(to_time, from_time, workload["name"], new_test_name, + self.__class__.__name__, "pass", index_status) else: - self.logger.info( - "Skipping {} scenario enabled: false".format(scenario['name'])) - else: - self.logger.info( - "Skipping {} benchmarks enabled: false".format(benchmark['name'])) - self.logger.debug("Creating Combined Rally Reports") - for run in results: - combined_html_name = 'all-rally-run-{}'.format(run) - self.gen_scenario_html(results[run], combined_html_name) - if os.path.isfile('{}.html'.format(combined_html_name)): - shutil.move('{}.html'.format(combined_html_name), - '{}/{}'.format(self.config['browbeat']['results'], dir_ts)) - else: - self.logger.error("Config file contains no rally benchmarks.") + self.get_time_dict(to_time, from_time, workload["name"], new_test_name, + self.__class__.__name__, "pass", ) + + else: + self.logger.error("Cannot find task_id") + self.update_fail_tests() + self.update_total_fail_tests() + self.get_time_dict(to_time, from_time, workload["name"], new_test_name, + self.__class__.__name__, "fail") + + for data in glob.glob("./{}*".format(test_name)): + shutil.move(data, result_dir) + + self.logger.debug("Creating Combined Rally Reports") + combined_html_name = "all-rally-run-{}".format(run_iteration) + self.gen_scenario_html(results, combined_html_name) + if os.path.isfile("{}.html".format(combined_html_name)): + shutil.move("{}.html".format(combined_html_name), "{}/{}/{}/{}".format(results_path, + self.result_dir_ts, self.__class__.__name__.lower(), workload["name"])) diff --git a/browbeat/schema/browbeat.yml b/browbeat/schema/browbeat.yml new file mode 100644 index 000000000..86a5ebf10 --- /dev/null +++ b/browbeat/schema/browbeat.yml @@ -0,0 +1,194 @@ +# This schema defines how Browbeat config files are formated. Generally items +# in a dictionary format will be alphabetically ordered however the Browbeat +# options will appear first in base level yaml dictionary. +name: Browbeat configuration schema +type: map +mapping: + # General Browbeat Configuration Items + browbeat: + required: True + type: map + mapping: + cloud_name: + type: str + required: True + rerun: + type: int + required: True + rerun_type: + type: str + enum: ['iteration', 'complete'] + required: True + ansible: + required: True + type: map + mapping: + hosts: + type: str + required: True + metadata_playbook: + type: str + required: True + ssh_config: + type: str + required: True + elasticsearch: + required: True + type: map + mapping: + enabled: + type: bool + required: True + host: + type: str + required: True + metadata_files: + type: seq + required: True + sequence: + - type: map + mapping: + name: + type: str + required: True + file: + type: str + required: True + port: + type: int + required: True + regather: + type: bool + required: True + grafana: + required: True + type: map + mapping: + dashboards: + type: seq + required: True + sequence: + - type: str + enabled: + type: bool + required: True + host: + type: str + required: True + port: + type: int + required: True + perfkit: + required: True + type: map + mapping: + default: + type: map + required: True + mapping: + image: + type: str + required: True + machine_type: + type: str + required: True + os_type: + type: str + required: True + enum: ['rhel', 'debian', 'ubuntu_container', 'windows'] + openstack_image_username: + type: str + required: True + openstack_floating_ip_pool: + type: str + required: True + openstack_network: + type: str + required: True + ignore_package_requirements: + type: bool + required: False + timing_measurements: + type: str + required: False + sleep_after: + type: number + required: True + sleep_before: + type: number + required: True + rally: + required: True + type: map + mapping: + plugins: + type: seq + required: False + sequence: + - type: map + required: False + allowempty: True + sleep_after: + type: number + required: True + sleep_before: + type: number + required: True + shaker: + required: True + type: map + mapping: + server: + type: str + required: True + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + port: + type: int + required: True + flavor: + type: str + required: True + join_timeout: + type: int + required: True + sleep_before: + type: number + required: True + sleep_after: + type: number + required: True + shaker_region: + type: str + required: true + external_host: + type: str + required: False + pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ + yoda: + type: map + required: True + mapping: + instackenv: + type: str + required: True + stackrc: + type: str + required: True + workloads: + required: True + type: seq + sequence: + - type: map + required: True + allowempty: True + mapping: + enabled: + type: bool + required: True + name: + type: str + required: True + type: + type: str + required: True + enum: ['perfkit', 'rally', 'shaker', 'yoda'] diff --git a/browbeat/schema/perfkit.yml b/browbeat/schema/perfkit.yml new file mode 100644 index 000000000..28afa2804 --- /dev/null +++ b/browbeat/schema/perfkit.yml @@ -0,0 +1,48 @@ +# This schema defines how a PerfKitBenchmarker workload is formated +name: PerfKitBenchmarker workload schema +type: map +allowempty: True +mapping: + # Required items to be a PerfKit workload + benchmarks: + type: str + required: True + enabled: + type: bool + required: True + name: + type: str + required: True + type: + type: str + required: True + enum: ["perfkit"] + # Over-ridable defaults: + ignore_package_requirements: + type: bool + required: False + image: + type: str + required: False + machine_type: + type: str + required: False + openstack_floating_ip_pool: + type: str + required: False + openstack_image_username: + type: str + required: False + openstack_network: + type: str + required: False + openstack_volume_size: + type: int + required: False + os_type: + type: str + required: False + enum: ['rhel', 'debian', 'ubuntu_container', 'windows'] + timing_measurements: + type: str + required: False diff --git a/browbeat/schema/rally.yml b/browbeat/schema/rally.yml new file mode 100644 index 000000000..25c23e938 --- /dev/null +++ b/browbeat/schema/rally.yml @@ -0,0 +1,38 @@ +# This schema defines how a Rally workload is formated +name: Rally workload schema +type: map +mapping: + # Required items to be a Rally workload + concurrency: + type: seq + required: True + sequence: + - type: int + enabled: + type: bool + required: True + name: + type: str + required: True + times: + type: int + required: True + type: + type: str + required: True + enum: ["rally"] + scenarios: + type: seq + sequence: + - type: map + allowempty: True + mapping: + enabled: + type: bool + required: True + file: + type: str + required: True + name: + type: str + required: True diff --git a/browbeat/schema/shaker.yml b/browbeat/schema/shaker.yml new file mode 100644 index 000000000..7d7ccfb76 --- /dev/null +++ b/browbeat/schema/shaker.yml @@ -0,0 +1,19 @@ +# This schema defines how a Rally workload is formated +name: Shaker workload schema +type: map +allowempty: True +mapping: + # Required items to be a Shaker workload + enabled: + type: bool + required: True + file: + type: str + required: True + name: + type: str + required: True + type: + type: str + required: True + enum: ["shaker"] diff --git a/browbeat/schema/yoda.yml b/browbeat/schema/yoda.yml new file mode 100644 index 000000000..e2ea16564 --- /dev/null +++ b/browbeat/schema/yoda.yml @@ -0,0 +1,71 @@ +# This schema defines how a Yoda workload is formated +name: Yoda workload schema +type: map +mapping: + # Required items to be a Yoda workload + enabled: + required: True + type: bool + name: + type: str + required: True + timeout: + type: int + required: True + times: + type: int + required: True + type: + type: str + required: True + enum: ["yoda"] + yoda_type: + type: str + required: True + enum: ['introspection', 'overcloud'] + # Optional/type-specific items for Yoda workload: + batch_size: + type: int + required: False + cloud: + type: seq + required: False + sequence: + - type: map + allowempty: True + mapping: + node: + type: str + required: True + start_scale: + type: int + required: True + end_scale: + type: int + required: True + instackenv: + type: str + required: False + keep_stack: + type: bool + required: False + max_fail_amnt: + type: float + required: False + method: + type: str + required: False + node_pinning: + type: bool + required: False + ntp_server: + type: str + required: False + step: + type: int + required: False + templates: + type: seq + required: False + sequence: + - type: str diff --git a/browbeat/shaker.py b/browbeat/shaker.py index f6bf57bda..7ecb10a4d 100644 --- a/browbeat/shaker.py +++ b/browbeat/shaker.py @@ -23,16 +23,18 @@ import elastic import grafana from path import get_overcloudrc from path import get_workload_venv +from path import results_path import workloadbase import tools class Shaker(workloadbase.WorkloadBase): - def __init__(self, config): + def __init__(self, config, result_dir_ts): self.logger = logging.getLogger('browbeat.shaker') self.overcloudrc = get_overcloudrc() self.config = config + self.result_dir_ts = result_dir_ts self.tools = tools.Tools(self.config) self.grafana = grafana.Grafana(self.config) self.elastic = elastic.Elastic(self.config, self.__class__.__name__.lower()) @@ -42,24 +44,14 @@ class Shaker(workloadbase.WorkloadBase): self.scenario_count = 0 def shaker_checks(self): - cmd = "source {}; glance image-list | grep -w shaker-image".format(self.overcloudrc) + cmd = "source {}; source {}; glance image-list | grep -w shaker-image".format( + get_workload_venv('shaker', True), self.overcloudrc) if self.tools.run_cmd(cmd)['stdout'] == "": self.logger.error("Shaker Image is not built, try again") exit(1) else: self.logger.info("Shaker image is built, continuing") - def get_stats(self): - self.logger.info( - "Current number of Shaker tests executed: {}".format( - self.test_count)) - self.logger.info( - "Current number of Shaker tests passed: {}".format( - self.pass_count)) - self.logger.info( - "Current number of Shaker tests failed: {}".format( - self.error_count)) - def accommodation_to_dict(self, accommodation): accommodation_dict = {} for item in accommodation: @@ -406,57 +398,46 @@ class Shaker(workloadbase.WorkloadBase): else: self.result_check(result_dir, test_name, scenario, to_time, from_time) - def run_workloads(self): + def run_workload(self, workload, run_iteration): self.logger.info("Starting Shaker workloads") time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp)) - scenarios = self.config.get('shaker')['scenarios'] - venv = get_workload_venv('shaker', False) - default_time = 60 - self.shaker_checks() - if (scenarios is not None and len(scenarios) > 0): - for scenario in scenarios: - if scenario['enabled']: - self.update_scenarios() - self.update_total_scenarios() - shaker_uuid = uuid.uuid4() - es_ts = datetime.datetime.utcnow() - es_list = [] - if "time" in scenario: - test_time = scenario['time'] - else: - test_time = default_time - for interval in range(0, test_time + 9): - es_list.append( - datetime.datetime.utcnow() + - datetime.timedelta(0, interval)) - for run in range(self.config['browbeat']['rerun']): - self.logger.info("Scenario: {}".format(scenario['name'])) - self.logger.info("Run: {}".format(run)) - fname = os.path.join(venv, scenario['file']) - self.set_scenario(scenario, fname, default_time) - self.logger.debug("Set Scenario File: {}".format( - fname)) - result_dir = self.tools.create_results_dir( - self.config['browbeat'][ - 'results'], time_stamp, "shaker", - scenario['name'] + "-" + str(run)) - workload = self.__class__.__name__ - self.workload_logger(result_dir, workload) - time_stamp1 = datetime.datetime.now().strftime( - "%Y%m%d-%H%M%S") - test_name = "{}-browbeat-{}-{}-{}".format( - time_stamp1, "shaker", scenario['name'], run) - self.run_scenario( - scenario, result_dir, test_name, fname, shaker_uuid, - es_ts, es_list, run) - self.get_stats() - else: - self.logger.info( - "Skipping {} as scenario enabled: false".format( - scenario['name'])) - self.final_stats(self.scenario_count) - else: - self.logger.error( - "Configuration file contains no shaker scenarios") + venv = get_workload_venv('shaker', False) + self.shaker_checks() + + self.update_scenarios() + self.update_total_scenarios() + shaker_uuid = uuid.uuid4() + es_ts = datetime.datetime.utcnow() + es_list = [] + # Default test time to 60 + test_time = workload.get("time", 60) + for interval in range(0, test_time + 9): + es_list.append( + datetime.datetime.utcnow() + + datetime.timedelta(0, interval)) + + rerun_range = range(self.config["browbeat"]["rerun"]) + if self.config["browbeat"]["rerun_type"] == "complete": + # Compelete rerun type means force + rerun_range = range(run_iteration, run_iteration + 1) + + for run in rerun_range: + self.logger.info("Scenario: {}".format(workload['name'])) + self.logger.info("Run: {}".format(run)) + fname = os.path.join(venv, workload['file']) + self.set_scenario(workload, fname, 60) + self.logger.debug("Set Scenario File: {}".format(fname)) + result_dir = self.tools.create_results_dir( + results_path, self.result_dir_ts, "shaker", + workload['name'] + "-" + str(run)) + self.workload_logger(self.__class__.__name__) + time_stamp1 = datetime.datetime.now().strftime( + "%Y%m%d-%H%M%S") + test_name = "{}-browbeat-{}-{}-{}".format( + time_stamp1, "shaker", workload['name'], run) + self.run_scenario( + workload, result_dir, test_name, fname, shaker_uuid, + es_ts, es_list, run) + self.final_stats(self.scenario_count) diff --git a/browbeat/tools.py b/browbeat/tools.py index f99546d2d..2bce3bee6 100644 --- a/browbeat/tools.py +++ b/browbeat/tools.py @@ -10,14 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import logging import os import re import subprocess -import yaml - -from pykwalify import core as pykwalify_core -from pykwalify import errors as pykwalify_errors import perfkit import rally @@ -27,10 +24,9 @@ import yoda class Tools(object): - def __init__(self, config=None): + def __init__(self, config): self.logger = logging.getLogger('browbeat.tools') self.config = config - return None # Returns true if ping successful, false otherwise def is_pingable(self, ip): @@ -88,47 +84,24 @@ class Tools(object): return False return the_directory - def _load_config(self, path, validate=True): - try: - stream = open(path, 'r') - except IOError: - self.logger.error( - "Configuration file {} passed is missing".format(path)) - exit(1) - config = yaml.safe_load(stream) - stream.close() - self.config = config - if validate: - self.validate_yaml() - return config + def run_workload(self, workload, result_dir_ts, run_iteration): + """Creates workload object and runs a specific workload. - def validate_yaml(self): - self.logger.info( - "Validating the configuration file passed by the user") - stream = open("browbeat/validate.yaml", 'r') - schema = yaml.safe_load(stream) - check = pykwalify_core.Core( - source_data=self.config, schema_data=schema) - try: - check.validate(raise_exception=True) - self.logger.info("Validation successful") - except pykwalify_errors.SchemaError as e: - self.logger.error("Schema Validation failed") - raise Exception('File does not conform to schema: {}'.format(e)) - - def _run_workload_provider(self, provider): - self.logger = logging.getLogger('browbeat') - if provider == "perfkit": - workloads = perfkit.PerfKit(self.config) - elif provider == "rally": - workloads = rally.Rally(self.config) - elif provider == "shaker": - workloads = shaker.Shaker(self.config) - elif provider == "yoda": - workloads = yoda.Yoda(self.config) + :param workload: Dictionary of workload attributes defined by browbeat config + :param result_dir_ts: Result directory timestamp + :param run_iteration: Iteration for a specific run + """ + if workload["type"] == "perfkit": + workloads = perfkit.PerfKit(self.config, result_dir_ts) + elif workload["type"] == "rally": + workloads = rally.Rally(self.config, result_dir_ts) + elif workload["type"] == "shaker": + workloads = shaker.Shaker(self.config, result_dir_ts) + elif workload["type"] == "yoda": + workloads = yoda.Yoda(self.config, result_dir_ts) else: - self.logger.error("Unknown workload provider: {}".format(provider)) - workloads.run_workloads() + self.logger.error("Unknown workload provider: {}".format(workload["type"])) + workloads.run_workload(copy.deepcopy(workload), run_iteration) def check_metadata(self): meta = self.config['elasticsearch']['metadata_files'] @@ -140,12 +113,11 @@ class Tools(object): return True def gather_metadata(self): - os.putenv("ANSIBLE_SSH_ARGS", - " -F {}".format(self.config['ansible']['ssh_config'])) + os.putenv("ANSIBLE_SSH_ARGS", " -F {}".format(self.config['ansible']['ssh_config'])) ansible_cmd = \ 'ansible-playbook -i {} {}' \ - .format(self.config['ansible']['hosts'], self.config['ansible']['metadata']) + .format(self.config['ansible']['hosts'], self.config['ansible']['metadata_playbook']) self.run_cmd(ansible_cmd) if not self.check_metadata(): self.logger.warning("Metadata could not be gathered") diff --git a/browbeat/validate.yaml b/browbeat/validate.yaml deleted file mode 100644 index 211036516..000000000 --- a/browbeat/validate.yaml +++ /dev/null @@ -1,331 +0,0 @@ -name: Browbeat configuration schema -type: map -mapping: - browbeat: - required: True - type: map - mapping: - results: - type: str - required: True - rerun: - type: int - required: True - cloud_name: - type: str - required: True - - elasticsearch: - required: True - type: map - mapping: - enabled: - type: bool - required: True - host: - type: str - required: True - pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ - regather: - type: bool - required: True - port: - type: int - required: True - metadata_files: - type: seq - required: True - sequence: - - type: map - allowempty: True - mapping: - name: - type: str - required: True - file: - type: str - required: True - - ansible: - required: True - type: map - allowempty: True - mapping: - ssh_config: - type: str - required: True - hosts: - type: str - required: True - adjust: - type: map - mapping: - keystone_token: - type: str - neutron_l3: - type: str - nova_db: - type: str - workers: - type: str - shaker_build: - type: str - metadata: - type: str - required: True - - grafana: - required: True - type: map - mapping: - enabled: - type: bool - required: True - grafana_ip: - type: str - pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ - grafana_port: - type: int - dashboards: - type: seq - sequence: - - type: str - - perfkit: - required: False - type: map - allowempty: True - mapping: - enabled: - type: bool - required: True - sleep_before: - type: number - required: True - sleep_after: - type: number - required: True - default: - type: map - required: True - mapping: - image: - type: str - required: True - machine_type: - type: str - required: True - os_type: - type: str - required: True - enum: ['rhel', 'debian', 'ubuntu_container', 'windows'] - openstack_image_username: - type: str - required: True - openstack_floating_ip_pool: - type: str - required: True - openstack_network: - type: str - required: True - ignore_package_requirements: - type: bool - required: False - timing_measurements: - type: str - required: False - benchmarks: - type: seq - sequence: - - type: map - allowempty: True - mapping: - name: - type: str - required: True - enabled: - type: bool - required: True - benchmarks: - type: str - required: True - - shaker: - required: False - allowempty: True - type: map - mapping: - enabled: - type: bool - required: True - server: - type: str - required: True - pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ - port: - type: int - required: True - flavor: - type: str - required: True - join_timeout: - type: int - required: True - sleep_before: - type: number - required: True - sleep_after: - type: number - required: True - shaker_region: - type: str - required: true - external_host: - type: str - required: False - pattern: ^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$ - scenarios: - type: seq - sequence: - - type: map - allowempty: True - mapping: - name: - type: str - required: True - enabled: - type: bool - required: True - file: - type: str - required: True - - rally: - required: False - type: map - allowempty: True - mapping: - enabled: - type: bool - required: True - sleep_before: - type: number - required: True - sleep_after: - type: number - required: True - benchmarks: - type: seq - required: True - sequence: - - type: map - mapping: - name: - type: str - required: True - enabled: - required: True - type: bool - concurrency: - type: seq - required: True - sequence: - - type: int - times: - type: int - required: True - scenarios: - type: seq - sequence: - - type: map - allowempty: True - mapping: - name: - type: str - required: True - enabled: - type: bool - required: True - file: - type: str - required: True - - yoda: - required: False - type: map - allowempty: True - mapping: - enabled: - type: bool - required: True - instackenv: - type: str - required: True - stackrc: - type: str - required: True - benchmarks: - type: seq - required: True - sequence: - - type: map - mapping: - name: - type: str - required: True - type: - type: str - required: True - enabled: - required: True - type: bool - ntp_server: - type: str - required: False - templates: - type: seq - required: False - sequence: - - type: str - instackenv: - type: str - required: false - times: - type: int - required: True - step: - type: int - required: False - method: - type: str - required: False - timeout: - type: int - required: True - max_fail_amnt: - type: float - required: False - batch_size: - type: int - required: False - keep_stack: - type: bool - required: False - node_pinning: - type: bool - required: False - cloud: - type: seq - sequence: - - type: map - allowempty: True - mapping: - node: - type: str - required: True - start_scale: - type: int - required: True - end_scale: - type: int - required: True diff --git a/browbeat/workloadbase.py b/browbeat/workloadbase.py index c2400fff6..c5e9d22ea 100644 --- a/browbeat/workloadbase.py +++ b/browbeat/workloadbase.py @@ -15,6 +15,8 @@ import logging import os import yaml +from browbeat.path import results_path + class WorkloadBase(object): __metaclass__ = abc.ABCMeta @@ -26,8 +28,11 @@ class WorkloadBase(object): index_failures = 0 browbeat = {} + def __init__(self): + self.result_dir_ts = "" + @abc.abstractmethod - def run_workloads(self): + def run_workload(self, workload, run_iteration): pass @abc.abstractmethod @@ -61,17 +66,15 @@ class WorkloadBase(object): def update_index_failures(self): WorkloadBase.index_failures += 1 - def workload_logger(self, result_dir, workload): - base = result_dir.split('/') - if not os.path.isfile("{}/{}/browbeat-{}-run.log".format(base[0], base[1], workload)): - file = logging.FileHandler( - "{}/{}/browbeat-{}-run.log".format(base[0], base[1], workload)) - file.setLevel(logging.DEBUG) - formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)5s - %(message)s') - file.setFormatter(formatter) - self.logger.addHandler(file) - return None + def workload_logger(self, workload): + workload_result_dir = os.path.join(results_path, self.result_dir_ts) + if not os.path.isfile("{}/browbeat-{}-run.log".format(workload_result_dir, workload)): + filehandler = logging.FileHandler( + "{}/browbeat-{}-run.log".format(workload_result_dir, workload)) + filehandler.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)5s - %(message)s') + filehandler.setFormatter(formatter) + self.logger.addHandler(filehandler) def get_time_dict(self, to_time, from_time, benchmark, test_name, workload, status, index_status="disabled"): diff --git a/browbeat/yoda.py b/browbeat/yoda.py index b346bbccd..4e27d45be 100644 --- a/browbeat/yoda.py +++ b/browbeat/yoda.py @@ -30,15 +30,17 @@ import requests import elastic import grafana +from path import results_path import workloadbase import tools class Yoda(workloadbase.WorkloadBase): - def __init__(self, config): + def __init__(self, config, result_dir_ts): self.logger = logging.getLogger('browbeat.yoda') self.config = config + self.result_dir_ts = result_dir_ts self.tools = tools.Tools(self.config) self.grafana = grafana.Grafana(self.config) self.elastic = elastic.Elastic(self.config, self.__class__.__name__.lower()) @@ -497,7 +499,7 @@ class Yoda(workloadbase.WorkloadBase): def setup_scenario(self, benchmark_name, dir_ts): results_dir = self.tools.create_results_dir( - self.config['browbeat']['results'], dir_ts, benchmark_name, benchmark_name) + results_path, dir_ts, benchmark_name, benchmark_name) if isinstance(results_dir, bool): self.logger.error( @@ -505,8 +507,7 @@ class Yoda(workloadbase.WorkloadBase): exit(1) self.logger.debug("Created result directory: {}".format(results_dir)) - workload = self.__class__.__name__ - self.workload_logger(results_dir, workload) + self.workload_logger(self.__class__.__name__) return results_dir def introspection_workload( @@ -640,7 +641,7 @@ class Yoda(workloadbase.WorkloadBase): results = out[0] changed = out[1] - def run_workloads(self): + def run_workload(self, workload, run_iteration): """Iterates through all yoda scenarios in browbeat yaml config file""" self.logger.info("Starting YODA workloads") es_ts = datetime.datetime.utcnow() @@ -667,39 +668,24 @@ class Yoda(workloadbase.WorkloadBase): conn = connection.Connection(**auth_args) instackenv = self.config.get('yoda')['instackenv'] - benchmarks = self.config.get('yoda')['benchmarks'] - if (benchmarks is not None and len(benchmarks) > 0): - for benchmark in benchmarks: - if benchmark['enabled']: - results_dir = self.setup_scenario( - benchmark['name'], dir_ts) - times = benchmark['times'] - if 'instackenv' not in benchmark: - benchmark['instackenv'] = instackenv - for rerun in range(self.config['browbeat']['rerun']): - for run in range(times): - if benchmark['type'] == "overcloud": - self.overcloud_workload(benchmark, - run, - results_dir, - env_setup, - conn) - elif benchmark['type'] == "introspection": - self.introspection_workload(benchmark, - run, - results_dir, - env_setup, - conn) - else: - self.logger.error( - "Could not identify YODA workload!") - exit(1) - self.update_scenarios() + results_dir = self.setup_scenario(workload['name'], dir_ts) + times = workload['times'] + if 'instackenv' not in workload: + workload['instackenv'] = instackenv + # Correct iteration/rerun + rerun_range = range(self.config["browbeat"]["rerun"]) + if self.config["browbeat"]["rerun_type"] == "complete": + rerun_range = range(run_iteration, run_iteration + 1) + + for run in rerun_range: + for run in range(times): + if workload['yoda_type'] == "overcloud": + self.overcloud_workload(workload, run, results_dir, env_setup, conn) + elif workload['yoda_type'] == "introspection": + self.introspection_workload(workload, run, results_dir, env_setup, conn) else: - self.logger.info( - "Skipping {} benchmarks enabled: false".format( - benchmark['name'])) - else: - self.logger.error("Config file contains no yoda benchmarks.") + self.logger.error("Could not identify YODA workload!") + exit(1) + self.update_scenarios() diff --git a/ci-scripts/config/browbeat-ci.yaml b/ci-scripts/config/browbeat-ci.yaml deleted file mode 100644 index bc85dc9ef..000000000 --- a/ci-scripts/config/browbeat-ci.yaml +++ /dev/null @@ -1,237 +0,0 @@ -# Tests to be compleated for the install-and-check.sh script minimal and short workloads are performed -# to confirm functionality. -browbeat: - results : results/ - rerun: 1 - cloud_name: openstack -elasticsearch: - enabled: false - host: 1.1.1.1 - port: 9200 - regather: true - metadata_files: - - name: hardware-metadata - file: metadata/hardware-metadata.json - - name: environment-metadata - file: metadata/environment-metadata.json - - name: software-metadata - file: metadata/software-metadata.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml -grafana: - enabled: true - grafana_ip: 1.1.1.1 - grafana_port: 3000 - dashboards: - - openstack-general-system-performance -perfkit: - enabled: true - sleep_before: 0 - sleep_after: 0 - venv: /home/stack/perfkit-venv/bin/activate - default: - image: centos7 - machine_type: m1.small - os_type: rhel - openstack_image_username: centos - openstack_floating_ip_pool: browbeat_public - openstack_network: browbeat_private - benchmarks: - - name: fio-centos-m1-small - enabled: false - benchmarks: fio - data_disk_size: 4 -rally: - enabled: true - sleep_before: 5 - sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate - plugins: - - netcreate-boot: rally/rally-plugins/netcreate-boot - - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping - - subnet-router-create: rally/rally-plugins/subnet-router-create - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 16 - - 8 - - 16 - times: 50 - scenarios: - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - sla_max_avg_duration: 6 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: cinder - enabled: true - concurrency: - - 1 - - 2 - times: 4 - scenarios: - - name: create-attach-volume-centos - enabled: true - file: rally/cinder/cinder-create-and-attach-volume-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: keystonebasic - enabled: false - concurrency: - - 8 - times: 50 - scenarios: - - name: create-and-list-tenants - enabled: true - file: rally/keystonebasic/create_and_list_tenants-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - times: 50 - - name: neutron - enabled: true - concurrency: - - 2 - - 4 - times: 10 - scenarios: - - name: create-list-network - enabled: true - file: rally/neutron/neutron-create-list-network-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: nova - enabled: true - concurrency: - - 1 - - 2 - times: 16 - scenarios: - - name: boot-snapshot-delete - enabled: true - file: rally/nova/nova-boot-snapshot-cc.yml - image_name: cirros - flavor_name: m1.tiny - - name: plugins - enabled: true - concurrency: - - 8 - - 16 - times: 16 - scenarios: - - name: netcreate-boot - enabled: true - image_name: cirros - flavor_name: m1.tiny - file: rally/rally-plugins/netcreate-boot/netcreate_boot.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 -#shaker scenarios require atleast 2 compute nodes -shaker: - enabled: true - server: localhost - port: 5555 - flavor: m1.small - join_timeout: 600 - sleep_before: 5 - sleep_after: 5 - venv: /home/stack/shaker-venv - dns_nameserver: 192.168.23.1 - shaker_region: regionOne - scenarios: - - name: l2-4-1 - enabled: true - density: 4 - compute: 1 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l2-8-1 - enabled: true - density: 8 - compute: 1 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l2-4-2 - enabled: true - density: 4 - compute: 2 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l2-4-8 - enabled: true - density: 8 - compute: 2 - progression: linear - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml - - name: l3-north-south-4-1 - enabled: true - placement: single_room - density: 4 - compute: 1 - progression: null - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-north-south-8-1 - enabled: false - placement: single_room - density: 8 - compute: 1 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-north-south-4-2 - enabled: true - placement: single_room - density: 4 - compute: 2 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-north-south-8-2 - enabled: true - placement: single_room - density: 8 - compute: 2 - progression: null - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_north_south.yaml - - name: l3-east-west-4-1 - enabled: true - density: 4 - compute: 1 - placement: single_room - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml - - name: l3-east-west-8-1 - enabled: true - density: 8 - compute: 1 - placement: single_room - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml - - name: l3-east-west-4-2 - enabled: true - density: 4 - compute: 2 - placement: single_room - time: 60 - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml - - name: l3-east-west-8-2 - enabled: true - density: 8 - compute: 2 - time: 60 - placement: single_room - file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l3_east_west.yaml diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml index 68a46df4c..e6f5d325f 100644 --- a/conf/browbeat-glance-complete.yaml +++ b/conf/browbeat-glance-complete.yaml @@ -1,13 +1,17 @@ # Complete set of Glance Benchmarks browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -17,49 +21,65 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: true - grafana_ip: 1.1.1.1 - grafana_port: 3000 + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: - - glance-create-boot-delete: rally/rally-plugins/glance-create-boot-delete - benchmarks: - - name: glance - enabled: true - concurrency: - - 8 - times: 10 - scenarios: - - name: create-image-boot-instances - enabled: true - image_location: /home/stack/cirros.qcow2 - instances: 2 - flavor_name: m1.small - file: rally/glance/create-image-and-boot-instances-cc.yml - - name: create-and-delete-image - enabled: true - image_location: /home/stack/cirros.qcow2 - flavor_name: m1.small - file: rally/glance/create-and-delete-image-cc.yml - - name: create-and-list-image - enabled: true - image_location: /home/stack/cirros.qcow2 - file: rally/glance/create-and-list-image-cc.yml - - name: list-images - enabled: true - file: rally/glance/list-images-cc.yml + - glance: rally/rally-plugins/glance +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: glance + enabled: true + type: rally + concurrency: + - 8 + times: 10 + scenarios: + - name: create-image-boot-instances + enabled: true + image_location: /home/stack/cirros.qcow2 + instances: 2 + flavor_name: m1.small + file: rally/glance/create-image-and-boot-instances-cc.yml + - name: create-and-delete-image + enabled: true + image_location: /home/stack/cirros.qcow2 + flavor_name: m1.small + file: rally/glance/create-and-delete-image-cc.yml + - name: create-and-list-image + enabled: true + image_location: /home/stack/cirros.qcow2 + file: rally/glance/create-and-list-image-cc.yml + - name: list-images + enabled: true + file: rally/glance/list-images-cc.yml diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml index e08ca5252..5951eb9e9 100644 --- a/conf/browbeat-keystone-complete.yaml +++ b/conf/browbeat-keystone-complete.yaml @@ -1,14 +1,18 @@ # Complete set of Keystone Stress tests run 3 times over multiple concurrencies with high repetitions(times) # This testing will take days to complete. (You have been warned) browbeat: - results: results/ - rerun: 3 cloud_name: openstack + rerun: 3 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -18,197 +22,215 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 64 - - 128 - - 192 - - 256 - - 320 - - 384 - - 448 - - 512 - times: 10000 - scenarios: - - name: authentic-ceilometer - enabled: true - file: rally/authenticate/validate_ceilometer-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-cinder - enabled: true - file: rally/authenticate/validate_cinder-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-glance - enabled: true - file: rally/authenticate/validate_glance-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-heat - enabled: true - file: rally/authenticate/validate_heat-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - sla_max_avg_duration: 6 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-monasca - enabled: false - file: rally/authenticate/validate_monasca-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-neutron - enabled: true - file: rally/authenticate/validate_neutron-cc.yml - sla_max_avg_duration: 18 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-nova - enabled: true - file: rally/authenticate/validate_nova-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: keystonebasic - enabled: true - concurrency: - - 128 - - 256 - times: 5000 - scenarios: - - name: create-user - enabled: true - file: rally/keystonebasic/create_user-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-delete-user - enabled: true - file: rally/keystonebasic/create_delete_user-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-user-set-enabled-and-delete - enabled: true - enabled_flag: true - file: rally/keystonebasic/create_user_set_enabled_and_delete-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-tenant - enabled: true - file: rally/keystonebasic/create_tenant-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authenticate_user_and_validate_token - enabled: true - file: rally/keystonebasic/authenticate_user_and_validate_token-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-tenant-with-users - enabled: true - file: rally/keystonebasic/create_tenant_with_users-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-users - enabled: true - file: rally/keystonebasic/create_and_list_users-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-tenants - enabled: true - file: rally/keystonebasic/create_and_list_tenants-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: add-and-remove-user-role - enabled: true - file: rally/keystonebasic/add_and_remove_user_role-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-delete-role - enabled: true - file: rally/keystonebasic/create_and_delete_role-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-add-and-list-user-roles - enabled: true - file: rally/keystonebasic/create_add_and_list_user_roles-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: get-entities - enabled: true - file: rally/keystonebasic/get_entities-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-delete-service - enabled: true - file: rally/keystonebasic/create_and_delete_service-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-update-and-delete-tenant - enabled: true - file: rally/keystonebasic/create_update_and_delete_tenant-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-user-update-password - enabled: true - file: rally/keystonebasic/create_user_update_password-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-services - enabled: true - file: rally/keystonebasic/create_and_list_services-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-ec2credentials - enabled: true - file: rally/keystonebasic/create_and_list_ec2credentials-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-delete-ec2credentials - enabled: true - file: rally/keystonebasic/create_and_delete_ec2credentials-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-get-role - enabled: true - file: rally/keystonebasic/create_and_get_role-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-add-list-roles - enabled: true - file: rally/keystonebasic/create_add_list_roles-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-update-user - enabled: true - file: rally/keystonebasic/create_and_update_user-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 + plugins: [] +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: authenticate + enabled: true + type: rally + concurrency: + - 64 + - 128 + - 192 + - 256 + - 320 + - 384 + - 448 + - 512 + times: 10000 + scenarios: + - name: authentic-ceilometer + enabled: true + file: rally/authenticate/validate_ceilometer-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-cinder + enabled: true + file: rally/authenticate/validate_cinder-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-glance + enabled: true + file: rally/authenticate/validate_glance-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-heat + enabled: true + file: rally/authenticate/validate_heat-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + sla_max_avg_duration: 6 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-monasca + enabled: false + file: rally/authenticate/validate_monasca-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-neutron + enabled: true + file: rally/authenticate/validate_neutron-cc.yml + sla_max_avg_duration: 18 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-nova + enabled: true + file: rally/authenticate/validate_nova-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: keystonebasic + enabled: true + type: rally + concurrency: + - 128 + - 256 + times: 5000 + scenarios: + - name: create-user + enabled: true + file: rally/keystonebasic/create_user-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-delete-user + enabled: true + file: rally/keystonebasic/create_delete_user-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-user-set-enabled-and-delete + enabled: true + enabled_flag: true + file: rally/keystonebasic/create_user_set_enabled_and_delete-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-tenant + enabled: true + file: rally/keystonebasic/create_tenant-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authenticate_user_and_validate_token + enabled: true + file: rally/keystonebasic/authenticate_user_and_validate_token-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-tenant-with-users + enabled: true + file: rally/keystonebasic/create_tenant_with_users-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-users + enabled: true + file: rally/keystonebasic/create_and_list_users-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-tenants + enabled: true + file: rally/keystonebasic/create_and_list_tenants-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: add-and-remove-user-role + enabled: true + file: rally/keystonebasic/add_and_remove_user_role-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-delete-role + enabled: true + file: rally/keystonebasic/create_and_delete_role-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-add-and-list-user-roles + enabled: true + file: rally/keystonebasic/create_add_and_list_user_roles-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: get-entities + enabled: true + file: rally/keystonebasic/get_entities-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-delete-service + enabled: true + file: rally/keystonebasic/create_and_delete_service-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-update-and-delete-tenant + enabled: true + file: rally/keystonebasic/create_update_and_delete_tenant-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-user-update-password + enabled: true + file: rally/keystonebasic/create_user_update_password-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-services + enabled: true + file: rally/keystonebasic/create_and_list_services-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-ec2credentials + enabled: true + file: rally/keystonebasic/create_and_list_ec2credentials-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-delete-ec2credentials + enabled: true + file: rally/keystonebasic/create_and_delete_ec2credentials-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-get-role + enabled: true + file: rally/keystonebasic/create_and_get_role-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-add-list-roles + enabled: true + file: rally/keystonebasic/create_add_list_roles-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-update-user + enabled: true + file: rally/keystonebasic/create_and_update_user-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml index da0bfd3e5..8ae0f57fb 100644 --- a/conf/browbeat-keystone-minimal.yaml +++ b/conf/browbeat-keystone-minimal.yaml @@ -1,14 +1,18 @@ # Complete set of Keystone Stress tests run minimal number of times. # Good sanity test to see if OSP will be able to handle the longer stress tests. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -18,190 +22,208 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 - benchmarks: - - name: authenticate - enabled: true - concurrency: - - 32 - times: 100 - scenarios: - - name: authentic-ceilometer - enabled: true - file: rally/authenticate/validate_ceilometer-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-cinder - enabled: true - file: rally/authenticate/validate_cinder-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-glance - enabled: true - file: rally/authenticate/validate_glance-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-heat - enabled: true - file: rally/authenticate/validate_heat-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-keystone - enabled: true - file: rally/authenticate/keystone-cc.yml - sla_max_avg_duration: 6 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-monasca - enabled: false - file: rally/authenticate/validate_monasca-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-neutron - enabled: true - file: rally/authenticate/validate_neutron-cc.yml - sla_max_avg_duration: 18 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authentic-nova - enabled: true - file: rally/authenticate/validate_nova-cc.yml - sla_max_avg_duration: 12 - sla_max_seconds: 30 - sla_max_failure: 0 - - name: keystonebasic - enabled: true - concurrency: - - 25 - times: 100 - scenarios: - - name: create-user - enabled: true - file: rally/keystonebasic/create_user-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-delete-user - enabled: true - file: rally/keystonebasic/create_delete_user-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-user-set-enabled-and-delete - enabled: true - enabled_flag: true - file: rally/keystonebasic/create_user_set_enabled_and_delete-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-tenant - enabled: true - file: rally/keystonebasic/create_tenant-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: authenticate_user_and_validate_token - enabled: true - file: rally/keystonebasic/authenticate_user_and_validate_token-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-tenant-with-users - enabled: true - users_per_tenant: 2 - file: rally/keystonebasic/create_tenant_with_users-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-users - enabled: true - file: rally/keystonebasic/create_and_list_users-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-tenants - enabled: true - file: rally/keystonebasic/create_and_list_tenants-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: add-and-remove-user-role - enabled: true - file: rally/keystonebasic/add_and_remove_user_role-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-delete-role - enabled: true - file: rally/keystonebasic/create_and_delete_role-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-add-and-list-user-roles - enabled: true - file: rally/keystonebasic/create_add_and_list_user_roles-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: get-entities - enabled: true - file: rally/keystonebasic/get_entities-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-delete-service - enabled: true - file: rally/keystonebasic/create_and_delete_service-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-update-and-delete-tenant - enabled: true - file: rally/keystonebasic/create_update_and_delete_tenant-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-user-update-password - enabled: true - file: rally/keystonebasic/create_user_update_password-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-services - enabled: true - file: rally/keystonebasic/create_and_list_services-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-list-ec2credentials - enabled: true - file: rally/keystonebasic/create_and_list_ec2credentials-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-delete-ec2credentials - enabled: true - file: rally/keystonebasic/create_and_delete_ec2credentials-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-get-role - enabled: true - file: rally/keystonebasic/create_and_get_role-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-add-list-roles - enabled: true - file: rally/keystonebasic/create_add_list_roles-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 - - name: create-and-update-user - enabled: true - file: rally/keystonebasic/create_and_update_user-cc.yml - sla_max_seconds: 30 - sla_max_failure: 0 + plugins: [] +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: authenticate + enabled: true + type: rally + concurrency: + - 32 + times: 100 + scenarios: + - name: authentic-ceilometer + enabled: true + file: rally/authenticate/validate_ceilometer-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-cinder + enabled: true + file: rally/authenticate/validate_cinder-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-glance + enabled: true + file: rally/authenticate/validate_glance-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-heat + enabled: true + file: rally/authenticate/validate_heat-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + sla_max_avg_duration: 6 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-monasca + enabled: false + file: rally/authenticate/validate_monasca-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-neutron + enabled: true + file: rally/authenticate/validate_neutron-cc.yml + sla_max_avg_duration: 18 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authentic-nova + enabled: true + file: rally/authenticate/validate_nova-cc.yml + sla_max_avg_duration: 12 + sla_max_seconds: 30 + sla_max_failure: 0 + - name: keystonebasic + enabled: true + type: rally + concurrency: + - 25 + times: 100 + scenarios: + - name: create-user + enabled: true + file: rally/keystonebasic/create_user-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-delete-user + enabled: true + file: rally/keystonebasic/create_delete_user-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-user-set-enabled-and-delete + enabled: true + enabled_flag: true + file: rally/keystonebasic/create_user_set_enabled_and_delete-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-tenant + enabled: true + file: rally/keystonebasic/create_tenant-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: authenticate_user_and_validate_token + enabled: true + file: rally/keystonebasic/authenticate_user_and_validate_token-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-tenant-with-users + enabled: true + users_per_tenant: 2 + file: rally/keystonebasic/create_tenant_with_users-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-users + enabled: true + file: rally/keystonebasic/create_and_list_users-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-tenants + enabled: true + file: rally/keystonebasic/create_and_list_tenants-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: add-and-remove-user-role + enabled: true + file: rally/keystonebasic/add_and_remove_user_role-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-delete-role + enabled: true + file: rally/keystonebasic/create_and_delete_role-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-add-and-list-user-roles + enabled: true + file: rally/keystonebasic/create_add_and_list_user_roles-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: get-entities + enabled: true + file: rally/keystonebasic/get_entities-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-delete-service + enabled: true + file: rally/keystonebasic/create_and_delete_service-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-update-and-delete-tenant + enabled: true + file: rally/keystonebasic/create_update_and_delete_tenant-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-user-update-password + enabled: true + file: rally/keystonebasic/create_user_update_password-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-services + enabled: true + file: rally/keystonebasic/create_and_list_services-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-list-ec2credentials + enabled: true + file: rally/keystonebasic/create_and_list_ec2credentials-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-delete-ec2credentials + enabled: true + file: rally/keystonebasic/create_and_delete_ec2credentials-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-get-role + enabled: true + file: rally/keystonebasic/create_and_get_role-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-add-list-roles + enabled: true + file: rally/keystonebasic/create_add_list_roles-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 + - name: create-and-update-user + enabled: true + file: rally/keystonebasic/create_and_update_user-cc.yml + sla_max_seconds: 30 + sla_max_failure: 0 diff --git a/conf/browbeat-workloads.yml b/conf/browbeat-workloads.yml index f1c9fb90c..fecc24a14 100644 --- a/conf/browbeat-workloads.yml +++ b/conf/browbeat-workloads.yml @@ -1,13 +1,17 @@ # Complete set of Workload Benchmarks browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -17,60 +21,76 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: enabled: true - grafana_ip: 1.1.1.1 - grafana_port: 3000 + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: - workloads: rally/rally-plugins/workloads - benchmarks: - - name: workloads - enabled: true - concurrency: - - 1 - times: 1 - scenarios: - - name: browbeat-linpack - enabled: true - image_name: browbeat-linpack - flavor_name: m1.small - external_network: - net_id: - file: rally/rally-plugins/workloads/linpack.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - - name: browbeat-pbench-uperf - enabled: true - user: root - image_name: browbeat-uperf - flavor_name: m1.small - # hypervisor_server: "nova:overcloud-compute-1.localdomain" - # hypervisor_client: "nova:overcloud-compute-0.localdomain" - external_network: - net_id: - protocols: tcp,udp - instances: 1 - num_pairs: 1 - samples: 1 - test_types: stream,rr - message_sizes: 64,1024,16384 - test_name: browbeat-pbench-uperf - send_results: true - cloudname: - elastic_host: - elastic_port: 9200 - file: rally/rally-plugins/workloads/pbench-uperf.yml +workloads: + - name: workloads + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: browbeat-linpack + enabled: true + image_name: browbeat-linpack + flavor_name: m1.small + external_network: + net_id: + file: rally/rally-plugins/workloads/linpack.yml + + - name: browbeat-pbench-uperf + enabled: true + user: root + image_name: browbeat-uperf + flavor_name: m1.small + # hypervisor_server: "nova:overcloud-compute-1.localdomain" + # hypervisor_client: "nova:overcloud-compute-0.localdomain" + external_network: + net_id: + protocols: tcp,udp + instances: 1 + num_pairs: 1 + samples: 1 + test_types: stream,rr + message_sizes: 64,1024,16384 + test_name: browbeat-pbench-uperf + send_results: true + cloudname: + elastic_host: + elastic_port: 9200 + file: rally/rally-plugins/workloads/pbench-uperf.yml diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml index 81c1fa81d..8bfefe03e 100644 --- a/conf/ceilometer-minimal.yaml +++ b/conf/ceilometer-minimal.yaml @@ -1,13 +1,17 @@ # Ceilometer Stress tests run minimal number of times. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -17,151 +21,175 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: [] - benchmarks: - - name: CeilometerAlarms - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: create_alarm - enabled: true - file: rally/CeilometerAlarms/create_alarm-cc.yml - - name: create_alarm_and_get_history - enabled: true - file: rally/CeilometerAlarms/create_alarm_and_get_history-cc.yml - - name: create_and_delete_alarm - enabled: true - file: rally/CeilometerAlarms/create_and_delete_alarm-cc.yml - - name: create_and_list_alarm - enabled: true - file: rally/CeilometerAlarms/create_and_list_alarm-cc.yml - - name: create_and_update_alarm - enabled: true - file: rally/CeilometerAlarms/create_and_update_alarm-cc.yml - - name: list_alarms - enabled: true - file: rally/CeilometerAlarms/list_alarms-cc.yml - - name: CeilometerEvents - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: create_user_and_get_event - enabled: true - file: rally/CeilometerEvents/create_user_and_get_event-cc.yml - - name: create_user_and_list_event_types - enabled: true - file: rally/CeilometerEvents/create_user_and_list_event_types-cc.yml - - name: create_user_and_list_events - enabled: true - file: rally/CeilometerEvents/create_user_and_list_events-cc.yml - - name: CeilometerMeters - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: list_matched_meters - enabled: true - file: rally/CeilometerMeters/list_matched_meters-cc.yml - - name: list_meters - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: CeilometerQueries - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: create_and_query_alarm_history - enabled: true - file: rally/CeilometerQueries/create_and_query_alarm_history-cc.yml - - name: create_and_query_alarms - enabled: true - file: rally/CeilometerQueries/create_and_query_alarms-cc.yml - - name: create_and_query_samples - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: CeilometerResource - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: get_tenant_resources - enabled: true - file: rally/CeilometerResource/get_tenant_resources-cc.yml - - name: list_matched_resources - enabled: true - file: rally/CeilometerResource/list_matched_resources-cc.yml - - name: list_resources - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: CeilometerSamples - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: list_matched_samples - enabled: true - file: rally/CeilometerSamples/list_matched_samples-cc.yml - - name: list_samples - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: CeilometerStats - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: create_meter_and_get_stats - enabled: true - file: rally/CeilometerStats/create_meter_and_get_stats-cc.yml - - name: get_stats - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - - name: CeilometerTraits - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: create_user_and_list_trait_descriptions - enabled: true - file: rally/CeilometerTraits/create_user_and_list_trait_descriptions-cc.yml - - name: create_user_and_list_traits - enabled: true - file: rally/CeilometerTraits/create_user_and_list_traits-cc.yml - # Other Scenarios involving Ceilometer: - - name: Authenticate - enabled: false - concurrency: - - 2 - times: 10 - scenarios: - - name: validate_ceilometer - enabled: true - file: rally/authenticate/validate_ceilometer-cc.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: CeilometerAlarms + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: create_alarm + enabled: true + file: rally/CeilometerAlarms/create_alarm-cc.yml + - name: create_alarm_and_get_history + enabled: true + file: rally/CeilometerAlarms/create_alarm_and_get_history-cc.yml + - name: create_and_delete_alarm + enabled: true + file: rally/CeilometerAlarms/create_and_delete_alarm-cc.yml + - name: create_and_list_alarm + enabled: true + file: rally/CeilometerAlarms/create_and_list_alarm-cc.yml + - name: create_and_update_alarm + enabled: true + file: rally/CeilometerAlarms/create_and_update_alarm-cc.yml + - name: list_alarms + enabled: true + file: rally/CeilometerAlarms/list_alarms-cc.yml + - name: CeilometerEvents + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: create_user_and_get_event + enabled: true + file: rally/CeilometerEvents/create_user_and_get_event-cc.yml + - name: create_user_and_list_event_types + enabled: true + file: rally/CeilometerEvents/create_user_and_list_event_types-cc.yml + - name: create_user_and_list_events + enabled: true + file: rally/CeilometerEvents/create_user_and_list_events-cc.yml + - name: CeilometerMeters + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: list_matched_meters + enabled: true + file: rally/CeilometerMeters/list_matched_meters-cc.yml + - name: list_meters + enabled: true + file: rally/CeilometerMeters/list_meters-cc.yml + - name: CeilometerQueries + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: create_and_query_alarm_history + enabled: true + file: rally/CeilometerQueries/create_and_query_alarm_history-cc.yml + - name: create_and_query_alarms + enabled: true + file: rally/CeilometerQueries/create_and_query_alarms-cc.yml + - name: create_and_query_samples + enabled: true + file: rally/CeilometerQueries/create_and_query_samples-cc.yml + - name: CeilometerResource + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: get_tenant_resources + enabled: true + file: rally/CeilometerResource/get_tenant_resources-cc.yml + - name: list_matched_resources + enabled: true + file: rally/CeilometerResource/list_matched_resources-cc.yml + - name: list_resources + enabled: true + file: rally/CeilometerResource/list_resources-cc.yml + - name: CeilometerSamples + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: list_matched_samples + enabled: true + file: rally/CeilometerSamples/list_matched_samples-cc.yml + - name: list_samples + enabled: true + file: rally/CeilometerSamples/list_samples-cc.yml + - name: CeilometerStats + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: create_meter_and_get_stats + enabled: true + file: rally/CeilometerStats/create_meter_and_get_stats-cc.yml + - name: get_stats + enabled: true + file: rally/CeilometerStats/get_stats-cc.yml + - name: CeilometerTraits + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: create_user_and_list_trait_descriptions + enabled: true + file: rally/CeilometerTraits/create_user_and_list_trait_descriptions-cc.yml + - name: create_user_and_list_traits + enabled: true + file: rally/CeilometerTraits/create_user_and_list_traits-cc.yml + # Other Scenarios involving Ceilometer: + - name: Authenticate + enabled: false + type: rally + concurrency: + - 2 + times: 10 + scenarios: + - name: validate_ceilometer + enabled: true + file: rally/authenticate/validate_ceilometer-cc.yml diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml index 8028892ca..967b5bb12 100644 --- a/conf/ceilometer-persist-response-timing.yaml +++ b/conf/ceilometer-persist-response-timing.yaml @@ -3,14 +3,18 @@ # each iteration it measures response time of several Ceilometer APIs. # Currently it maxs at 200 instances browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 10 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -20,387 +24,77 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: + - browbeat: rally/rally-plugins/browbeat - nova: rally/rally-plugins/nova - benchmarks: - # Ceilometer Baseline - - name: Ceilometer01 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-01 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-01 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-01 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-01 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-01 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist01 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer02 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-02 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-02 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-02 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-02 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-02 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist02 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer03 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-03 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-03 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-03 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-03 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-03 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist03 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer04 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-04 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-04 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-04 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-04 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-04 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist04 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer05 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-05 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-05 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-05 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-05 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-05 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist05 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer06 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-06 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-06 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-06 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-06 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-06 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist06 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer07 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-07 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-07 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-07 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-07 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-07 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist07 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer08 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-08 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-08 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-08 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-08 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-08 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist08 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer09 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-09 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-09 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-09 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-09 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-09 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist09 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Measure Ceilometer Response Timings - - name: Ceilometer10 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-10 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-10 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-10 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-10 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-10 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml - # Boot 20 instances - - name: BrowbeatNovaPersist10 - enabled: true - concurrency: - - 5 - times: 20 - scenarios: - - name: nova-boot-20-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - file: rally/rally-plugins/nova/nova_boot_persist.yml - # Final Measure Ceilometer Response Timings - - name: Ceilometer11 - enabled: true - concurrency: - - 8 - times: 500 - scenarios: - - name: list_meters-11 - enabled: true - file: rally/CeilometerMeters/list_meters-cc.yml - - name: create_and_query_samples-11 - enabled: true - file: rally/CeilometerQueries/create_and_query_samples-cc.yml - - name: list_resources-11 - enabled: true - file: rally/CeilometerResource/list_resources-cc.yml - - name: list_samples-11 - enabled: true - file: rally/CeilometerSamples/list_samples-cc.yml - - name: get_stats-11 - enabled: true - file: rally/CeilometerStats/get_stats-cc.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + # Measure Ceilometer Response Timings + - name: Ceilometer + enabled: true + type: rally + concurrency: + - 8 + times: 500 + scenarios: + - name: list_meters + enabled: true + file: rally/CeilometerMeters/list_meters-cc.yml + - name: create_and_query_samples-01 + enabled: true + file: rally/CeilometerQueries/create_and_query_samples-cc.yml + - name: list_resources + enabled: true + file: rally/CeilometerResource/list_resources-cc.yml + - name: list_samples + enabled: true + file: rally/CeilometerSamples/list_samples-cc.yml + - name: get_stats + enabled: true + file: rally/CeilometerStats/get_stats-cc.yml + # Boot 20 instances + - name: BrowbeatNovaPersist + enabled: true + type: rally + concurrency: + - 5 + times: 20 + scenarios: + - name: nova-boot-20 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + file: rally/rally-plugins/nova/nova_boot_persist.yml diff --git a/conf/gnocchi-minimal.yaml b/conf/gnocchi-minimal.yaml index 015906384..1245acb00 100644 --- a/conf/gnocchi-minimal.yaml +++ b/conf/gnocchi-minimal.yaml @@ -1,14 +1,18 @@ # Complete set of Gnocchi Stress tests run minimal number of times. # Good sanity test to see if OSP will be able to handle the longer stress tests. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -18,121 +22,137 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance -rally: - enabled: true +perfkit: sleep_before: 0 sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true +rally: + sleep_before: 5 + sleep_after: 5 plugins: - gnocchi: rally/rally-plugins/gnocchi - benchmarks: - - name: Gnocchi - enabled: true - concurrency: - - 5 - times: 10 - scenarios: - - name: gnocchi-archive-policy-list - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-archive-policy-list.yml - - name: gnocchi-create-archive-policy - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-archive-policy.yml - - name: gnocchi-create-delete-archive-policy - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-archive-policy.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - - name: gnocchi-archive-policy-rule-list - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-archive-policy-rule-list.yml - - name: gnocchi-create-archive-policy-rule - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-archive-policy-rule.yml - - name: gnocchi-create-delete-archive-policy-rule - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-archive-policy-rule.yml +workloads: + - name: Gnocchi + enabled: true + type: rally + concurrency: + - 5 + times: 10 + scenarios: + - name: gnocchi-archive-policy-list + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-archive-policy-list.yml + - name: gnocchi-create-archive-policy + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-archive-policy.yml + - name: gnocchi-create-delete-archive-policy + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-archive-policy.yml - - name: gnocchi-capabilities-list - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-capabilities-list.yml + - name: gnocchi-archive-policy-rule-list + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-archive-policy-rule-list.yml + - name: gnocchi-create-archive-policy-rule + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-archive-policy-rule.yml + - name: gnocchi-create-delete-archive-policy-rule + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-archive-policy-rule.yml - # Aggregation has several potential scenarios due to parameterization - - name: gnocchi-metric-aggregation-single-page - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-aggregation.yml - - name: gnocchi-metric-aggregation-all - enabled: true - all_metrics: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-aggregation.yml - - name: gnocchi-metric-aggregation-agg-mean-refresh - enabled: true - all_metrics: true - aggregation: mean - refresh: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-aggregation.yml - # Get Measures has several potential scenarios due to parameterization - - name: gnocchi-metric-get-measures-single-page - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-metric-get-measures-all - enabled: true - all_metrics: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-metric-get-measures-agg-mean-refresh - enabled: true - all_metrics: true - aggregation: mean - refresh: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - # List/Create/Create&Delete Metrics - - name: gnocchi-metric-list - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-metric-list.yml - - name: gnocchi-create-metric - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-metric.yml - - name: gnocchi-create-delete-metric - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml + - name: gnocchi-capabilities-list + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-capabilities-list.yml - - name: gnocchi-resource-list - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-resource-list.yml - - name: gnocchi-create-resource - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-resource.yml - - name: gnocchi-create-delete-resource - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Aggregation has several potential scenarios due to parameterization + - name: gnocchi-metric-aggregation-single-page + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-aggregation.yml + - name: gnocchi-metric-aggregation-all + enabled: true + all_metrics: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-aggregation.yml + - name: gnocchi-metric-aggregation-agg-mean-refresh + enabled: true + all_metrics: true + aggregation: mean + refresh: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-aggregation.yml + # Get Measures has several potential scenarios due to parameterization + - name: gnocchi-metric-get-measures-single-page + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + - name: gnocchi-metric-get-measures-all + enabled: true + all_metrics: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + - name: gnocchi-metric-get-measures-agg-mean-refresh + enabled: true + all_metrics: true + aggregation: mean + refresh: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + # List/Create/Create&Delete Metrics + - name: gnocchi-metric-list + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-metric-list.yml + - name: gnocchi-create-metric + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-metric.yml + - name: gnocchi-create-delete-metric + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-resource-type-list - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-resource-type-list.yml - - name: gnocchi-create-resource-type - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-resource-type.yml - - name: gnocchi-create-delete-resource-type - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource-type.yml + - name: gnocchi-resource-list + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-resource-list.yml + - name: gnocchi-create-resource + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-resource.yml + - name: gnocchi-create-delete-resource + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - name: gnocchi-status-get - enabled: true - file: rally/rally-plugins/gnocchi/gnocchi-status-get.yml - - name: gnocchi-status-get-detailed - enabled: true - detailed: True - file: rally/rally-plugins/gnocchi/gnocchi-status-get.yml + - name: gnocchi-resource-type-list + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-resource-type-list.yml + - name: gnocchi-create-resource-type + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-resource-type.yml + - name: gnocchi-create-delete-resource-type + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource-type.yml + + - name: gnocchi-status-get + enabled: true + file: rally/rally-plugins/gnocchi/gnocchi-status-get.yml + - name: gnocchi-status-get-detailed + enabled: true + detailed: True + file: rally/rally-plugins/gnocchi/gnocchi-status-get.yml diff --git a/conf/perfkit-benchmarks.yaml b/conf/perfkit-benchmarks.yaml index 87ea84176..6e776d623 100644 --- a/conf/perfkit-benchmarks.yaml +++ b/conf/perfkit-benchmarks.yaml @@ -5,14 +5,18 @@ # each of the benchmarks. The settings for the benchmarks below # are not tuned for best performance analysis either. browbeat: - results : results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -22,23 +26,13 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance perfkit: - enabled: true sleep_before: 0 sleep_after: 0 default: @@ -50,80 +44,119 @@ perfkit: openstack_network: browbeat_private timing_measurements: runtimes ignore_package_requirements: true - benchmarks: - - name: aerospike-centos-m1-small - enabled: false - benchmarks: aerospike - - name: block_storage_workload-database-centos-m1-small - enabled: false - benchmarks: block_storage_workload - openstack_volume_size: 20 - workload_mode: database - - name: block_storage_workload-logging-centos-m1-small - enabled: false - benchmarks: block_storage_workload - openstack_volume_size: 20 - workload_mode: logging - - name: block_storage_workload-streaming-centos-m1-small - enabled: false - benchmarks: block_storage_workload - openstack_volume_size: 20 - workload_mode: streaming - - name: cluster_boot-centos-m1-small - enabled: false - benchmarks: cluster_boot - config_override: "cluster_boot.vm_groups.default.vm_count=4" - - name: copy_throughput-cp-centos-m1-small - enabled: false - benchmarks: copy_throughput - copy_benchmark_mode: cp - openstack_volume_size: 20 - - name: copy_throughput-dd-centos-m1-small - enabled: false - benchmarks: copy_throughput - copy_benchmark_mode: dd - openstack_volume_size: 20 - - name: copy_throughput-scp-centos-m1-small - enabled: false - benchmarks: copy_throughput - copy_benchmark_mode: scp - openstack_volume_size: 20 - - name: fio-centos-m1-small - enabled: false - benchmarks: fio - openstack_volume_size: 20 - - name: fio-centos-m1-small-10m - enabled: false - benchmarks: fio - openstack_volume_size: 20 - fio_generate_scenarios: all - fio_runtime: 600 - fio_working_set_size: 4 - - name: iperf-centos-m1-small - enabled: false - benchmarks: iperf - - name: mesh_network-centos-m1-small - enabled: false - benchmarks: mesh_network - num_vms: 3 - - name: netperf-centos-m1-small - enabled: false - benchmarks: netperf - - name: ping-centos-m1-small - enabled: false - benchmarks: ping - - name: redis_ycsb-centos-m1-small - enabled: false - benchmarks: redis_ycsb - ycsb_client_vms: 2 - - name: scimark2-centos-m1-small - enabled: false - benchmarks: scimark2 - - name: sysbench_oltp-centos-m1-small - enabled: false - benchmarks: sysbench_oltp - openstack_volume_size: 20 - - name: unixbench-centos-m1-small - enabled: false - benchmarks: unixbench - openstack_volume_size: 20 +rally: + sleep_before: 5 + sleep_after: 5 + plugins: + - glance: rally/rally-plugins/glance + - neutron: rally/rally-plugins/neutron + - netcreate-boot: rally/rally-plugins/netcreate-boot + - workloads: rally/rally-plugins/workloads +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: aerospike-centos-m1-small + enabled: false + type: perfkit + benchmarks: aerospike + - name: block_storage_workload-database-centos-m1-small + enabled: false + type: perfkit + benchmarks: block_storage_workload + openstack_volume_size: 20 + workload_mode: database + - name: block_storage_workload-logging-centos-m1-small + enabled: false + type: perfkit + benchmarks: block_storage_workload + openstack_volume_size: 20 + workload_mode: logging + - name: block_storage_workload-streaming-centos-m1-small + enabled: false + type: perfkit + benchmarks: block_storage_workload + openstack_volume_size: 20 + workload_mode: streaming + - name: cluster_boot-centos-m1-small + enabled: false + type: perfkit + benchmarks: cluster_boot + config_override: "cluster_boot.vm_groups.default.vm_count=4" + - name: copy_throughput-cp-centos-m1-small + enabled: false + type: perfkit + benchmarks: copy_throughput + copy_benchmark_mode: cp + openstack_volume_size: 20 + - name: copy_throughput-dd-centos-m1-small + enabled: false + type: perfkit + benchmarks: copy_throughput + copy_benchmark_mode: dd + openstack_volume_size: 20 + - name: copy_throughput-scp-centos-m1-small + enabled: false + type: perfkit + benchmarks: copy_throughput + copy_benchmark_mode: scp + openstack_volume_size: 20 + - name: fio-centos-m1-small + enabled: false + type: perfkit + benchmarks: fio + openstack_volume_size: 20 + - name: fio-centos-m1-small-10m + enabled: false + type: perfkit + benchmarks: fio + openstack_volume_size: 20 + fio_generate_scenarios: all + fio_runtime: 600 + fio_working_set_size: 4 + - name: iperf-centos-m1-small + enabled: false + type: perfkit + benchmarks: iperf + - name: mesh_network-centos-m1-small + enabled: false + type: perfkit + benchmarks: mesh_network + num_vms: 3 + - name: netperf-centos-m1-small + enabled: false + type: perfkit + benchmarks: netperf + - name: ping-centos-m1-small + enabled: false + type: perfkit + benchmarks: ping + - name: redis_ycsb-centos-m1-small + enabled: false + type: perfkit + benchmarks: redis_ycsb + ycsb_client_vms: 2 + - name: scimark2-centos-m1-small + enabled: false + type: perfkit + benchmarks: scimark2 + - name: sysbench_oltp-centos-m1-small + enabled: false + type: perfkit + benchmarks: sysbench_oltp + openstack_volume_size: 20 + - name: unixbench-centos-m1-small + enabled: false + type: perfkit + benchmarks: unixbench + openstack_volume_size: 20 diff --git a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml index cd5add6a6..6ecb6bf96 100644 --- a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml @@ -7,14 +7,18 @@ # how system resources and api responsiveness degrade over each new set of # instances booted. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 10 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -24,23 +28,25 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: @@ -48,355 +54,55 @@ rally: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - gnocchi: rally/rally-plugins/gnocchi - benchmarks: +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - # Baseline - - name: BrowbeatGnocchiBaseline - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-Baseline - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml +workloads: - - name: BrowbeatNovaPersistWithNetwork01 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.0.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi01 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-01 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Baseline Gnocchi API + - name: BrowbeatGnocchiResponsiveness + enabled: true + type: rally + concurrency: + - 8 + times: 5000 + scenarios: + - name: gnocchi-metric-get-measures-agg-mean-Baseline + enabled: true + all_metrics: true + aggregation: mean + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + - name: gnocchi-create-delete-metric-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml + - name: gnocchi-create-delete-resource-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - name: BrowbeatNovaPersistWithNetwork02 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.4.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi02 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-02 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork03 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.8.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi03 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-03 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-03 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-03 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork04 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.12.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi04 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-04 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-04 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-04 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork05 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.16.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi05 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-05 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-05 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-05 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork06 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.20.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi06 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-06 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-06 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-06 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork07 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.24.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi07 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-07 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-07 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-07 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork08 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.28.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi08 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-08 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-08 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-08 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork09 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.32.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi09 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-09 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-09 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-09 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork10 - enabled: true - concurrency: - - 40 - times: 1000 - scenarios: - - name: nova-boot-persist-with-network-1000-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.36.0/22 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi10 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-10 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-10 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-10 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Boot 1000 instances + - name: BrowbeatNovaPersistWithNetwork + enabled: true + type: rally + concurrency: + - 40 + times: 1000 + scenarios: + - name: nova-boot-persist-with-network-1000 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + network_start_cidr: 10.1.0.0/22 + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml index 7d11d3875..19a5642ed 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml @@ -7,14 +7,18 @@ # how system resources and api responsiveness degrade over each new set of # instances booted. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 10 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -24,23 +28,25 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: @@ -48,355 +54,55 @@ rally: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - gnocchi: rally/rally-plugins/gnocchi - benchmarks: +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - # Baseline - - name: BrowbeatGnocchiBaseline - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-Baseline - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml +workloads: - - name: BrowbeatNovaPersistWithNetwork01 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.0.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi01 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-01 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Baseline Gnocchi API + - name: BrowbeatGnocchiResponsiveness + enabled: true + type: rally + concurrency: + - 8 + times: 5000 + scenarios: + - name: gnocchi-metric-get-measures-agg-mean-Baseline + enabled: true + all_metrics: true + aggregation: mean + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + - name: gnocchi-create-delete-metric-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml + - name: gnocchi-create-delete-resource-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - name: BrowbeatNovaPersistWithNetwork02 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.2.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi02 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-02 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork03 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.4.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi03 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-03 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-03 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-03 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork04 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.6.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi04 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-04 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-04 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-04 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork05 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.8.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi05 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-05 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-05 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-05 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork06 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.10.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi06 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-06 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-06 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-06 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork07 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.12.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi07 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-07 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-07 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-07 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork08 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.14.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi08 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-08 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-08 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-08 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork09 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.16.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi09 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-09 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-09 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-09 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork10 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.18.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi10 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-10 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-10 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-10 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Boot 100 instances + - name: BrowbeatNovaPersistWithNetwork + enabled: true + type: rally + concurrency: + - 20 + times: 100 + scenarios: + - name: nova-boot-persist-with-network-100 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + network_start_cidr: 10.1.0.0/23 + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml index 7131df02c..ccf514446 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml @@ -7,14 +7,18 @@ # how system resources and api responsiveness degrade over each new set of # instances booted. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 2 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -24,23 +28,25 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: @@ -48,91 +54,55 @@ rally: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - gnocchi: rally/rally-plugins/gnocchi - benchmarks: +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - # Baseline - - name: BrowbeatGnocchiBaseline - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-Baseline - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml +workloads: - - name: BrowbeatNovaPersistWithNetwork01 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.0.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi01 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-01 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Baseline Gnocchi API + - name: BrowbeatGnocchiResponsiveness + enabled: true + type: rally + concurrency: + - 8 + times: 5000 + scenarios: + - name: gnocchi-metric-get-measures-agg-mean-Baseline + enabled: true + all_metrics: true + aggregation: mean + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + - name: gnocchi-create-delete-metric-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml + - name: gnocchi-create-delete-resource-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - name: BrowbeatNovaPersistWithNetwork02 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.2.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi02 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-02 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Boot 500 instances + - name: BrowbeatNovaPersistWithNetwork + enabled: true + type: rally + concurrency: + - 20 + times: 500 + scenarios: + - name: nova-boot-persist-with-network-500 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + network_start_cidr: 10.1.0.0/23 + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml diff --git a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml index 34f3a976d..ea49365ff 100644 --- a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml @@ -7,14 +7,18 @@ # how system resources and api responsiveness degrade over each new set of # instances booted. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 10 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -24,23 +28,25 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 5 sleep_after: 5 plugins: @@ -48,355 +54,55 @@ rally: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - gnocchi: rally/rally-plugins/gnocchi - benchmarks: +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" - # Baseline - - name: BrowbeatGnocchiBaseline - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-Baseline - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-Baseline - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml +workloads: - - name: BrowbeatNovaPersistWithNetwork01 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.0.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi01 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-01 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-01 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Baseline Gnocchi API + - name: BrowbeatGnocchiResponsiveness + enabled: true + type: rally + concurrency: + - 8 + times: 5000 + scenarios: + - name: gnocchi-metric-get-measures-agg-mean-Baseline + enabled: true + all_metrics: true + aggregation: mean + file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml + - name: gnocchi-create-delete-metric-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml + - name: gnocchi-create-delete-resource-Baseline + enabled: true + times: 1000 + file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - name: BrowbeatNovaPersistWithNetwork02 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.2.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi02 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-02 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-02 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork03 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.4.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi03 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-03 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-03 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-03 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork04 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.6.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi04 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-04 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-04 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-04 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork05 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.8.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi05 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-05 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-05 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-05 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork06 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.10.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi06 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-06 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-06 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-06 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork07 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.12.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi07 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-07 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-07 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-07 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork08 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.14.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi08 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-08 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-08 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-08 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork09 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.16.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi09 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-09 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-09 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-09 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml - - - name: BrowbeatNovaPersistWithNetwork10 - enabled: true - concurrency: - - 20 - times: 500 - scenarios: - - name: nova-boot-persist-with-network-500-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - network_start_cidr: 10.1.18.0/23 - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: BrowbeatGnocchi10 - enabled: true - concurrency: - - 8 - times: 5000 - scenarios: - - name: gnocchi-metric-get-measures-agg-mean-10 - enabled: true - all_metrics: true - aggregation: mean - file: rally/rally-plugins/gnocchi/gnocchi-metric-get-measures.yml - - name: gnocchi-create-delete-metric-10 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-metric.yml - - name: gnocchi-create-delete-resource-10 - enabled: true - times: 1000 - file: rally/rally-plugins/gnocchi/gnocchi-create-delete-resource.yml + # Boot 500 instances + - name: BrowbeatNovaPersistWithNetwork + enabled: true + type: rally + concurrency: + - 20 + times: 500 + scenarios: + - name: nova-boot-persist-with-network-500 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + network_start_cidr: 10.1.0.0/23 + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml diff --git a/conf/telemetry-nova-persist-instances-with-network-fip.yaml b/conf/telemetry-nova-persist-instances-with-network-fip.yaml index e25ed0dce..85c009e08 100644 --- a/conf/telemetry-nova-persist-instances-with-network-fip.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-fip.yaml @@ -4,14 +4,18 @@ # booted. The instances have a nic and a fip. # You must pre-populate (external_net_name and private_net_id) browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -21,114 +25,130 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 0 sleep_after: 0 plugins: - browbeat: rally/rally-plugins/browbeat - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - benchmarks: - # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms - - name: BrowbeatNovaPersistWithNetworkFIP01 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-fip-100-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml - - name: nova-boot-persist-with-network-fip-100-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms + - name: BrowbeatNovaPersistWithNetworkFIP01 + enabled: true + type: rally + concurrency: + - 20 + times: 100 + scenarios: + - name: nova-boot-persist-with-network-fip-100-01 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-02 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-03 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-04 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-05 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-06 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-07 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-08 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-09 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml + - name: nova-boot-persist-with-network-fip-100-10 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_fip.yml diff --git a/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml b/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml index c3646f253..45cb685dd 100644 --- a/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-volume-fip.yaml @@ -4,14 +4,18 @@ # booted. The instances have a nic, volume and fip. # You must pre-populate (external_net_name and private_net_id) browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -21,124 +25,140 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 0 sleep_after: 0 plugins: - browbeat: rally/rally-plugins/browbeat - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - benchmarks: - # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms - - name: BrowbeatNovaPersistWithNetworkVolumeFIP01 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-volume-fip-100-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml - - name: nova-boot-persist-with-network-volume-fip-100-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - external_net_name: - private_net_id: - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms + - name: BrowbeatNovaPersistWithNetworkVolumeFIP01 + enabled: true + type: rally + concurrency: + - 20 + times: 100 + scenarios: + - name: nova-boot-persist-with-network-volume-fip-100-01 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-02 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-03 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-04 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-05 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-06 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-07 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-08 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-09 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml + - name: nova-boot-persist-with-network-volume-fip-100-10 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + external_net_name: + private_net_id: + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume_fip.yml diff --git a/conf/telemetry-nova-persist-instances-with-network-volume.yaml b/conf/telemetry-nova-persist-instances-with-network-volume.yaml index 9328b4d29..5f040756d 100644 --- a/conf/telemetry-nova-persist-instances-with-network-volume.yaml +++ b/conf/telemetry-nova-persist-instances-with-network-volume.yaml @@ -3,14 +3,18 @@ # with a concurrency of 20 instances. A total of 1000 instances are # booted. The instances have a nic and volume. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -20,114 +24,130 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 0 sleep_after: 0 plugins: - browbeat: rally/rally-plugins/browbeat - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - benchmarks: - # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms - - name: BrowbeatNovaPersistWithNetworkVolume01 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-volume-100-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.0.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.2.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.4.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.6.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.8.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.10.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.12.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.14.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.16.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml - - name: nova-boot-persist-with-network-volume-100-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - volume_size: 1 - cleanup_delay: 1800 - network_start_cidr: 10.1.18.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms + - name: BrowbeatNovaPersistWithNetworkVolume01 + enabled: true + type: rally + concurrency: + - 20 + times: 100 + scenarios: + - name: nova-boot-persist-with-network-volume-100-01 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.0.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-02 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.2.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-03 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.4.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-04 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.6.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-05 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.8.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-06 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.10.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-07 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.12.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-08 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.14.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-09 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.16.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml + - name: nova-boot-persist-with-network-volume-100-10 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + volume_size: 1 + cleanup_delay: 1800 + network_start_cidr: 10.1.18.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network_volume.yml diff --git a/conf/telemetry-nova-persist-instances-with-network.yaml b/conf/telemetry-nova-persist-instances-with-network.yaml index 27204aeb4..d5951abfe 100644 --- a/conf/telemetry-nova-persist-instances-with-network.yaml +++ b/conf/telemetry-nova-persist-instances-with-network.yaml @@ -5,14 +5,18 @@ # additional telemetry load. View how system resources degrade over each # new set of instances booted. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -22,104 +26,120 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 0 sleep_after: 0 plugins: - browbeat: rally/rally-plugins/browbeat - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron - benchmarks: - # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms - - name: BrowbeatNovaPersistWithNetwork01 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-persist-with-network-100-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.0.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.2.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.4.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.6.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.8.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.10.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.12.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.14.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.16.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml - - name: nova-boot-persist-with-network-100-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - network_start_cidr: 10.1.18.0/23 - file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms + - name: BrowbeatNovaPersistWithNetwork01 + enabled: true + type: rally + concurrency: + - 20 + times: 100 + scenarios: + - name: nova-boot-persist-with-network-100-01 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.0.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-02 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.2.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-03 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.4.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-04 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.6.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-05 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.8.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-06 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.10.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-07 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.12.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-08 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.14.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-09 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.16.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml + - name: nova-boot-persist-with-network-100-10 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + network_start_cidr: 10.1.18.0/23 + file: rally/rally-plugins/nova/nova_boot_persist_with_network.yml diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml index b6ef36f8f..4393f7c1a 100644 --- a/conf/telemetry-nova-persist-instances.yaml +++ b/conf/telemetry-nova-persist-instances.yaml @@ -4,14 +4,18 @@ # booted. View how system resources degrade over each new set of # instances booted. browbeat: - results: results/ - rerun: 1 cloud_name: openstack + rerun: 1 + rerun_type: iteration +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config elasticsearch: enabled: false - regather: false host: 1.1.1.1 port: 9200 + regather: false metadata_files: - name: hardware-metadata file: metadata/hardware-metadata.json @@ -21,93 +25,110 @@ elasticsearch: file: metadata/software-metadata.json - name: version file: metadata/version.json -ansible: - ssh_config: ansible/ssh-config - hosts: ansible/hosts - adjust: - keystone_token: ansible/browbeat/adjustment-keystone-token.yml - neutron_l3: ansible/browbeat/adjustment-l3.yml - nova_db: ansible/browbeat/adjustment-db.yml - workers: ansible/browbeat/adjustment-workers.yml - metadata: ansible/gather/site.yml grafana: - enabled: false - grafana_ip: 1.1.1.1 - grafana_port: 3000 + enabled: true + host: example.grafana.com + port: 3000 dashboards: - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true rally: - enabled: true sleep_before: 0 sleep_after: 0 plugins: - browbeat: rally/rally-plugins/browbeat - nova: rally/rally-plugins/nova - benchmarks: - # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms - - name: BrowbeatNovaPersist01 - enabled: true - concurrency: - - 20 - times: 100 - scenarios: - - name: nova-boot-100-01 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-02 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-03 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-04 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-05 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-06 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-07 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-08 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-09 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml - - name: nova-boot-100-10 - enabled: true - image_name: cirros - flavor_name: m1.xtiny - cleanup_delay: 1800 - file: rally/rally-plugins/nova/nova_boot_persist.yml +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + + # Boots a total of 1000 instances, 100 at a time with 30minutes between booting storms + - name: BrowbeatNovaPersist01 + enabled: true + type: rally + concurrency: + - 20 + times: 100 + scenarios: + - name: nova-boot-100-01 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-02 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-03 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-04 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-05 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-06 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-07 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-08 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-09 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml + - name: nova-boot-100-10 + enabled: true + image_name: cirros + flavor_name: m1.xtiny + cleanup_delay: 1800 + file: rally/rally-plugins/nova/nova_boot_persist.yml diff --git a/test-requirements.txt b/test-requirements.txt index ae9399dcf..b6f6cf95a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,6 +7,7 @@ hacking<0.11,>=0.10.0 ansible-lint pykwalify coverage>=3.6 +pytest==3.2.1 python-subunit>=0.0.18 sphinx>=1.3,!=1.6.1 oslosphinx>=2.5.0 # Apache-2.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/data/invalid_browbeat.yml b/tests/data/invalid_browbeat.yml new file mode 100644 index 000000000..76e434f7a --- /dev/null +++ b/tests/data/invalid_browbeat.yml @@ -0,0 +1,89 @@ +# Invalid due to invalid_flag key +browbeat: + cloud_name: browbeat-test + rerun: 1 + rerun_type: complete + invalid_flag: invalid +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config +elasticsearch: + enabled: false + host: browbeat.test.com + port: 9200 + regather: false + metadata_files: + - name: hardware-metadata + file: metadata/hardware-metadata.json + - name: environment-metadata + file: metadata/environment-metadata.json + - name: software-metadata + file: metadata/software-metadata.json + - name: version + file: metadata/version.json +grafana: + enabled: true + host: browbeat.test.com + port: 3000 + dashboards: + - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true +rally: + sleep_before: 0 + sleep_after: 0 +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: browbeat-test-perfkit-ping + enabled: false + type: perfkit + benchmarks: ping + - name: browbeat-test-authenticate + enabled: false + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: browbeat-test-authentic-keystone + enabled: false + file: rally/authenticate/keystone-cc.yml + - name: browbeat-test-shaker-l2 + enabled: false + type: shaker + density: 1 + compute: 1 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: browbeat-test-introspect-batch + enabled: false + type: yoda + yoda_type: introspection + method: individual + times: 3 + timeout: 900 + batch_size: 2 diff --git a/tests/data/invalid_browbeat_workload.yml b/tests/data/invalid_browbeat_workload.yml new file mode 100644 index 000000000..3f4654343 --- /dev/null +++ b/tests/data/invalid_browbeat_workload.yml @@ -0,0 +1,68 @@ +# Invalid due to workload invalid +browbeat: + cloud_name: browbeat-test + rerun: 1 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config +elasticsearch: + enabled: false + host: browbeat.test.com + port: 9200 + regather: false + metadata_files: + - name: hardware-metadata + file: metadata/hardware-metadata.json + - name: environment-metadata + file: metadata/environment-metadata.json + - name: software-metadata + file: metadata/software-metadata.json + - name: version + file: metadata/version.json +grafana: + enabled: true + host: browbeat.test.com + port: 3000 + dashboards: + - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true +rally: + sleep_before: 0 + sleep_after: 0 +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + # Missing name + - enabled: false + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: browbeat-test-authentic-keystone + enabled: false + file: rally/authenticate/keystone-cc.yml diff --git a/tests/data/valid_browbeat.yml b/tests/data/valid_browbeat.yml new file mode 100644 index 000000000..dec04e9c7 --- /dev/null +++ b/tests/data/valid_browbeat.yml @@ -0,0 +1,88 @@ +# Valid Browbeat Config +browbeat: + cloud_name: browbeat-test + rerun: 1 + rerun_type: complete +ansible: + hosts: ansible/hosts + metadata_playbook: ansible/gather/site.yml + ssh_config: ansible/ssh-config +elasticsearch: + enabled: false + host: browbeat.test.com + port: 9200 + regather: false + metadata_files: + - name: hardware-metadata + file: metadata/hardware-metadata.json + - name: environment-metadata + file: metadata/environment-metadata.json + - name: software-metadata + file: metadata/software-metadata.json + - name: version + file: metadata/version.json +grafana: + enabled: true + host: browbeat.test.com + port: 3000 + dashboards: + - openstack-general-system-performance +perfkit: + sleep_before: 0 + sleep_after: 0 + default: + image: centos7 + machine_type: m1.small + os_type: rhel + openstack_image_username: centos + openstack_floating_ip_pool: browbeat_public + openstack_network: browbeat_private + timing_measurements: runtimes + ignore_package_requirements: true +rally: + sleep_before: 0 + sleep_after: 0 +shaker: + server: 1.1.1.1 + port: 5555 + flavor: m1.small + join_timeout: 600 + sleep_before: 0 + sleep_after: 0 + shaker_region: regionOne + external_host: 2.2.2.2 +yoda: + instackenv: "/home/stack/instackenv.json" + stackrc: "/home/stack/stackrc" + +workloads: + - name: browbeat-test-perfkit-ping + enabled: false + type: perfkit + benchmarks: ping + - name: browbeat-test-authenticate + enabled: false + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: browbeat-test-authentic-keystone + enabled: false + file: rally/authenticate/keystone-cc.yml + - name: browbeat-test-shaker-l2 + enabled: false + type: shaker + density: 1 + compute: 1 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - name: browbeat-test-introspect-batch + enabled: false + type: yoda + yoda_type: introspection + method: individual + times: 3 + timeout: 900 + batch_size: 2 diff --git a/tests/data/workloads.yml b/tests/data/workloads.yml new file mode 100644 index 000000000..449008f9c --- /dev/null +++ b/tests/data/workloads.yml @@ -0,0 +1,87 @@ +# Valid and invalid workload schemas for testing per workload +perfkit: + - valid: true + data: + name: valid-test-perfkit-ping + enabled: false + type: perfkit + benchmarks: ping + - valid: false + data: + opps_name: invalid-test-perfkit-ping + enabled: false + type: perfkit + benchmarks: ping +rally: + - valid: true + data: + name: valid-test-authenticate-01 + enabled: true + type: rally + concurrency: + - 1 + times: 1 + scenarios: + - name: valid-test-authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + - name: valid-test-authentic-neutron + enabled: false + file: rally/authenticate/validate_neutron-cc.yml + - valid: false + data: + name: invalid-test-authenticate-01 + enabled: true + type: rally-incorrect + concurrency: + - 1 + times: 1 + scenarios: + - name: invalid-test-authentic-keystone + enabled: true + file: rally/authenticate/keystone-cc.yml + - name: invalid-test-authentic-neutron + enabled: false + file: rally/authenticate/validate_neutron-cc.yml +shaker: + - valid: true + data: + name: valid-test-shaker-l2 + enabled: false + type: shaker + density: 1 + compute: 1 + progression: linear + time: 60 + file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml + - valid: false + data: + name: invalid-test-shaker-l2 + enabled: false + type: shaker + density: 1 + compute: 1 + progression: linear + time: 60 + opps_file: lib/python2.7/site-packages/shaker/scenarios/openstack/dense_l2.yaml +yoda: + - valid: true + data: + name: valid-test-yoda-introspection + enabled: false + type: yoda + yoda_type: introspection + method: individual + times: 3 + timeout: 900 + batch_size: 2 + - valid: false + data: + name: invalid-test-yoda-introspection + enabled: false + type: yoda + fake_yoda_type: introspection + method: individual + times: 3 + timeout: 900 + batch_size: 2 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 000000000..79ecf2bbb --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import yaml + +from browbeat.config import load_browbeat_config +from browbeat.config import _validate_yaml + +test_browbeat_configs = { + "tests/data/valid_browbeat.yml": True, + "tests/data/invalid_browbeat.yml": False, + "tests/data/invalid_browbeat_workload.yml": False +} + + +@pytest.mark.parametrize("config", test_browbeat_configs.keys()) +def test_load_browbeat_config(config): + """Tests valid and invalid Browbeat configuration.""" + if test_browbeat_configs[config]: + # Valid configuration (No exception) + loaded_config = load_browbeat_config(config) + assert loaded_config["browbeat"]["cloud_name"] == "browbeat-test" + else: + # Invalid configuration, test for exception + with pytest.raises(Exception) as exception_data: + load_browbeat_config(config) + assert "SchemaError" in str(exception_data) + + +@pytest.mark.parametrize("schema", ["perfkit", "rally", "shaker", "yoda"]) +def test__validate_yaml(schema): + """Tests valid and invalid Browbeat workload configurations.""" + with open("tests/data/workloads.yml", "r") as config_file: + config_data = yaml.safe_load(config_file) + + for workload_config in config_data[schema]: + if workload_config["valid"]: + # Valid configuration (No exception) + _validate_yaml(schema, workload_config["data"]) + else: + # Invalid configuration, test for exception + with pytest.raises(Exception) as exception_data: + _validate_yaml(schema, workload_config["data"]) + assert "SchemaError" in str(exception_data) diff --git a/tox.ini b/tox.ini index b89eea058..0638b31b4 100644 --- a/tox.ini +++ b/tox.ini @@ -17,10 +17,10 @@ commands = bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \ ansible-lint \ -x ANSIBLE0013,ANSIBLE0012,ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0016" - pykwalify -d browbeat-config.yaml -s browbeat/validate.yaml - pykwalify -d browbeat-complete.yaml -s browbeat/validate.yaml + pykwalify -d browbeat-config.yaml -s browbeat/schema/browbeat.yml + pykwalify -d browbeat-complete.yaml -s browbeat/schema/browbeat.yml bash -c "set -e; for config in $(ls conf/); do \ - pykwalify -d conf/$config -s browbeat/validate.yaml; done" + echo conf/$config; pykwalify -d conf/$config -s browbeat/schema/browbeat.yml; done" [testenv:pep8] commands = flake8 {posargs} @@ -28,6 +28,10 @@ commands = flake8 {posargs} [testenv:venv] commands = {posargs} +[testenv:py27] +basepython = python2.7 +commands = pytest {posargs} + [testenv:cover] commands = python setup.py test --coverage --testr-args='{posargs}'