diff --git a/.gitignore b/.gitignore index e625780..668fb42 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ # C extensions *.so +# log files +*.log + # Packages *.egg *.egg-info @@ -52,3 +55,7 @@ ChangeLog *~ .*.swp .*sw? +.settings/ + +# Vagrant +.vagrant/ diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..258f1f8 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,130 @@ +# -*- mode: ruby -*- + +# Takes proxy configuratiom from host environment +http_proxy = ENV["http_proxy"] +https_proxy = ENV["https_proxy"] +no_proxy = ENV["no_proxy"] +if http_proxy or https_proxy + require 'vagrant-proxyconf' +end + +# This where the working directory is going to be copied/mounted inside of +# produced VM. +source_dir="/vagrant" + +# This is the synchronization strategy to copy or mount wirking directory +# inside the VM. For more details read below doc: +# https://www.vagrantup.com/docs/synced-folders/ +sync_type="rsync" + +# For the purpose of deplying openstack code inside provisioned VM it is using +# git to fetch source code from upstream. The repository server can be changed +# by setting this variable to point to some other place. +# A common alternative value is "https://github.com" +# This value is taken from host env configuration when available. +# If this value is empty then the default git server knwon by devstak is used +# that in the moment this doc is written is "https://git.openstack.org" +git_base = ENV["GIT_BASE"] or "" + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "ubuntu/xenial64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Collectd will fail to install if it is not able to resolve the ip from its + # hostname. Setting the host name here will make Vagrant configuring + # /etc/hosts fixing this problem + config.vm.hostname = "vagrant" + + # Create a private network, which allows host-only access to the machine + # using a specific IP. This internal private network will be used by collectd + config.vm.network "private_network", ip: "192.168.0.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + config.vm.synced_folder ".", source_dir, create:true, type: sync_type + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # Display the VirtualBox GUI when booting the machine + vb.gui = false + + # Customize the amount of memory on the VM. This ammout is recommended + # to make OpenStack working. + vb.memory = "2048" + end + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Fix slow connection problems + # config.vm.provider "virtualbox" do |v| + # v.customize ["modifyvm", :id, "--nictype1", "virtio"] + # end + + # Configure proxy variables + if Vagrant.has_plugin?("vagrant-proxyconf") + if http_proxy + config.proxy.http = http_proxy + end + if https_proxy + config.proxy.https = https_proxy + end + if no_proxy + config.proxy.no_proxy = no_proxy + end + end + + # Use the same DNS server as the host machine + config.vm.provision "file", source: "/etc/resolv.conf", + destination: "~/resolv.conf" + config.vm.provision "shell", privileged: false, + inline: "sudo mv ~/resolv.conf /etc/resolv.conf" + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + # Below script is going to be executed inside of the VM as regular user + config.vm.provision "shell", privileged: false, inline: <<-SHELL + set -x + + # forward configuration from the hos to the guest VM + export GIT_BASE="#{git_base}" + export SOURCE_DIR="#{source_dir}" + + # run provisioning script + "$SOURCE_DIR"/tools/vagrant_provision.sh + SHELL +end diff --git a/doc/source/vagrant.rst b/doc/source/vagrant.rst new file mode 100644 index 0000000..6224b16 --- /dev/null +++ b/doc/source/vagrant.rst @@ -0,0 +1,96 @@ +======= +Vagrant +======= + +This project is distributed with a Vagrantfile intended to be used by +developers to run functional tests on a Linux virtual machine. + +The software provided by the appliance are: +- Ubuntu Server 16.04 LTS appliance running inside Virtual Box +- The collectd provided by Ubuntu +- Python 2.7 and 3.4 provided by Ubuntu +- This plugin +- Project dependencies provided by PIP as specified by project requirements + +To use Vagrant to test this plugin you have to install following software +up to date software on your host machine: +- Vagrant +- VirtualBox 5 + + +Vagrant +------- + +Documentation to install Vagrant can be found on there website at + + https://www.vagrantup.com/docs/installation/ + +A recent version of Vagrant is required to utilize the vagrant file provided by +this repo. If you have an older version already installed please upgrade by +following the documentation at + + https://www.vagrantup.com/docs/installation/upgrading.html + + +Virtual Box +----------- + +Virtual Box version greater than 5 should works fine. + +Many Linux distributions should provide a valid version of VirtualBox. + +The Virtual Box web site hosts some documentation and links to help you +downloading and installing Virtual Box on your machine. Please see it there: + + https://www.virtualbox.org/wiki/Downloads + + +Proxy configuration +------------------- + +If you are working beside a proxy server then you have to install +vagrant-proxyconf plugin:: + + vagrant plugin install vagrant-proxyconf + +You can have more details about the plugin on below site: + + https://github.com/tmatilai/vagrant-proxyconf + +To use it, before creating target appliance, you have to configure standard +enviroment variable on your host machine like below:: + + export http_proxy=http://:/ + export https_proxy=$https_proxy + export no_proxy=localhost,$(hostname) + + +Create and use the appliance +---------------------------- + +Once all required software is installed and configured you can create +and launch the appliance as follows (it can take several minutes) from +the project folder:: + + vagrant up + +Once the appliance is running connect to the running VM you can type following:: + + vagrant ssh + +During the provisioning process collectd and ceilometer are going to be +installed and launched using devstack. To force updating the project and +restacking it again please use one of the following commands:: + + # when the machine is down and I want to take it up, update the software + # and start all services (including collectd and ceilometer) + vagrant up --provision + + # when the machine is running and I want to take reboot it, update the + # software and re-start all services (including collectd and ceilometer) + vagrant reload --provision + + +To launch unit tests inside the appliance you can type following:: + + vagrant ssh -c 'cd /vagrant && tox' diff --git a/local.conf b/local.conf new file mode 100644 index 0000000..15a79a2 --- /dev/null +++ b/local.conf @@ -0,0 +1,19 @@ +# -*- mode: sh -*- + +[[local|localrc]] +ADMIN_PASSWORD="secret" +DATABASE_PASSWORD="$ADMIN_PASSWORD" +RABBIT_PASSWORD="$ADMIN_PASSWORD" +SERVICE_PASSWORD="$ADMIN_PASSWORD" + +# Setup only required services +disable_all_services +enable_service "mysql" "rabbitmq" "dstat" "keystone" + +# Setup ceilometer +enable_plugin "ceilometer" "${GIT_BASE:-https://git.openstack.org}/openstack/ceilometer.git" + +# Setup collectd-ceilometer plugin +COLLECTD_INSTALL=True +COLLECTD_CEILOMETER_VERBOSE=True +enable_plugin "collectd-ceilometer-plugin" "${GIT_BASE:-https://git.openstack.org}/openstack/collectd-ceilometer-plugin.git" diff --git a/tools/get_proxy_env.py b/tools/get_proxy_env.py new file mode 100644 index 0000000..0462231 --- /dev/null +++ b/tools/get_proxy_env.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The purpose of this script is generating environment variables used to +configure /etc/profile inside of the virtual machine. + +The main purpose of this script is allowing vagrant to configure properly +the no_proxy environment variable adding entries that could break openstack +components internal connectivity. +""" + +import os +import subprocess +import sys + + +PROXY_ENV = ''' +export HTTP_PROXY={http_proxy} +export http_proxy={http_proxy} +export HTTPS_PROXY={https_proxy} +export https_proxy={https_proxy} +export NO_PROXY={no_proxy} +export no_proxy={no_proxy} +''' + + +def get_proxy_env(): + """Get proxy profile file content to be sourced by Bash + + It produces the content of proxy profile file sourced by bash + to have proxy env set up in the VM + """ + + return PROXY_ENV.format( + http_proxy=os.environ.get('http_proxy', ''), + https_proxy=os.environ.get('https_proxy', ''), + no_proxy=','.join(get_no_proxy())) + + +def get_no_proxy_hosts(): + "Gets current host names contained inside the no_proxy variable" + for e in os.environ.get('no_proxy', '').split(','): + yield e.strip() + + +def get_host_ips(): + "Gets all known IPS addresses of the host." + for line in subprocess.check_output(['ifconfig']).split('\n'): + fields = line.strip().split(' ', 2) + if fields[0] == 'inet': + if fields[1].startswith('addr:'): + yield fields[1].split(':', 1)[1].strip() + + +def get_known_hosts(): + "Gets all known host names and IPs find in /etc/hosts." + with open('/etc/hosts', 'r') as hosts_file: + for line in hosts_file.readlines(): + # strip out all comments + stripped_line = line.split('#', 1)[0].strip() + for e in stripped_line.split(): + yield e.strip() + + +def get_no_proxy(): + """Get content for no_proxy environment variable. + + Get a list of addresses and ips to connect wich the proxy shouldn't be used + """ + + # the resulted list is returned sorted to improve reapeatibily over + # produced output over multiple executions. Sets are know being + # not reliable acording the order items are iterated. + return sorted( + (set(get_no_proxy_hosts()) | # take existing no_proxy entries + set(get_host_ips()) | # add ips assinged to all existing ifaces + set(get_known_hosts())) # add all hosts and ips taken from /etc/hosts + - set([''])) # remove empty string that could be here by mistake + + +if __name__ == '__main__': + # stdout is used to produce the result of this script + out = sys.stdout + # redirect stdout to stderr to avoid to pollute the output + sys.stdout = sys.stderr + out.write(get_proxy_env()) diff --git a/tools/vagrant_provision.sh b/tools/vagrant_provision.sh new file mode 100755 index 0000000..f8f1004 --- /dev/null +++ b/tools/vagrant_provision.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Vagrant provisioning script. + +SOURCE_DIR="${SOURCE_DIR:-$(cd $(dirname $(dirname ${BASH_SOURCE[0]})) && pwd)}" +STACK_DIR="${STACK_DIR:-/opt/stack}" + +PROXY_FILE="/etc/profile.d/proxy.sh" +COLLECTD_CEILOMETER_DIR="$STACK_DIR/collectd-ceilometer-plugin" +DEVSTACK_DIR="$STACK_DIR/devstack" +DEVSTACK_REPO="${GIT_BASE:-https://git.openstack.org}/openstack-dev/devstack.git" + +echo "Start provisioning." +set -ex +export DEBIAN_FRONTEND="noninteractive" + +echo "Update and install required packages." +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install -y git python-minimal python-setuptools\ + python-libvirt libvirt-bin + +echo "Fix proxy variables." +if [ -r "$PROXY_FILE" ]; then + echo "Fix proxy enviroment." + TMP_PROXY_FILE=$(mktemp) + python "$SOURCE_DIR/tools/get_proxy_env.py" > "$TMP_PROXY_FILE" + sudo mv "$TMP_PROXY_FILE" "$PROXY_FILE" + source "$PROXY_FILE" +fi + +echo "Install and update main Python packages." +sudo easy_install pip +sudo pip install -U urllib3 pip setuptools tox + +echo "Prepare devstack destination dir: $STACK_DIR." +sudo mkdir -p "$STACK_DIR" +sudo chown "$USER.$USER" $STACK_DIR + +# This makes sure the collectd ceilometer plugin deployed is the one contained +# in the same folder as the Vagrantfile +if [ "$COLLECTD_CEILOMETER_DIR" != "$SOURCE_DIR" ]; then + sudo rm -fR "$COLLECTD_CEILOMETER_DIR" || true + sudo ln -sfn "$SOURCE_DIR" "$COLLECTD_CEILOMETER_DIR" +fi + +if ! [ -d "$DEVSTACK_DIR" ]; then + echo "Clone devstack." + git clone "$DEVSTACK_REPO" "$DEVSTACK_DIR" +fi + +pushd "$DEVSTACK_DIR" + cp "$SOURCE_DIR/local.conf" . + + # unstack if stacking procedure has been already started before + if [ -f ".stacking" ]; then + echo "Unstack." + (./unstack.sh && rm -fR ".stacking") || true + fi + + echo "Start stacking." + + # GIT_BASE is forwarded from host machine to the guest virtual machine to make + # devstack behaving like as it would have been executed on the bare host machine + if [ -n "$GIT_BASE" ]; then + # This is going to replace default values specified by devstack + # but not the ones defined inside local.conf + export GIT_BASE=$GIT_BASE + fi + + touch ".stacking" + # This tells collectd the source directory of the plugin is the one + # where the Vagrantfile is (typically /vagrant) + COLLECTD_CEILOMETER_DIR="$SOURCE_DIR" "./stack.sh" +popd + +echo "Provisioned."