diff --git a/libvirt/README.rst b/libvirt/README.rst new file mode 100644 index 0000000..b5f0396 --- /dev/null +++ b/libvirt/README.rst @@ -0,0 +1,65 @@ +StarlingX Deployment on Libvirt +=============================== + +This is a quick reference for deploying StarlingX on libvirt/qemu systems. +It assumes you have a working libvirt/qemu installation for a non-root user +and that your user has NOPASSWD sudo permissions. + +Overview +-------- + +We create 4 bridges to use for the STX cloud. This is done in an initial step +separate from the VM management. + +Depending on which basic configuration is chosen, we create a number of VMs +for one or more controllers and storage nodes. + +These scripts are configured using environment variables that all have built-in +defaults. On shared systems you probably do not want to use the defaults. +The simplest way to handle this is to keep an rc file that can be sourced into +an interactive shell that configures everything. Here's an example:: + + export CONTROLLER=madcloud + export COMPUTE=madnode + export BRIDGE_INTERFACE=madbr + export INTERNAL_NETWORK=172.30.20.0/24 + export INTERNAL_IP=172.30.20.1/24 + export EXTERNAL_NETWORK=192.168.20.0/24 + export EXTERNAL_IP=192.168.20.1/24 + +There is also a script ``cleanup_network.sh`` that will remove networking +configuration from libvirt. + +Networking +---------- + +Configure the bridges using ``setup_network.sh`` before doing anything else. It +will create 4 bridges named ``stxbr1``, ``stxbr2``, ``stxbr3`` and ``stxbr4``. +Set the BRIDGE_INTERFACE environment variable if you need to change stxbr to +something unique. + +The ``destroy_network.sh`` script does the reverse, and should not be used lightly. +It should also only be used after all of the VMs created below have been destroyed. + +Controllers +----------- + +There are two scripts for creating the controllers: ``setup_allinone.sh`` and +``setup_standard_controller.sh``. They are operated in the same manner but build +different StarlingX cloud configurations. Choose wisely. + +You need an ISO file for the installation, these scripts take a name with the +``-i`` option:: + + ./setup_allinone.sh -i stx-2018-08-28-93.iso + +And the setup will begin. The scripts create one or more VMs and start the boot +of the first controller, named oddly enough ``controller-0``. If you have Xwindows +available you will get virt-manager running. If not, Ctrl-C out of that attempt if +it doesn't return to a shell prompt. Then connect to the serial console:: + + virsh console madcloud-0 + +Continue the usual SterlingX installation from this point forward. + +Tear down the VMs using ``destroy_allinone.sh`` and ``destroy_standard_controller.sh``. diff --git a/libvirt/cleanup_network.sh b/libvirt/cleanup_network.sh new file mode 100755 index 0000000..ebace0b --- /dev/null +++ b/libvirt/cleanup_network.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# cleanup_network.sh - Cleans up network interfaces - not safe to run blindly! + +NETWORK_DEFAULT=${NETWORK_DEFAULT:-default} +BRIDGE_INTERFACE=${BRIDGE_INTERFACE=stxbr0} + +if virsh net-list --name | grep ${NETWORK_DEFAULT} ; then + sudo virsh net-destroy ${NETWORK_DEFAULT} + sudo virsh net-undefine ${NETWORK_DEFAULT} + sudo rm -rf /etc/libvirt/qemu/networks/autostart/${NETWORK_DEFAULT}.xml +fi + +if [ -d "/sys/class/net/${BRIDGE_INTERFACE}" ]; then + sudo ifconfig ${BRIDGE_INTERFACE} down || true + sudo brctl delbr ${BRIDGE_INTERFACE} || true +fi diff --git a/libvirt/compute.xml b/libvirt/compute.xml index 9e7c699..85474c0 100644 --- a/libvirt/compute.xml +++ b/libvirt/compute.xml @@ -55,14 +55,14 @@
- +
- + @@ -70,14 +70,14 @@
- +
- + diff --git a/libvirt/controller.xml b/libvirt/controller.xml index 18c52de..e346813 100644 --- a/libvirt/controller.xml +++ b/libvirt/controller.xml @@ -66,7 +66,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -83,7 +83,7 @@ - + @@ -91,7 +91,7 @@ - + diff --git a/libvirt/controller_allinone.xml b/libvirt/controller_allinone.xml index 3fe5b5e..9249183 100644 --- a/libvirt/controller_allinone.xml +++ b/libvirt/controller_allinone.xml @@ -66,7 +66,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -83,7 +83,7 @@ - + @@ -91,7 +91,7 @@ - + diff --git a/libvirt/destroy_allinone.sh b/libvirt/destroy_allinone.sh index 4c23de6..9389c16 100755 --- a/libvirt/destroy_allinone.sh +++ b/libvirt/destroy_allinone.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -CONTROLLER=controller-allinone +BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr} +CONTROLLER=${CONTROLLER:-controller-allinone} DOMAIN_DIRECTORY=vms -NETWORK_INTERFACE=virbr for i in {0..1}; do CONTROLLER_NODE=${CONTROLLER}-${i} @@ -19,11 +19,3 @@ for i in {0..1}; do [ -e ${DOMAIN_FILE} ] && rm ${DOMAIN_FILE} fi done - -for i in {1..4}; do - NETWORK_INTERFACE_NAME=${NETWORK_INTERFACE}$i - if [ -d "/sys/class/net/${NETWORK_INTERFACE_NAME}" ]; then - sudo ifconfig ${NETWORK_INTERFACE_NAME} down - sudo brctl delbr ${NETWORK_INTERFACE_NAME} - fi -done diff --git a/libvirt/destroy_network.sh b/libvirt/destroy_network.sh new file mode 100755 index 0000000..04e4de5 --- /dev/null +++ b/libvirt/destroy_network.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr} +INTERNAL_NETWORK=${INTERNAL_NETWORK:-10.10.10.0/24} +INTERNAL_IP=${INTERNAL_IP:-10.10.10.1/24} +EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-192.168.204.0/24} +EXTERNAL_IP=${EXTERNAL_IP:-192.168.204.1/24} + +for i in {1..4}; do + BRIDGE_INTERFACE_NAME=${BRIDGE_INTERFACE}$i + if [ -d "/sys/class/net/${BRIDGE_INTERFACE_NAME}" ]; then + sudo ifconfig ${BRIDGE_INTERFACE_NAME} down + sudo brctl delbr ${BRIDGE_INTERFACE_NAME} + fi +done diff --git a/libvirt/destroy_standard_controller.sh b/libvirt/destroy_standard_controller.sh index d0fa73a..d74292c 100755 --- a/libvirt/destroy_standard_controller.sh +++ b/libvirt/destroy_standard_controller.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -CONTROLLER=controller -COMPUTE=compute +BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr} +CONTROLLER=${CONTROLLER:-controller} +COMPUTE=${COMPUTE:-compute} DOMAIN_DIRECTORY=vms -NETWORK_INTERFACE=virbr for i in {0..1}; do CONTROLLER_NODE=${CONTROLLER}-${i} @@ -36,11 +36,3 @@ for i in {0..1}; do [ -e ${DOMAIN_FILE} ] && rm ${DOMAIN_FILE} fi done - -for i in {1..4}; do - NETWORK_INTERFACE_NAME=${NETWORK_INTERFACE}${i} - if [ -d "/sys/class/net/${NETWORK_INTERFACE_NAME}" ]; then - sudo ifconfig ${NETWORK_INTERFACE_NAME} down - sudo brctl delbr ${NETWORK_INTERFACE_NAME} - fi -done diff --git a/libvirt/install_packages.sh b/libvirt/install_packages.sh index 114b95c..c9ae310 100755 --- a/libvirt/install_packages.sh +++ b/libvirt/install_packages.sh @@ -1,24 +1,11 @@ #!/usr/bin/env bash - -NETWORK_DEFAULT=default -INTERFACE=virbr0 +# install_packages.sh - install required packages sudo apt-get install virt-manager libvirt-bin qemu-system -y -if virsh net-list --name | grep ${NETWORK_DEFAULT} ; then - sudo virsh net-destroy ${NETWORK_DEFAULT} - sudo virsh net-undefine ${NETWORK_DEFAULT} - sudo rm -rf /etc/libvirt/qemu/networks/autostart/${NETWORK_DEFAULT}.xml -fi - cat << EOF | sudo tee /etc/libvirt/qemu.conf user = "root" group = "root" EOF sudo service libvirt-bin restart - -if [ -d "/sys/class/net/${INTERFACE}" ]; then - sudo ifconfig ${INTERFACE} down || true - sudo brctl delbr ${INTERFACE} || true -fi diff --git a/libvirt/setup_allinone.sh b/libvirt/setup_allinone.sh index f3678d5..b228efc 100755 --- a/libvirt/setup_allinone.sh +++ b/libvirt/setup_allinone.sh @@ -26,43 +26,37 @@ if [ -z "${ISOIMAGE}" ]; then exit -1 fi +ISOIMAGE=$(readlink -f "$ISOIMAGE") FILETYPE=$(file --mime-type -b ${ISOIMAGE}) if ([ "$FILETYPE" != "application/x-iso9660-image" ]); then echo "$ISOIMAGE is not an application/x-iso9660-image type" exit -1 fi -CONTROLLER=controller-allinone +BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr} +CONTROLLER=${CONTROLLER:-controller-allinone} DOMAIN_DIRECTORY=vms DOMAIN_FILE=$DOMAIN_DIRECTORY/$CONTROLLER.xml -NETWORK_INTERFACE=virbr bash destroy_allinone.sh [ ! -d ${DOMAIN_DIRECTORY} ] && mkdir ${DOMAIN_DIRECTORY} -for i in {1..4}; do - sudo brctl addbr ${NETWORK_INTERFACE}$i -done - -sudo ifconfig ${NETWORK_INTERFACE}1 10.10.10.1/24 up -sudo ifconfig ${NETWORK_INTERFACE}2 192.168.204.1/24 up -sudo ifconfig ${NETWORK_INTERFACE}3 up -sudo ifconfig ${NETWORK_INTERFACE}4 up -sudo iptables -t nat -A POSTROUTING -s 10.10.10.0/24 -j MASQUERADE - - for i in {0..1}; do CONTROLLER_NODE=${CONTROLLER}-${i} sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-0.img 600G sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-1.img 200G - ISOIMAGE=`pwd`/`ls ${ISOIMAGE}` + ISOIMAGE=${ISOIMAGE} DOMAIN_FILE=${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml cp controller.xml ${DOMAIN_FILE} sed -i -e " s,NAME,${CONTROLLER_NODE}, s,DISK0,/var/lib/libvirt/images/${CONTROLLER_NODE}-0.img, s,DISK1,/var/lib/libvirt/images/${CONTROLLER_NODE}-1.img, + s,%BR1%,${BRIDGE_INTERFACE}1, + s,%BR2%,${BRIDGE_INTERFACE}2, + s,%BR3%,${BRIDGE_INTERFACE}3, + s,%BR4%,${BRIDGE_INTERFACE}4, " ${DOMAIN_FILE} if [ $i -eq 0 ]; then sed -i -e "s,ISO,${ISOIMAGE}," ${DOMAIN_FILE} diff --git a/libvirt/setup_network.sh b/libvirt/setup_network.sh new file mode 100755 index 0000000..7d4ed8a --- /dev/null +++ b/libvirt/setup_network.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +usage() { + echo "$0 [-h]" + echo "" + echo "Options:" +# echo " -i: StarlingX ISO image" + echo "" +} + +while getopts "i:" o; do + case "${o}" in + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr} +INTERNAL_NETWORK=${INTERNAL_NETWORK:-10.10.10.0/24} +INTERNAL_IP=${INTERNAL_IP:-10.10.10.1/24} +EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-192.168.204.0/24} +EXTERNAL_IP=${EXTERNAL_IP:-192.168.204.1/24} + +if [[ -r /sys/class/net/${BRIDGE_INTERFACE}1 ]]; then + echo "${BRIDGE_INTERFACE}1 exists, cowardly refusing to overwrite it, exiting..." + exit 1 +fi + +for i in {1..4}; do + sudo brctl addbr ${BRIDGE_INTERFACE}$i +done + +sudo ifconfig ${BRIDGE_INTERFACE}1 $INTERNAL_IP up +sudo ifconfig ${BRIDGE_INTERFACE}2 $EXTERNAL_IP up +sudo ifconfig ${BRIDGE_INTERFACE}3 up +sudo ifconfig ${BRIDGE_INTERFACE}4 up +sudo iptables -t nat -A POSTROUTING -s $EXTERNAL_NETWORK -j MASQUERADE diff --git a/libvirt/setup_standard_controller.sh b/libvirt/setup_standard_controller.sh index 5cdef0d..6ed6f08 100755 --- a/libvirt/setup_standard_controller.sh +++ b/libvirt/setup_standard_controller.sh @@ -28,42 +28,37 @@ if [ -z "${ISOIMAGE}" ]; then exit -1 fi +ISOIMAGE=$(readlink -f "$ISOIMAGE") FILETYPE=$(file --mime-type -b ${ISOIMAGE}) if ([ "$FILETYPE" != "application/x-iso9660-image" ]); then echo "$ISOIMAGE is not an application/x-iso9660-image type" exit -1 fi -CONTROLLER=controller -COMPUTE=compute +BRIDGE_INTERFACE=${BRIDGE_INTERFACE:-stxbr} +CONTROLLER=${CONTROLLER:-controller} +COMPUTE=${COMPUTE:-compute} DOMAIN_DIRECTORY=vms -NETWORK_INTERFACE=virbr bash destroy_standard_controller.sh [ ! -d ${DOMAIN_DIRECTORY} ] && mkdir ${DOMAIN_DIRECTORY} -for i in {1..4}; do - sudo brctl addbr ${NETWORK_INTERFACE}$i -done - -sudo ifconfig ${NETWORK_INTERFACE}1 10.10.10.1/24 up -sudo ifconfig ${NETWORK_INTERFACE}2 192.168.204.1/24 up -sudo ifconfig ${NETWORK_INTERFACE}3 up -sudo ifconfig ${NETWORK_INTERFACE}4 up -sudo iptables -t nat -A POSTROUTING -s 10.10.10.0/24 -j MASQUERADE - for i in {0..1}; do CONTROLLER_NODE=${CONTROLLER}-${i} sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-0.img 200G sudo qemu-img create -f qcow2 /var/lib/libvirt/images/${CONTROLLER_NODE}-1.img 200G - ISOIMAGE=`pwd`/`ls ${ISOIMAGE}` + ISOIMAGE=${ISOIMAGE} DOMAIN_FILE=${DOMAIN_DIRECTORY}/${CONTROLLER_NODE}.xml cp controller.xml ${DOMAIN_FILE} sed -i -e " s,NAME,${CONTROLLER_NODE}, s,DISK0,/var/lib/libvirt/images/${CONTROLLER_NODE}-0.img, s,DISK1,/var/lib/libvirt/images/${CONTROLLER_NODE}-1.img, + s,%BR1%,${BRIDGE_INTERFACE}1, + s,%BR2%,${BRIDGE_INTERFACE}2, + s,%BR3%,${BRIDGE_INTERFACE}3, + s,%BR4%,${BRIDGE_INTERFACE}4, " ${DOMAIN_FILE} if [ $i -eq 0 ]; then sed -i -e "s,ISO,${ISOIMAGE}," ${DOMAIN_FILE} @@ -86,6 +81,10 @@ for i in {0..1}; do s,NAME,${COMPUTE_NODE},; s,DISK0,/var/lib/libvirt/images/${COMPUTE_NODE}-0.img,; s,DISK1,/var/lib/libvirt/images/${COMPUTE_NODE}-1.img, + s,%BR1%,${BRIDGE_INTERFACE}1, + s,%BR2%,${BRIDGE_INTERFACE}2, + s,%BR3%,${BRIDGE_INTERFACE}3, + s,%BR4%,${BRIDGE_INTERFACE}4, " ${DOMAIN_FILE} sudo virsh define ${DOMAIN_FILE} done