diff --git a/dib-elements/README.rst b/dib-elements/README.rst deleted file mode 100644 index e701fce..0000000 --- a/dib-elements/README.rst +++ /dev/null @@ -1,74 +0,0 @@ -Using diskimage-builder to build opendev-ci nodes -==================================================== - -In addition to being able to just download and consume images that are the -same as what run devstack, it's easy to make your own for local dev or -testing - or just for fun. - -Install diskimage-builder -------------------------- - -Install the dependencies: - -:: - - sudo apt-get install kpartx qemu-utils curl python-yaml debootstrap - -Install diskimage-builder: - -:: - - sudo -H pip install diskimage-builder - - -Build an image --------------- - -Building an image is simple, we have a script! - -:: - - bash tools/build-image.sh - -See the script for environment variables to set distribution, etc. By default -it builds an ubuntu-minimal based image. You should be left with a .qcow2 -image file of your selected distribution. - -Infra uses the -minimal build type for building Ubuntu/CentOS/Fedora. For -example: ubuntu-minimal. - -It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space -to avoid the disappointment of a full-disk mid-way through the script -run. - -While testing, consider exporting DIB_OFFLINE=true, to skip updating the cache. - -Mounting the image ------------------- - -If you would like to examine the contents of the image, you can mount it on -a loopback device using qemu-nbd. - -:: - - sudo apt-get install qemu-utils - sudo modprobe nbd max_part=16 - sudo mkdir -p /tmp/newimage - sudo qemu-nbd -c /dev/nbd1 /path/to/opendev-ci-node-precise.qcow2 - sudo mount /dev/nbd1p1 /tmp/newimage - -or use the scripts - -:: - - sudo apt-get install qemu-utils - sudo modprobe nbd max_part=16 - sudo tools/mount-image.sh opendev-ci-node-precise.qcow2 - sudo tools/umount-image.sh - -Other things ------------- - -It's a qcow2 image, so you can do tons of things with it. You can upload it -to glance, you can boot it using kvm, and you can even copy it to a cloud -server, replace the contents of the server with it and kexec the new kernel. diff --git a/dib-elements/bindep-fallback.txt b/dib-elements/bindep-fallback.txt deleted file mode 100644 index 5936a19..0000000 --- a/dib-elements/bindep-fallback.txt +++ /dev/null @@ -1,114 +0,0 @@ -# This is the fallback list for packages to install. Do not add -# additional packages here. Repositories should use bindep and create -# their own bindep.txt files if the list below is not -# working for them. - -build-essential [platform:dpkg] -curl [!platform:gentoo] -net-misc/curl [platform:gentoo] -dev-libs/cyrus-sasl [platform:gentoo] -cyrus-sasl-devel [platform:rpm] -media-fonts/nanumfont [platform:gentoo] -fonts-nanum [platform:dpkg] -media-fonts/takao-fonts [platform:gentoo] -fonts-takao [platform:dpkg] -gawk -gettext [!platform:suse] -gettext-runtime [platform:suse] -graphviz [!platform:gentoo] -media-gfx/graphviz [platform:gentoo] -language-pack-en [platform:ubuntu] -libcurl-devel [platform:rpm] -libcurl4-gnutls-dev [platform:dpkg] -libevent-dev [platform:dpkg] -libevent-devel [platform:rpm] -dev-libs/libevent [platform:gentoo] -libffi-dev [platform:dpkg] -libffi-devel [platform:redhat platform:suse] -virtual/libffi [platform:gentoo] -libjerasure-dev [platform:ubuntu] -dev-libs/jerasure [platform:gentoo] -libjpeg-dev [platform:dpkg] -libjpeg8-devel [platform:suse] -libjpeg-turbo-devel [platform:redhat] -media-libs/libjpeg-turbo [platform:gentoo] -libldap2-dev [platform:dpkg] -net-nds/openldap [platform:gentoo] -libmysqlclient-dev [platform:dpkg] -libpcap-dev [platform:dpkg] -libpcap-devel [platform:rpm] -net-libs/libpcap [platform:gentoo] -libpq-dev [platform:dpkg] -librrd-dev [platform:dpkg] -net-analyzer/rrdtool [platform:gentoo] -libsasl2-dev [platform:dpkg] -libselinux-python [platform:redhat] -python-selinux [platform:suse] -sys-libs/libselinux [platform:gentoo] -libsqlite3-dev [platform:dpkg] -libuuid-devel [platform:rpm] -libvirt-dev [platform:dpkg] -libvirt-devel [platform:rpm] -app-emulation/libvirt [platform:gentoo] -libvirt-python [platform:rpm !platform:suse] -dev-python/libvirt-python [platform:gentoo] -libxml2-dev [platform:dpkg] -libxml2-devel [platform:rpm] -libxml2-utils [platform:dpkg] -dev-libs/libxml2 [platform:gentoo] -libxslt-devel [platform:rpm] -libxslt1-dev [platform:dpkg] -dev-libs/libxslt [platform:gentoo] -locales [platform:debian] -mariadb [platform:rpm] -mariadb-server [platform:redhat] -mariadb-devel [platform:redhat] -libmysqlclient-devel [platform:suse] -dev-db/mariadb [platform:gentoo] -memcached -mysql-client [platform:dpkg] -mysql-server [platform:dpkg] -openldap-devel [platform:redhat] -openldap2-devel [platform:suse] -pkg-config [platform:dpkg platform:suse] -pkgconfig [platform:redhat] -virtual/pkgconfig [platform:gentoo] -postgresql -postgresql-client [platform:dpkg] -postgresql-devel [platform:rpm] -postgresql-server [platform:rpm] -pypy [platform:ubuntu] -pypy-dev [platform:ubuntu] -python-dev [platform:dpkg] -python-devel [platform:rpm] -dev-lang/python [platform:gentoo] -python-libvirt [platform:dpkg] -python-lxml [!platform:gentoo !platform:fedora] -python2-lxml [platform:fedora] -dev-python/lxml [platform:gentoo] -# Note that python3-all-dev includes python3-all, added -# both here for documentary purpose. -python3-all [platform:dpkg] -python3-all-dev [platform:dpkg] -python3-devel [platform:fedora platform:suse] -# python3-devel does not pull in the python3 package on openSUSE so -# we need to be explicit. The python3 package contains the XML module -# which is required by a python3 virtualenv. Similarly, in python2, -# the XML module is located in python-xml which is not pulled in -# by python-devel as well. See https://bugzilla.suse.com/show_bug.cgi?id=1046990 -python3 [platform:suse] -python-xml [platform:suse] -rrdtool-devel [platform:rpm] -sqlite [platform:redhat] -sqlite-devel [platform:redhat] -sqlite3-devel [platform:suse] -sqlite3 [platform:dpkg] -dev-db/sqlite [platform:gentoo] -swig -unzip -uuid-dev [platform:dpkg] -xsltproc [platform:dpkg] -zip -zlib-devel [platform:rpm] -zlib1g-dev [platform:dpkg] -sys-libs/zlib [platform:gentoo] diff --git a/dib-elements/cache-devstack/README.rst b/dib-elements/cache-devstack/README.rst deleted file mode 100644 index 073f045..0000000 --- a/dib-elements/cache-devstack/README.rst +++ /dev/null @@ -1,17 +0,0 @@ -cache-devstack -============== - -Pre-cache a range of things into CI images. This element uses the -``source-repositories`` element to acquire files to be cached. The -standard cache location is ``/opt/cache/files``. - -A number of strategies are used to get the files to be cached. - -We have a number of ``source-repository-*`` files for each package -package that should be cached into images. - -``extra-data.d/55-cache-devstack-repos`` goes through each devstack -branch and runs the ``tools/image_list.sh`` script to dynamically -build a list of files to cache as requested by devstack. This is -mostly virtual machine images, but also some other peripheral packages. - diff --git a/dib-elements/cache-devstack/element-deps b/dib-elements/cache-devstack/element-deps deleted file mode 100644 index 7b2a984..0000000 --- a/dib-elements/cache-devstack/element-deps +++ /dev/null @@ -1 +0,0 @@ -openstack-repos diff --git a/dib-elements/cache-devstack/source-repository-dstat_graph b/dib-elements/cache-devstack/source-repository-dstat_graph deleted file mode 100644 index eb49e25..0000000 --- a/dib-elements/cache-devstack/source-repository-dstat_graph +++ /dev/null @@ -1 +0,0 @@ -dstat_graph git /opt/cache/dstat_graph/ https://opendev.org/opendev/dstat_graph master diff --git a/dib-elements/cache-devstack/source-repository-images b/dib-elements/cache-devstack/source-repository-images deleted file mode 100644 index 4e56a16..0000000 --- a/dib-elements/cache-devstack/source-repository-images +++ /dev/null @@ -1,15 +0,0 @@ -cirros-0.5.2-x86_64-disk.img file /opt/cache/files/cirros-0.5.2-x86_64-disk.img https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-disk.img -cirros-0.5.2-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.5.2-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-x86_64-uec.tar.gz -cirros-0.5.2-aarch64-disk.img file /opt/cache/files/cirros-0.5.2-aarch64-disk.img https://download.cirros-cloud.net/0.5.2/cirros-0.5.2-aarch64-disk.img -cirros-0.5.3-x86_64-disk.img file /opt/cache/files/cirros-0.5.3-x86_64-disk.img https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-disk.img -cirros-0.5.3-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.5.3-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-x86_64-uec.tar.gz -cirros-0.5.3-aarch64-disk.img file /opt/cache/files/cirros-0.5.3-aarch64-disk.img https://download.cirros-cloud.net/0.5.3/cirros-0.5.3-aarch64-disk.img -cirros-0.6.1-x86_64-disk.img file /opt/cache/files/cirros-0.6.1-x86_64-disk.img https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img -cirros-0.6.1-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.6.1-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-uec.tar.gz -cirros-0.6.1-aarch64-disk.img file /opt/cache/files/cirros-0.6.1-aarch64-disk.img https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img -cirros-0.6.2-x86_64-disk.img file /opt/cache/files/cirros-0.6.2-x86_64-disk.img https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-disk.img -cirros-0.6.2-x86_64-uec.tar.gz file /opt/cache/files/cirros-0.6.2-x86_64-uec.tar.gz https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-x86_64-uec.tar.gz -cirros-0.6.2-aarch64-disk.img file /opt/cache/files/cirros-0.6.2-aarch64-disk.img https://download.cirros-cloud.net/0.6.2/cirros-0.6.2-aarch64-disk.img -etcd-v3.1.10-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.1.10-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.1.10/etcd-v3.1.10-linux-amd64.tar.gz -etcd-v3.2.17-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.2.17-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.2.17/etcd-v3.2.17-linux-amd64.tar.gz -etcd-v3.3.12-linux-amd64.tar.gz file /opt/cache/files/etcd-v3.3.12-linux-amd64.tar.gz https://github.com/etcd-io/etcd/releases/download/v3.3.12/etcd-v3.3.12-linux-amd64.tar.gz diff --git a/dib-elements/cache-devstack/source-repository-pip b/dib-elements/cache-devstack/source-repository-pip deleted file mode 100644 index 9c309aa..0000000 --- a/dib-elements/cache-devstack/source-repository-pip +++ /dev/null @@ -1,2 +0,0 @@ -get-pip-py file /opt/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py -get-pip-py-py36 file /opt/cache/files/get-pip.py-py36 https://bootstrap.pypa.io/pip/3.6/get-pip.py diff --git a/dib-elements/cache-devstack/source-repository-stackviz b/dib-elements/cache-devstack/source-repository-stackviz deleted file mode 100644 index 48a4647..0000000 --- a/dib-elements/cache-devstack/source-repository-stackviz +++ /dev/null @@ -1 +0,0 @@ -stackviz file /opt/cache/files/stackviz-latest.tar.gz https://tarballs.openstack.org/stackviz/dist/stackviz-latest.tar.gz diff --git a/dib-elements/cache-devstack/source-repository-zanatacli b/dib-elements/cache-devstack/source-repository-zanatacli deleted file mode 100644 index 63e9bcb..0000000 --- a/dib-elements/cache-devstack/source-repository-zanatacli +++ /dev/null @@ -1 +0,0 @@ -zanata-cli file /opt/cache/files/zanata-cli-4.3.3-dist.tar.gz https://search.maven.org/remotecontent?filepath=org/zanata/zanata-cli/4.3.3/zanata-cli-4.3.3-dist.tar.gz diff --git a/dib-elements/control-plane-minimal/README.rst b/dib-elements/control-plane-minimal/README.rst deleted file mode 100644 index 9cc95e6..0000000 --- a/dib-elements/control-plane-minimal/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -control-plane-minimal ---------------------- - -Essential packages for an OpenDev infra control-plane image. diff --git a/dib-elements/control-plane-minimal/element-deps b/dib-elements/control-plane-minimal/element-deps deleted file mode 100644 index 89d13d0..0000000 --- a/dib-elements/control-plane-minimal/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -openssh-server -package-installs diff --git a/dib-elements/control-plane-minimal/package-installs.yaml b/dib-elements/control-plane-minimal/package-installs.yaml deleted file mode 100644 index 96097aa..0000000 --- a/dib-elements/control-plane-minimal/package-installs.yaml +++ /dev/null @@ -1,11 +0,0 @@ -acpid: -coreutils: -cron: -util-linux: -python3: - phase: pre-install.d -ntp: -ntpdate: -lvm2: -haveged: -iptables: diff --git a/dib-elements/control-plane-minimal/post-install.d/80-enable-haveged b/dib-elements/control-plane-minimal/post-install.d/80-enable-haveged deleted file mode 100755 index 49f6031..0000000 --- a/dib-elements/control-plane-minimal/post-install.d/80-enable-haveged +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "$DIB_INIT_SYSTEM" in - systemd) - systemctl enable haveged.service - ;; - *) - echo "Unsupported init system $DIB_INIT_SYSTEM" - exit 1 - ;; -esac diff --git a/dib-elements/control-plane-minimal/post-install.d/80-enable-infra-services b/dib-elements/control-plane-minimal/post-install.d/80-enable-infra-services deleted file mode 100755 index e7698dc..0000000 --- a/dib-elements/control-plane-minimal/post-install.d/80-enable-infra-services +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "$DIB_INIT_SYSTEM" in - systemd) - systemctl enable ntp.service - ;; - *) - echo "Unsupported init system $DIB_INIT_SYSTEM" - exit 1 - ;; -esac diff --git a/dib-elements/control-plane-minimal/post-install.d/89-sshd b/dib-elements/control-plane-minimal/post-install.d/89-sshd deleted file mode 100755 index 46cc60f..0000000 --- a/dib-elements/control-plane-minimal/post-install.d/89-sshd +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Copyright (C) 2011-2013 OpenStack Foundation -# Copyright 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# NOTE(pabelanger): Glean configures access for root user, so allow us to -# properly login. -sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \ - && echo "PermitRootLogin yes" >> /etc/ssh/sshd_config - -# NOTE(clarkb): Glean configures ssh keys only and not passwords. Disable -# unnecessary password auth. -sed -i -e '/PasswordAuthentication/d' /etc/ssh/sshd_config \ - && echo "PasswordAuthentication no" >> /etc/ssh/sshd_config diff --git a/dib-elements/infra-package-needs/README.rst b/dib-elements/infra-package-needs/README.rst deleted file mode 100644 index 55fa581..0000000 --- a/dib-elements/infra-package-needs/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -infra-package-needs -------------------- - -Install required packages for a OpenDev infra host diff --git a/dib-elements/infra-package-needs/element-deps b/dib-elements/infra-package-needs/element-deps deleted file mode 100644 index 77cf654..0000000 --- a/dib-elements/infra-package-needs/element-deps +++ /dev/null @@ -1,3 +0,0 @@ -ensure-venv -openssh-server -package-installs diff --git a/dib-elements/infra-package-needs/install.d/10-packages b/dib-elements/infra-package-needs/install.d/10-packages deleted file mode 100755 index 2fe1b7e..0000000 --- a/dib-elements/infra-package-needs/install.d/10-packages +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -if [[ "$DISTRO_NAME" =~ ^(centos|rocky)$ && ! ${DIB_RELEASE} =~ '9' ]] ; then - # 9-stream we are just using the default rngd - # Note: $YUM exposed by centos|rhel-like environment, correct across releases - ${YUM} -y install --enablerepo=epel haveged -fi diff --git a/dib-elements/infra-package-needs/install.d/40-install-bindep b/dib-elements/infra-package-needs/install.d/40-install-bindep deleted file mode 100755 index 3cea319..0000000 --- a/dib-elements/infra-package-needs/install.d/40-install-bindep +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Copyright (C) 2015 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# Install bindep into a virtualenv -# This is in /usr instead of /usr/local due to this bug on precise: -# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588 - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -python3 -m venv /usr/bindep-env -if [[ ${DIB_RELEASE} == 'xenial' ]]; then - # The pip on xenial can't figure out it shouldn't install - # the latest pip; this is the last to support 3.5 - /usr/bindep-env/bin/pip install --upgrade pip==20.3.4 -else - /usr/bindep-env/bin/pip install --upgrade pip -fi -/usr/bindep-env/bin/pip install bindep diff --git a/dib-elements/infra-package-needs/install.d/89-rsyslog b/dib-elements/infra-package-needs/install.d/89-rsyslog deleted file mode 100755 index fe3b0d6..0000000 --- a/dib-elements/infra-package-needs/install.d/89-rsyslog +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -if [ "$DISTRO_NAME" == "ubuntu" ]; then - rsyslog_dir="$(dirname $0)/../rsyslog.d" - cp -RP $rsyslog_dir/* /etc/rsyslog.d/ -fi diff --git a/dib-elements/infra-package-needs/package-installs.yaml b/dib-elements/infra-package-needs/package-installs.yaml deleted file mode 100644 index 3bf1a69..0000000 --- a/dib-elements/infra-package-needs/package-installs.yaml +++ /dev/null @@ -1,28 +0,0 @@ -acl: -acpid: -coreutils: -cron: -util-linux: -build-essential: -python3-dev: -uuid-runtime: -traceroute: -ntp: -ntpdate: -gentoolkit: -at: -strace: -tcpdump: -rsyslog: -git: -rsync: -parted: -wget: -iputils-ping: -iproute2: -dnsutils: -haveged: -iptables: -redhat-rpm-config: -redhat-lsb-core: -gnupg2: diff --git a/dib-elements/infra-package-needs/pkg-map b/dib-elements/infra-package-needs/pkg-map deleted file mode 100644 index 2742099..0000000 --- a/dib-elements/infra-package-needs/pkg-map +++ /dev/null @@ -1,117 +0,0 @@ -{ - "release": { - "centos": { - "8": { - "ntp": "chrony", - "ntpdate": "" - }, - "9-stream": { - "ntp": "chrony", - "ntpdate": "", - "iptables": "iptables-services" - } - }, - "debian": { - "bookworm": { - "ntp": "", - "ntpdate": "" - } - }, - "rocky": { - "8": { - "ntp": "chrony", - "ntpdate": "", - "coreutils": "" - }, - "9": { - "ntp": "chrony", - "ntpdate": "", - "iptables": "iptables-services", - "coreutils": "" - } - }, - "ubuntu": { - "focal": { - "ntp": "", - "ntpdate": "" - }, - "jammy": { - "ntp": "", - "ntpdate": "" - }, - "noble": { - "ntp": "", - "ntpdate": "" - } - } - }, - "distro": { - "fedora": { - "python3-dev": "python3-devel", - "iptables": "iptables-services", - "haveged": "haveged", - "ntp": "chrony", - "ntpdate": "", - "redhat-rpm-config": "redhat-rpm-config", - "redhat-lsb-core": "redhat-lsb-core" - }, - "openeuler": { - "cron": "cronie", - "build-essential": "glibc-devel gcc make", - "dnsutils": "bind-utils", - "iproute2": "iproute", - "iputils-ping": "iputils", - "ntp": "ntp ntp-perl", - "python3-dev": "python3-devel", - "iptables": "iptables-services", - "haveged": "haveged" - }, - "gentoo": { - "acl": "sys-apps/acl", - "build-essential": "", - "cron": "sys-process/cronie", - "dnsutils": "net-dns/bind-tools", - "gentoolkit": "app-portage/gentoolkit", - "git": "dev-vcs/git", - "gnupg2": "app-crypt/gnupg", - "iptables": "net-firewall/iptables", - "iputils-ping": "net-misc/iputils", - "ntp": "net-misc/ntp", - "ntpdate": "net-misc/ntp", - "python3-dev": "", - "traceroute": "net-analyzer/traceroute", - "uuid-runtime": "" - } - }, - "family": { - "redhat": { - "cron": "cronie", - "build-essential": "glibc-devel gcc make", - "dnsutils": "bind-utils", - "iproute2": "iproute", - "iputils-ping": "iputils", - "ntp": "ntp ntp-perl", - "python3-dev": "", - "uuid-runtime": "", - "iptables": "iptables-services", - "haveged": "" - }, - "suse": { - "build-essential": "glibc-devel gcc make", - "dnsutils": "bind-utils", - "git": "git-core", - "gnupg2": "gpg2", - "iputils-ping": "iputils", - "ntpdate": "", - "python3-dev": "python3-devel", - "iptables": "iptables", - "uuid-runtime": "uuidd" - } - }, - "default": { - "iptables": "iptables-persistent", - "gentoolkit": "", - "redhat-rpm-config": "", - "redhat-lsb-core": "" - } -} diff --git a/dib-elements/infra-package-needs/post-install.d/80-enable-haveged b/dib-elements/infra-package-needs/post-install.d/80-enable-haveged deleted file mode 100755 index b07a4d9..0000000 --- a/dib-elements/infra-package-needs/post-install.d/80-enable-haveged +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -if [[ "$DISTRO_NAME" =~ ^(centos|rocky)$ && ${DIB_RELEASE} =~ '9' ]]; then - # 9/9-stream we are just using the default rngd - exit 0 -fi - -case "$DIB_INIT_SYSTEM" in - upstart) - # nothing to do - exit 0 - ;; - systemd) - systemctl enable haveged.service - ;; - openrc) - rc-update add haveged default - ;; - sysv) - exit 0 - ;; - *) - echo "Unsupported init system $DIB_INIT_SYSTEM" - exit 1 - ;; -esac diff --git a/dib-elements/infra-package-needs/post-install.d/80-enable-infra-services b/dib-elements/infra-package-needs/post-install.d/80-enable-infra-services deleted file mode 100755 index cb8767e..0000000 --- a/dib-elements/infra-package-needs/post-install.d/80-enable-infra-services +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "$DIB_INIT_SYSTEM" in - upstart) - # nothing to do - exit 0 - ;; - systemd) - # stick with default systemd timesyncd on bookworm, focal and beyond - if [[ ":bookworm: :focal: :jammy: :noble:" =~ :${DIB_RELEASE}: ]]; then - exit 0 - elif [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then - systemctl enable ntp.service - elif [[ ( $DISTRO_NAME == "centos" && $DIB_RELEASE > 7 ) || $DISTRO_NAME == "fedora" || $DISTRO_NAME == "rocky" ]]; then - systemctl enable chronyd - else - systemctl enable ntpd.service - fi - ;; - openrc) - rc-update add ntp-client default - rc-update add acpid default - ;; - sysv) - # ntp is enabled by default, nothing to do - exit 0 - ;; - *) - echo "Unsupported init system $DIB_INIT_SYSTEM" - exit 1 - ;; -esac diff --git a/dib-elements/infra-package-needs/post-install.d/89-sshd b/dib-elements/infra-package-needs/post-install.d/89-sshd deleted file mode 100755 index 0f1b374..0000000 --- a/dib-elements/infra-package-needs/post-install.d/89-sshd +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# Copyright (C) 2011-2013 OpenStack Foundation -# Copyright 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# NOTE(pabelanger): Glean configures access for root user, so allow us to -# properly login. -sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \ - && echo "PermitRootLogin yes" >> /etc/ssh/sshd_config - -# NOTE(clarkb): Glean configures ssh keys only and not passwords. Disable -# unnecessary password auth. -sed -i -e '/PasswordAuthentication/d' /etc/ssh/sshd_config \ - && echo "PasswordAuthentication no" >> /etc/ssh/sshd_config - -# NOTE(clarkb): SSH scanners may be affecting Zuul ssh connectivity -# Default LoginGraceTime is 120. Reduce that to 30 to cycle connections more -# quickly. -sed -i -e '/LoginGraceTime/d' /etc/ssh/sshd_config \ - && echo "LoginGraceTime 30" >> /etc/ssh/sshd_config - -# NOTE(clarkb): SSH scanners may be affecting Zuul ssh connectivity -# Default MaxStartups is 10:30:100 which means after 10 unauthenticated -# connections randomly drop 30% of connections with an increasing -# percentage until 100 connections is reached. -sed -i -e '/MaxStartups/d' /etc/ssh/sshd_config \ - && echo "MaxStartups 30:10:100" >> /etc/ssh/sshd_config diff --git a/dib-elements/infra-package-needs/pre-install.d/00-gentoo-useflags b/dib-elements/infra-package-needs/pre-install.d/00-gentoo-useflags deleted file mode 100755 index b5c3bb7..0000000 --- a/dib-elements/infra-package-needs/pre-install.d/00-gentoo-useflags +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# needed for unbound -if [ "${DISTRO_NAME}" == "gentoo" ]; then - mkdir -p /etc/portage/package.use - echo "app-admin/rsyslog openssl" >> /etc/portage/package.use/rsyslog - echo "dev-lang/python sqlite" >> /etc/portage/package.use/python - echo "dev-libs/openssl static-libs -bindist" >> /etc/portage/package.use/openssl - echo "net-misc/openssh -bindist" >> /etc/portage/package.use/openssh - echo "sys-apps/systemd audit curl gcrypt importd lzma nat" >> /etc/portage/package.use/systemd - echo "sys-libs/zlib static-libs" >> /etc/portage/package.use/zlib -fi diff --git a/dib-elements/infra-package-needs/rsyslog.d/50-default.conf b/dib-elements/infra-package-needs/rsyslog.d/50-default.conf deleted file mode 100644 index 528af29..0000000 --- a/dib-elements/infra-package-needs/rsyslog.d/50-default.conf +++ /dev/null @@ -1,68 +0,0 @@ -# Default rules for rsyslog. -# -# For more information see rsyslog.conf(5) and /etc/rsyslog.conf - -# -# First some standard log files. Log by facility. -# -auth,authpriv.* /var/log/auth.log -*.*;auth,authpriv.none -/var/log/syslog -#cron.* /var/log/cron.log -#daemon.* -/var/log/daemon.log -kern.* -/var/log/kern.log -#lpr.* -/var/log/lpr.log -mail.* -/var/log/mail.log -#user.* -/var/log/user.log - -# -# Logging for the mail system. Split it up so that -# it is easy to write scripts to parse these files. -# -#mail.info -/var/log/mail.info -#mail.warn -/var/log/mail.warn -mail.err /var/log/mail.err - -# -# Logging for INN news system. -# -news.crit /var/log/news/news.crit -news.err /var/log/news/news.err -news.notice -/var/log/news/news.notice - -# -# Some "catch-all" log files. -# -#*.=debug;\ -# auth,authpriv.none;\ -# news.none;mail.none -/var/log/debug -#*.=info;*.=notice;*.=warn;\ -# auth,authpriv.none;\ -# cron,daemon.none;\ -# mail,news.none -/var/log/messages - -# -# Emergencies are sent to everybody logged in. -# -*.emerg :omusrmsg:* - -# -# I like to have messages displayed on the console, but only on a virtual -# console I usually leave idle. -# -#daemon,mail.*;\ -# news.=crit;news.=err;news.=notice;\ -# *.=debug;*.=info;\ -# *.=notice;*.=warn /dev/tty8 - -# The named pipe /dev/xconsole is for the `xconsole' utility. To use it, -# you must invoke `xconsole' with the `-file' option: -# -# $ xconsole -file /dev/xconsole [...] -# -# NOTE: adjust the list below, or you'll go crazy if you have a reasonably -# busy site.. -# -#daemon.*;mail.*;\ -# news.err;\ -# *.=debug;*.=info;\ -# *.=notice;*.=warn |/dev/xconsole diff --git a/dib-elements/initialize-urandom/README.rst b/dib-elements/initialize-urandom/README.rst deleted file mode 100644 index c44a20b..0000000 --- a/dib-elements/initialize-urandom/README.rst +++ /dev/null @@ -1,2 +0,0 @@ -This uses haveged to quickly initialize the nonblocking kernel random -number generator at boot. diff --git a/dib-elements/initialize-urandom/element-deps b/dib-elements/initialize-urandom/element-deps deleted file mode 100644 index 69a71fd..0000000 --- a/dib-elements/initialize-urandom/element-deps +++ /dev/null @@ -1,2 +0,0 @@ -dib-init-system -install-static diff --git a/dib-elements/initialize-urandom/init-scripts/systemd/initialize-urandom.service b/dib-elements/initialize-urandom/init-scripts/systemd/initialize-urandom.service deleted file mode 100644 index cc4489e..0000000 --- a/dib-elements/initialize-urandom/init-scripts/systemd/initialize-urandom.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Quickly initialize the nonblocking kernel random number generator at boot. -Before=network-pre.target -Wants=network-pre.target - -[Service] -Type=oneshot -User=root -ExecStart=/usr/local/bin/initialize-urandom.py -RemainAfterExit=true - -[Install] -WantedBy=multi-user.target diff --git a/dib-elements/initialize-urandom/post-install.d/80-initialize-urandom b/dib-elements/initialize-urandom/post-install.d/80-initialize-urandom deleted file mode 100755 index 6174349..0000000 --- a/dib-elements/initialize-urandom/post-install.d/80-initialize-urandom +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -case "$DIB_INIT_SYSTEM" in - systemd) - sudo chmod 0644 /usr/lib/systemd/system/initialize-urandom.service - systemctl enable initialize-urandom.service - ;; - *) - echo "Unsupported init system" - exit 1 - ;; -esac diff --git a/dib-elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py b/dib-elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py deleted file mode 100755 index fa21529..0000000 --- a/dib-elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2016 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import ctypes -import errno -import fcntl -import os -import struct -import subprocess - -"""Add entropy to the kernel until the nonblocking pool is -initialized. - -The Linux kernel has 3 entropy pools: input, blocking, and -nonblocking. Normally entropy accumulates in the input pool and as it -is depleted by the other pools, it is transferred from the input pool -to the others. - -The blocking pool corresponds to /dev/random, where reads from that -device return random numbers only as long as there is sufficient -entropy in the blocking pool. When that entropy is depleted, further -reads from /dev/random block until it is replenished. - -The nonblocking pool corresponds to /dev/urandom, where reads never -block. Even if there is no entropy in the nonblocking pool, random -numbers are still returned. - -The algorithms in use in Linux 3.17 require 128 bits of entropy in -order to initialize the random number generators associated with each -pool. Naturally, reads from /dev/random will not return until the -associated generator is initialized. Reads from /dev/urandom will not -block -- even if the generator is not initialized. The kernel will -output a notice[1] if this happens. - -In order to avoid the situation where urandom is used when -uninitialized, the kernel diverts entropy from timers and interrupts -to the nonblocking pool (instead of the input pool) until it is -initialized. In this way, as the system boots, the nonblocking pool -accumulates entropy first, reducing the time period during which -urandom might produce numbers from an uninitialized generator, and -then the input and blocking pools are filled. - -Beginning with Linux 3.17, the getrandom(2) syscall was added[2] so -that user-space programs that generally would like to use /dev/urandom -can do so without opening a file descriptor and, more relevant here, -can ensure that they do so only after the generator is initialized -(which otherwise is not possible with the /dev/urandom interface). - -Unfortunately, programs which use this interface during early boot may -need to wait some time for the nonblocking pool to accumulate enough -entropy to initialize, and therefore for getrandom to return. -Particularly in the case of a VM, this may take considerable time. - -There are many methods of addressing this shortcoming: - -* Store data from /dev/random at shutdown and use it to seed the - entropy pool at the next boot. Most GNU/Linux distributions do - this. On Ubuntu Xenial, this task is performed by systemd[3]. - Unfortunately, while writes to /dev/random (which is the method - systemd uses to seed the system at boot) do add data to the pool, - they do not increase the internal tracking of the amount of entropy - in the pool. Therefore, for the purposes of determining whether the - nonblocking pool has accumulated 128 bits of entropy, they are not - counted. - -* Use haveged to maintain a sufficient amount of entropy. Haveged can - produce entropy very quickly, and when run at boot, will typically - immediately fill the entropy pool. Haveged performs an ioctl - operation on /dev/random rather than writing data to it, and this - ioctl allows it to specify how much entropy the data it supplies - contains. Therefore, unlike writes to /dev/random, ioctls do - increment the entropy counter. Unfortunately, data from ioctls are - *always* directed to the input pool. While entropy from timers and - interrupts are diverted to the nonblocking pool to speed its - initialization, data arriving from the ioctl instead end up in the - input pool for later use. - - When more entropy than is needed is supplied to the input pool, the - kernel will preemptively transfer some of that entropy to the - secondary (including nonblocking) pools. Since haveged supplies so - much data on startup, some of this entropy should be able to spill - over into the nonblocking pool to aid it in achieving the - initialization threshold. Unfortunately, at the stage of early boot - we are considering, the input pool's generator also has not been - initialized. When the kernel receives a large amount of data from - haveged over the ioctl, it pushes the input pool's generator over - the 128 bit threshold, and initializes the input pool's generator. - When a pool's generator is initialized, the entropy counter for that - pool is reset to zero. This leaves no entropy to spill over to the - nonblocking pool. Haveged is only able to see the entropy count for - the input pool, and therefore is unaware that further contributions - of entropy would aid (via spill-over) in seeding the nonblocking - pool. - - At this point it's worth discussing why the nonblocking pool is - still not initialized despite a full input pool. When a secondary - pool needs more entropy, it can pull from the input pool. However, - there is a timer that only allows the nonblocking pool to withdraw - entropy from the input pool every 60 seconds by default (this can be - adjusted via proc). If something during very early boot reads data - from /dev/urandom, a transfer (from the very likely empty) input - pool is initiated, starting the timer that will prevent another - transfer for 60 seconds, even if the input pool is later filled - (such as by haveged). This means that even with haveged running at - boot the delay due to a blocking getrandom(2) call may still be as - long as 60 seconds. - -* Use rng-tools for the same purpose as haveged. rng-tools operates - in a similar manner to haveged, supplying entropic data to the - kernel via ioctl. However, it does so in smaller chunks. This - means that once the input pool's generator surpasses the 128 bit - threshold for initialization, entropy from the next ioctl from - rng-tools will be available to spill over to the nonblocking pool, - and may be sufficient to initialize it. - - Because of this behavior, use of rng-tools may cause getrandom(2) to - return more quickly at boot, however, this may only happen due to a - quirk of implementation and relies on some specific values and - conditions for the amount of entropy in the input pool at the time - it is run. - -This program speeds initialization of the nonblocking pool by adding -entropy to the input pool in small chunks. To determine when the -nonblocking pool is initialized, it performs the nonblocking -getrandom(2) syscall requesting one byte of random data. As long as -the nonblocking pool is uninitialized, that call will fail and set -errno to EAGAIN. In that case, the program reads 64 bytes of data -from haveged and sends it to the kernel using the ioctl interface, -then repeats this in a loop. That will cause entropy to accumulate in -the input pool until it is initialized and reaches the spill-over -threshold. Further data will accumulate in the nonblocking pool until -it is initialized. Once that occurs, the getrandom(2) call will -return successfully, and the program will exit the loop. - -There are other ways this problem could be addressed (changes to -haveged or rng-tools to support behavior like this, or changes to the -kernel to direct entropy received via ioctl to the nonblocking pool -during initialization), however, this problem is likely to be -short-lived as the nonblocking generator is being replaced[4] in -current kernel versions and should not suffer from the same problem. - -[1] http://lxr.free-electrons.com/source/drivers/char/random.c?v=3.17#L1385 -[2] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\ -?id=c6e9d6f38894798696f23c8084ca7edbf16ee895 -[3] https://www.freedesktop.org/software/systemd/man/systemd-random-seed.\ -service.html -[4] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/\ -?id=e192be9d9a30555aae2ca1dc3aad37cba484cd4a -""" - - -class GeneratorNotInitializedError(Exception): - pass - - -class InterruptedError(Exception): - pass - - -class Pump(object): - # How much data, in bytes, to move at once. 64 is the size of the - # internal kernel buffer, so we match it. - CHUNK_SIZE = 64 - - # The syscall number for getrandom(2). - SYS_getrandom = 318 - - # The IOCTL to add entropy. - OP_RNDADDENTROPY = 0x40085203 - - # Flags for getrandom: - GRND_NONBLOCK = 0x0001 # Do not block - GRND_RANDOM = 0x0002 # Use /dev/random instead of urandom - - def __init__(self): - # Use ctypes to invoke getrandom since it is not available in - # python. os.urandom may call getrandom in some versions of - # python3, however, the blocking on initialization behavior is - # seen as a bug and so os.urandom will never block, even if - # getrandom would. See http://bugs.python.org/issue26839 - self._getrandom = ctypes.CDLL(None, use_errno=True).syscall - self._getrandom.restype = ctypes.c_long - # The arguments are syscall number, void *buf, - # size_t buflen, unsigned int flags. - self._getrandom.argtypes = (ctypes.c_long, ctypes.c_void_p, - ctypes.c_size_t, ctypes.c_uint) - - def getrandom(self, length, random=False, nonblock=False): - flags = 0 - if random: - flags |= self.GRND_RANDOM - if nonblock: - flags |= self.GRND_NONBLOCK - buf = ctypes.ARRAY(ctypes.c_char, length)() - r = self._getrandom(self.SYS_getrandom, buf, len(buf), flags) - if r == -1: - err = ctypes.get_errno() - if err == errno.EINVAL: - raise Exception("getrandom: Invalid argument") - elif err == errno.EFAULT: - raise Exception("getrandom: Buffer is outside " - "accessible address space") - elif err == errno.EAGAIN: - raise GeneratorNotInitializedError() - elif err == errno.EINTR: - raise InterruptedError() - return buf[:r] - - def isInitialized(self): - # Read one byte from getrandom to determine whether the - # nonblocking pool is initialized. - try: - r = self.getrandom(1, nonblock=True) - if len(r) != 1: - raise Exception("No data returned from getrandom") - print("Nonblocking pool initialized") - return True - except GeneratorNotInitializedError: - return False - - def run(self): - """Move data from haveged to the kernel until the nonblocking pool is - initialized. - - """ - if self.isInitialized(): - return - - random_fd = os.open('/dev/random', os.O_RDWR) - # Start haveged and tell it to supply unlimited data on - # stdout, and print summary information. - p = subprocess.Popen(['/usr/sbin/haveged', '-f', '-', '-n', '0', - '-v', '1'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - while not self.isInitialized(): - # Read a chunk from haveged. - data = b'' - while len(data) < self.CHUNK_SIZE: - data += p.stdout.read(self.CHUNK_SIZE - len(data)) - # The data structure is: - # struct rand_pool_info { - # int entropy_count; - # int buf_size; - # __u32 buf[0]; - # }; - arg = struct.pack('iis', len(data) * 8, len(data), data) - print("Moving %s bytes" % len(data)) - fcntl.ioctl(random_fd, self.OP_RNDADDENTROPY, arg) - # Now that the generator is initialized, stop haveged and - # print the summary information. - p.send_signal(2) - p.stdout.read() - print(p.stderr.read().decode('utf-8')) - - -if __name__ == '__main__': - p = Pump() - p.run() diff --git a/dib-elements/nodepool-base/README.rst b/dib-elements/nodepool-base/README.rst deleted file mode 100644 index e5052b9..0000000 --- a/dib-elements/nodepool-base/README.rst +++ /dev/null @@ -1,24 +0,0 @@ -============= -nodepool-base -============= - -Tasks to deal with image metadata and other Nodepool cloud specific tweaks. - -Name resolution ---------------- - -The image should have the unbound DNS resolver package installed, the -``nodepool-base`` element then configures it to forward DNS queries -to: - - ``NODEPOOL_STATIC_NAMESERVER_V4``, default: ``1.0.0.1`` - ``NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK``, default: ``8.8.8.8``. - -If ``NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6`` is set to ``1`` then -the following two servers will be configured as forwarders too - - ``NODEPOOL_STATIC_NAMESERVER_V6``, default: ``2606:4700:4700::1111`` - ``NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK``, default: ``2001:4860:4860::8888`` - -Note externally setting either of these values implies -``NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6=1`` diff --git a/dib-elements/nodepool-base/cleanup.d/50-root b/dib-elements/nodepool-base/cleanup.d/50-root deleted file mode 100755 index 79ea352..0000000 --- a/dib-elements/nodepool-base/cleanup.d/50-root +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -sudo rm -rf $TARGET_ROOT/root/.cache diff --git a/dib-elements/nodepool-base/element-deps b/dib-elements/nodepool-base/element-deps deleted file mode 100644 index 4bc83d4..0000000 --- a/dib-elements/nodepool-base/element-deps +++ /dev/null @@ -1,3 +0,0 @@ -ensure-venv -package-installs -zuul-worker diff --git a/dib-elements/nodepool-base/environment.d/75-nodepool-base-env b/dib-elements/nodepool-base/environment.d/75-nodepool-base-env deleted file mode 100644 index 4403426..0000000 --- a/dib-elements/nodepool-base/environment.d/75-nodepool-base-env +++ /dev/null @@ -1,4 +0,0 @@ -export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2606:4700:4700::1111} -export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-1.0.0.1} -export NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888} -export NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8} diff --git a/dib-elements/nodepool-base/finalise.d/89-boot-settings b/dib-elements/nodepool-base/finalise.d/89-boot-settings deleted file mode 100755 index 0aa7469..0000000 --- a/dib-elements/nodepool-base/finalise.d/89-boot-settings +++ /dev/null @@ -1,254 +0,0 @@ -#!/bin/bash -# Copyright (C) 2011-2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=set setu setpipefail indent -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -# -# Note that in OpenStack infra, the configure-unbound role [1] that is -# part of the base jobs will reconfigure unbound based on the host's -# ipv6 support very early in the job setup. Thus the following -# forwarder setup is only relevant to the initial boot and some parts -# of the integration-tests before configure-unbound role is used. -# -# [1] https://opendev.org/opendev/base-jobs/src/branch/master/roles/configure-unbound -# - -NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-1.0.0.1} -NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8} - -# Explicitly setting a v6 nameserver implies you want ipv6 -if [[ -n ${NODEPOOL_STATIC_NAMESERVER_V6:-} || -n ${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK} ]]; then - NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6=1 -fi - -if [[ ${NODEPOOL_STATIC_NAMESERVER_POPULATE_IPV6:-0} == 1 ]]; then - NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2606:4700:4700::1111} - NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888} - - dd of=/tmp/forwarding.conf < /tmp/unbound-logging.conf - -if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|opensuse|openeuler|rocky) ]] ; then - UNBOUND_CONFD=/etc/unbound/conf.d -elif [[ "$DISTRO_NAME" =~ 'gentoo' ]] ; then - UNBOUND_CONFD=/etc/unbound/conf.d - mkdir -p $UNBOUND_CONFD - echo "include: \"$UNBOUND_CONFD/*.conf\"" >> /etc/unbound/unbound.conf -else - UNBOUND_CONFD=/etc/unbound/unbound.conf.d -fi - -mv /tmp/unbound-logging.conf $UNBOUND_CONFD -chown root:root $UNBOUND_CONFD/unbound-logging.conf -chmod a+r $UNBOUND_CONFD/unbound-logging.conf - -touch /var/log/unbound.log -chown unbound /var/log/unbound.log -chmod 0644 /var/log/unbound.log -# NOTE(ianw) 2022-05-22 : not 100% sure why but emperically unbound -# runs under named context. Generally it doesn't log here on rhel-ish -# distros, but for consistency it's much easier if we just have one -# log file in the same place across distros. Thus set the context to -# allow it. -if [[ -e /usr/sbin/semanage ]]; then - semanage fcontext -a -t named_log_t /var/log/unbound.log - restorecon -v /var/log/unbound.log -fi - -if [[ "$DISTRO_NAME" =~ (opensuse) ]] ; then - # NOTE(ianw) 2021-03-17 : something about building the suse chroot - # has changed and /etc/init.d/ is no longer available. However, - # it rc-local.service (part of systemd) still runs. We should - # probably convert this all to a systemd job; but for now... - mkdir -p /etc/init.d - rclocal=/etc/init.d/boot.local -elif [[ "${DISTRO_NAME}" =~ "gentoo" ]]; then - rclocal=/etc/local.d/unbound.start - mkdir -p /etc/local.d -else - # You'd think rc.local would be simple ... - # - # On Redhat systems, systemd's rc-local service looks for an - # executable /etc/rc.d/rc.local file to run. On Debian/Ubuntu, the - # eqivalent file is /etc/rc.local, which is missing on Debian stretch. - # - # Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local - # correctly. Fedora, however, does not come with an rc.local file at - # all. Thus if we have a rc.d directory, but no rc.local file, we - # need to create it (if you don't have an rc.d directory, and don't - # have /etc/rc.local, then it's not clear what platform you are on). - # - # Bug [1] is filed to bring Fedora in-line with Centos, and has more - # details on all this. As at 2016-10-18 is unresolved. - # - # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052 - if [[ ! -e /etc/rc.local ]] && [[ ! -e /etc/debian_version ]]; then - if [[ ! -d /etc/rc.d ]]; then - echo "No rc.local and no rc.d directory! See comments in 89-boot-settings" - exit 1 - fi - touch /etc/rc.d/rc.local - ln -sf /etc/rc.d/rc.local /etc/rc.local - # permissions added below. selinux context will be fixed up at - # end of build. - fi - rclocal=/etc/rc.local -fi - -# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using -# rc.local...) -cat >$rclocal < /etc/resolv.conf - -EOF - -# Configure NetworkManager to not manage /etc/resolv.conf -if [ -d /etc/NetworkManager/conf.d ]; then - cat <>$rclocal <>$rclocal <> /etc/unbound/unbound.conf - -# Disable dlv. Per the unbound.conf manpage this should not be used -# anymore but is in use by some of our distros. The problem here is it -# does lookasides for DNSSEC which increases the number of queries and -# introduces more points of lookup failure. Disable it to avoid these -# problems. -sed -i -e 's/dlv-anchor-file:/#dlv-anchor-file:/g' /etc/unbound/unbound.conf - -# Tripleo uses dhcp -dhcp_file='' -if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then - dhcp_file=/etc/dhcp/dhclient-eth0.conf -elif [[ "$DISTRO_NAME" =~ (debian|ubuntu|gentoo) ]] ; then - dhcp_file=/etc/dhcp/dhclient.conf -fi -if [ -n "$dhcp_file" ] ; then - cat > $dhcp_file <"; -request subnet-mask, broadcast-address, routers, - interface-mtu, rfc3442-classless-static-routes; -supersede domain-name-servers 127.0.0.1; -supersede domain-search ""; -supersede domain-name ""; -EOF -fi - -# On bionic and later, the install of the new systemd-resolved in the -# chroot will see that there is no /etc/resolv.conf and assume it is a -# blank system where it will be the nameserver provider. It thus -# creates /etc/resolv.conf as a link back to its compatability files. -# To configure systemd-resolved's resolvers you need to modify -# /etc/systemd/resolved.conf; which would be possible, but we'd prefer -# to be consistent across all our platforms. -# -# dib will copy whatever is in /etc/resolv.conf.ORIG to -# /etc/resolv.conf as one of the final steps in image creation. Thus -# we are hard-coding resolution to localhost (unbound) here. -# -# Note that with /etc/resolv.conf as a regular file, systemd-resolved -# will also obey it for nameserver info when it starts at boot. -rm -f /etc/resolv.conf.ORIG -echo "nameserver 127.0.0.1" > /etc/resolv.conf.ORIG - -case "$DIB_INIT_SYSTEM" in - upstart) - # nothing to do - ;; - systemd) - systemctl enable unbound.service - ;; - openrc) - rc-update add unbound default - ;; - sysv) - # nothing to do - ;; - *) - echo "Unsupported init system $DIB_INIT_SYSTEM" - exit 1 - ;; -esac diff --git a/dib-elements/nodepool-base/finalise.d/89-glean b/dib-elements/nodepool-base/finalise.d/89-glean deleted file mode 100755 index 312024c..0000000 --- a/dib-elements/nodepool-base/finalise.d/89-glean +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Copyright (C) 2011-2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=set setu setpipefail indent -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -case "$DIB_INIT_SYSTEM" in - systemd) - glean_path_dib="/etc/systemd/system/glean@.service.d" - mkdir -p $glean_path_dib - nodepool_base="$(dirname $0)/../glean@.service.d" - cp -RP $nodepool_base/override.conf $glean_path_dib/override.conf - ;; - *) - echo "Skipping glean systemd configuration" - ;; -esac diff --git a/dib-elements/nodepool-base/finalise.d/89-journald-persistent b/dib-elements/nodepool-base/finalise.d/89-journald-persistent deleted file mode 100755 index a2af142..0000000 --- a/dib-elements/nodepool-base/finalise.d/89-journald-persistent +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Copyright (C) 2017 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -# Journald default is to not persist logs to disk if /var/log/journal is -# not present. Update the configuration to set storage to persistent which -# will create /var/log/journal if necessary and store logs on disk. This -# avoids the situation where test runs can fill the journald ring buffer -# deleting older logs that may be important to the job. -if [ -f /etc/systemd/journald.conf ] ; then - sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf -fi diff --git a/dib-elements/nodepool-base/finalise.d/99-nodepool-dir b/dib-elements/nodepool-base/finalise.d/99-nodepool-dir deleted file mode 100755 index f5567d0..0000000 --- a/dib-elements/nodepool-base/finalise.d/99-nodepool-dir +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright (C) 2011-2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -# Nodepool expects this dir to exist when it boots slaves. -# Nodepool writes environment info to this dir. We set the mode -# to 0777 so that any user can access this env info. -mkdir /etc/nodepool -chmod 0777 /etc/nodepool diff --git a/dib-elements/nodepool-base/glean@.service.d/override.conf b/dib-elements/nodepool-base/glean@.service.d/override.conf deleted file mode 100644 index 202216c..0000000 --- a/dib-elements/nodepool-base/glean@.service.d/override.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -Environment="ARGS=--interface %I --skip-dns" diff --git a/dib-elements/nodepool-base/install.d/05-record-details b/dib-elements/nodepool-base/install.d/05-record-details deleted file mode 100755 index 65aa3e5..0000000 --- a/dib-elements/nodepool-base/install.d/05-record-details +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright (C) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -echo $DIB_IMAGE_NAME > /etc/image-hostname.txt diff --git a/dib-elements/nodepool-base/install.d/06-record-builddate b/dib-elements/nodepool-base/install.d/06-record-builddate deleted file mode 100755 index 9db66e7..0000000 --- a/dib-elements/nodepool-base/install.d/06-record-builddate +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# Put a timestamp in the image file of the date the image was built. -# This is echoed into the logs on each run for easy cross-reference - -date --utc "+%Y-%m-%d %H:%M" > /etc/dib-builddate.txt diff --git a/dib-elements/nodepool-base/install.d/20-iptables b/dib-elements/nodepool-base/install.d/20-iptables deleted file mode 100755 index ff60961..0000000 --- a/dib-elements/nodepool-base/install.d/20-iptables +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -if [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then - rules_dir=/etc/iptables - ipv4_rules=${rules_dir}/rules.v4 - ipv6_rules=${rules_dir}/rules.v6 -elif [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora|openeuler|rocky) ]] ; then - rules_dir=/etc/sysconfig - ipv4_rules=${rules_dir}/iptables - ipv6_rules=${rules_dir}/ip6tables -elif [[ "$DISTRO_NAME" =~ 'opensuse' ]] ; then - rules_dir=/etc/sysconfig - ipv4_rules=${rules_dir}/iptables - ipv6_rules=${rules_dir}/ip6tables -elif [[ "$DISTRO_NAME" =~ 'gentoo' ]] ; then - rules_dir=/var/lib/iptables # not needed, part of the package install - ipv4_rules=/var/lib/iptables/rules-save - ipv6_rules=/var/lib/ip6tables/rules-save -else - echo "Unsupported operating system $DISTRO_NAME" - exit 1 -fi - -mkdir -p $rules_dir - -cat > $ipv4_rules << EOF -*filter -:INPUT ACCEPT [0:0] -:FORWARD ACCEPT [0:0] -:OUTPUT ACCEPT [0:0] -:openstack-INPUT - [0:0] --A INPUT -j openstack-INPUT --A openstack-INPUT -i lo -j ACCEPT --A openstack-INPUT -p icmp --icmp-type any -j ACCEPT -#-A openstack-INPUT -p udp --dport 5353 -d 224.0.0.251 -j ACCEPT -# SSH from anywhere without -m state to avoid hanging connections on iptables-restore --A openstack-INPUT -m tcp -p tcp --dport 22 -j ACCEPT --A openstack-INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT -# Public TCP ports --A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT -# Ports 69 and 6385 allow to allow ironic VM nodes to reach tftp and -# the ironic API from the neutron public net --A openstack-INPUT -s 172.24.4.0/23 -p udp -m udp --dport 69 -j ACCEPT --A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 6385 -j ACCEPT -# Ports 80, 8000, 8003, 8004 from the devstack neutron public net to allow -# nova servers to reach heat-api-cfn, heat-api-cloudwatch, heat-api --A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 80 -j ACCEPT --A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8000 -j ACCEPT --A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8003 -j ACCEPT --A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8004 -j ACCEPT --A openstack-INPUT -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: " --A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited -COMMIT -EOF - -cat > $ipv6_rules << EOF -*filter -:INPUT ACCEPT [0:0] -:FORWARD ACCEPT [0:0] -:OUTPUT ACCEPT [0:0] -:openstack-INPUT - [0:0] --A INPUT -j openstack-INPUT --A openstack-INPUT -i lo -j ACCEPT --A openstack-INPUT -p ipv6-icmp -j ACCEPT -# SSH from anywhere without -m state to avoid hanging connections on iptables-restore --A openstack-INPUT -m tcp -p tcp --dport 22 -j ACCEPT --A openstack-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT -# Public TCP ports --A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT --A openstack-INPUT -j REJECT --reject-with icmp6-adm-prohibited -COMMIT -EOF diff --git a/dib-elements/nodepool-base/install.d/50-disable-metadata-cloudinit b/dib-elements/nodepool-base/install.d/50-disable-metadata-cloudinit deleted file mode 100755 index 376f2f4..0000000 --- a/dib-elements/nodepool-base/install.d/50-disable-metadata-cloudinit +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. - -# dib-lint: disable=setu setpipefail -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -e - -# Make all cloud-init data sources match rackspace- only attempt to look -# at ConfigDrive, not at metadata service. This is not needed if there -# is no cloud-init -if [ -d /etc/cloud/cloud.cfg.d ] ; then - dd of=/etc/cloud/cloud.cfg.d/95_real_datasources.cfg < /etc/sysctl.d/99-cloudimg-ipv6.conf < $PRESEED <> $TMP_HOOKS_PATH/zuul-user-ssh-public-key -elif [ -n "$ZUUL_USER_SSH_PUBLIC_KEY" ]; then - # save the specific public key inside the chroot from env derectly - echo "$ZUUL_USER_SSH_PUBLIC_KEY" >> $TMP_HOOKS_PATH/zuul-user-ssh-public-key -else - die "Can not find public key for zuul user!" -fi diff --git a/dib-elements/zuul-worker/install.d/60-zuul-worker b/dib-elements/zuul-worker/install.d/60-zuul-worker deleted file mode 100755 index 7138bc1..0000000 --- a/dib-elements/zuul-worker/install.d/60-zuul-worker +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# Add zuul user and group. Note we don't want to rely on -# "useradd"'s group adding behaviour, because it might differ across -# distros. -groupadd zuul -useradd -m zuul -g zuul -s /bin/bash - -cat > /etc/sudoers.d/zuul << EOF -zuul ALL=(ALL) NOPASSWD:ALL -EOF -chmod 0440 /etc/sudoers.d/zuul - -visudo -c || die "Error setting zuul sudo!" - -# this was copied from outside the chroot by extras.d -_pub_key=/tmp/in_target.d/zuul-user-ssh-public-key -if [ ! -f $_pub_key ]; then - die "Can not find Zuul public key!" -fi - -mkdir -p /home/zuul/.ssh -chmod 700 /home/zuul/.ssh -cp $_pub_key /home/zuul/.ssh/authorized_keys - -# cleanup everything to the right owner -chown -R zuul:zuul /home/zuul - -# a lot of caching happens in extra-data.d (for "historical" reasons). -# We've put the cache stuff into /opt/cache/files, but again, for -# "historical" reasons, ensure this is available in /home/zuul -# -# We do this for zuul as relative paths to the current user's homedir -# are used in places like devstack. -# -# Check if the cache exists as we don't have a strict dependency on the -# devstack-cache element. This allows you to build an image without -# incurring the cost of caching all the things. -if [ -d /opt/cache/files ] ; then - mkdir -p /home/zuul/cache - chown zuul:zuul /home/zuul/cache - ln -sf /opt/cache/files /home/zuul/cache/files - # but make sure the cache is readable by everyone - chmod -R a+rX /opt/cache/files/* -fi - -# New versions of git don't let you clone repos as a different user -# than the user owning the repo by default for security reasons. -# As above we cache git repos during extra-data.d in /opt/git/ and they -# end up owned by root. Chown them to zuul here to avoid permissions -# issues with the most likely user to interact with the git cache( zuul). -if [ -d /opt/git ] ; then - chown -R zuul:zuul /opt/git -fi diff --git a/playbooks/opendev-build-diskimage-base/post-inner.yaml b/playbooks/opendev-build-diskimage-base/post-inner.yaml deleted file mode 100644 index 52a4929..0000000 --- a/playbooks/opendev-build-diskimage-base/post-inner.yaml +++ /dev/null @@ -1,27 +0,0 @@ -- name: Compress image - when: "upload_image_format in ['raw', 'vhd']" - command: zstd '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_format }}' -- name: Set extension - when: "upload_image_format in ['raw', 'vhd']" - set_fact: - upload_image_extension: '{{ upload_image_format }}.zst' -- name: Set extension - when: "upload_image_format not in ['raw', 'vhd']" - set_fact: - upload_image_extension: '{{ upload_image_format }}' -- name: Upload image - no_log: true - include_role: - name: image-upload-swift - vars: - cloud: - auth_type: 'v3applicationcredential' - auth: - auth_url: 'https://keystone.api.sjc3.rackspacecloud.com/v3' - application_credential_id: '{{ image_upload_secret.application_credential_id }}' - application_credential_secret: '{{ image_upload_secret.application_credential_secret }}' - user_domain_name: rackspace_cloud_domain - container: images-1f49951f5beb - filename: '{{ build_diskimage_image_root }}/{{ build_diskimage_image_name }}.{{ upload_image_extension }}' - name: '{{ zuul.build }}-{{ build_diskimage_image_name }}.{{ upload_image_extension }}' - delete_after: 259200 diff --git a/playbooks/opendev-build-diskimage-base/post.yaml b/playbooks/opendev-build-diskimage-base/post.yaml deleted file mode 100644 index 3c78c2c..0000000 --- a/playbooks/opendev-build-diskimage-base/post.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- hosts: all - tasks: - - name: Upload image - when: image_upload_secret is defined and zuul_success - with_items: '{{ build_diskimage_formats }}' - loop_control: - loop_var: upload_image_format - include_tasks: post-inner.yaml diff --git a/playbooks/opendev-build-diskimage-base/pre.yaml b/playbooks/opendev-build-diskimage-base/pre.yaml deleted file mode 100644 index 41e32f8..0000000 --- a/playbooks/opendev-build-diskimage-base/pre.yaml +++ /dev/null @@ -1,38 +0,0 @@ -- hosts: all - pre_tasks: - - name: Install custom element requirements - become: true - package: - state: present - name: - - python3-yaml - # The following are required for the image upload - - python3-openstacksdk - - python3-oslo.utils - - zstd - - name: Setup dib directories - become: true - shell: | - set -x - - mkdir -p /opt/dib_cache - chown {{ ansible_user }} /opt/dib_cache - chgrp {{ ansible_user }} /opt/dib_cache - - mkdir -p /opt/dib_tmp - - if [ -b /dev/vdb ]; then - DEV='/dev/vdb' - elif [ -b /dev/xvde ]; then - DEV='/dev/xvde' - fi - - if [ -n "$DEV" ]; then - mkfs.ext4 $DEV - mount $DEV /opt/dib_tmp - fi - - chown {{ ansible_user }} /opt/dib_tmp - chgrp {{ ansible_user }} /opt/dib_tmp - roles: - - make-source-repositories-cache diff --git a/roles/image-upload-swift/library/__init__.py b/roles/image-upload-swift/library/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/roles/image-upload-swift/library/image_upload_swift.py b/roles/image-upload-swift/library/image_upload_swift.py deleted file mode 100644 index 44081b0..0000000 --- a/roles/image-upload-swift/library/image_upload_swift.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2014 Rackspace Australia -# Copyright 2018 Red Hat, Inc -# Copyright 2024 Acme Gating, LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import argparse -import concurrent.futures -import datetime -import logging -import os -import sys -import traceback - -import openstack -import requests.exceptions -import keystoneauth1.exceptions - -from ansible.module_utils.basic import AnsibleModule - -SEGMENT_SIZE = 500000000 # 500MB - - -def get_cloud(cloud): - if isinstance(cloud, dict): - config = openstack.config.loader.OpenStackConfig().get_one(**cloud) - return openstack.connection.Connection( - config=config, - pool_executor=concurrent.futures.ThreadPoolExecutor( - max_workers=10 - )) - else: - return openstack.connect(cloud=cloud) - - -def _add_etag_to_manifest(self, *args, **kw): - return - - -def prune(cloud, container, delete_after): - # In case the automatic expiration doesn't work, manually prune old uploads - if not delete_after: - return - target = (datetime.datetime.now(datetime.UTC) - - datetime.timedelta(seconds=delete_after)) - endpoint = cloud.object_store.get_endpoint() - url = os.path.join(endpoint, container) - for obj in cloud.object_store.objects(container): - ts = datetime.datetime.fromisoformat(obj['last_modified']) - ts = ts.replace(tzinfo=datetime.UTC) - if ts < target: - path = os.path.join(url, obj.name) - try: - cloud.session.delete(path) - except keystoneauth1.exceptions.http.NotFound: - pass - - -def run(cloud, container, filename, name, delete_after=None): - # Monkey-patch sdk so that the SLO upload does not add the etag; - # this works around an issue with rackspace-flex. - cloud.object_store._add_etag_to_manifest = _add_etag_to_manifest - prune(cloud, container, delete_after) - headers = {} - if delete_after: - headers['X-Delete-After'] = str(delete_after) - endpoint = cloud.object_store.get_endpoint() - cloud.object_store.create_object( - container, - name=name, - filename=filename, - segment_size=SEGMENT_SIZE, - **headers) - url = os.path.join(endpoint, container, name) - return url - - -def ansible_main(): - module = AnsibleModule( - argument_spec=dict( - cloud=dict(required=True, type='raw'), - container=dict(required=True, type='str'), - filename=dict(required=True, type='path'), - name=dict(required=True, type='str'), - delete_after=dict(type='int'), - ) - ) - - p = module.params - cloud = get_cloud(p.get('cloud')) - try: - url = run( - cloud, - p.get('container'), - p.get('filename'), - p.get('name'), - delete_after=p.get('delete_after'), - ) - except (keystoneauth1.exceptions.http.HttpError, - requests.exceptions.RequestException): - s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name) - s += "\n" + traceback.format_exc() - module.fail_json( - changed=False, - msg=s, - cloud=cloud.name, - region_name=cloud.config.region_name) - module.exit_json( - changed=True, - url=url, - ) - - -def cli_main(): - parser = argparse.ArgumentParser( - description="Upload image to swift" - ) - parser.add_argument('--verbose', action='store_true', - help='show debug information') - parser.add_argument('cloud', - help='Name of the cloud to use when uploading') - parser.add_argument('container', - help='Name of the container to use when uploading') - parser.add_argument('filename', - help='the file to upload') - parser.add_argument('name', - help='the object name') - parser.add_argument('--delete-after', - help='Number of seconds to delete object after ' - 'upload. Default is 3 days (259200 seconds) ' - 'and if set to 0 X-Delete-After will not be set', - type=int) - - args = parser.parse_args() - - if args.verbose: - logging.basicConfig(level=logging.DEBUG) - # Set requests log level accordingly - logging.getLogger("requests").setLevel(logging.DEBUG) - logging.getLogger("keystoneauth").setLevel(logging.INFO) - logging.getLogger("stevedore").setLevel(logging.INFO) - logging.captureWarnings(True) - - url = run( - get_cloud(args.cloud), - args.container, - args.filename, - args.name, - delete_after=args.delete_after, - ) - print(url) - - -if __name__ == '__main__': - if not sys.stdin.isatty(): - ansible_main() - else: - cli_main() diff --git a/roles/image-upload-swift/tasks/main.yaml b/roles/image-upload-swift/tasks/main.yaml deleted file mode 100644 index c7fe4e4..0000000 --- a/roles/image-upload-swift/tasks/main.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Run the checksums in the background while we're uploading -- name: Get sha256 hash - stat: - path: '{{ filename }}' - checksum_algorithm: sha256 - async: 600 - poll: 0 - register: sha256_task - -- name: Get md5 hash - stat: - path: '{{ filename }}' - checksum_algorithm: md5 - async: 600 - poll: 0 - register: md5_task - -- name: Upload image to swift - no_log: true - image_upload_swift: - cloud: '{{ cloud }}' - container: '{{ container }}' - filename: '{{ filename }}' - name: '{{ name }}' - delete_after: '{{ delete_after }}' - register: upload_results - -- name: Wait for sha256 - async_status: - jid: "{{ sha256_task.ansible_job_id }}" - register: sha256 - until: sha256.finished - retries: 1 - delay: 10 - -- name: Wait for md5 - async_status: - jid: "{{ md5_task.ansible_job_id }}" - register: md5 - until: md5.finished - retries: 1 - delay: 10 - -- name: Return artifact to Zuul - zuul_return: - data: - zuul: - artifacts: - - name: '{{ upload_image_format }} image' - url: '{{ upload_results.url }}' - metadata: - type: 'zuul_image' - image_name: '{{ build_diskimage_image_name }}' - format: '{{ upload_image_format }}' - sha256: '{{ sha256.stat.checksum }}' - md5sum: '{{ md5.stat.checksum }}' diff --git a/roles/make-source-repositories-cache/README.rst b/roles/make-source-repositories-cache/README.rst deleted file mode 100644 index 73e3378..0000000 --- a/roles/make-source-repositories-cache/README.rst +++ /dev/null @@ -1,12 +0,0 @@ -Reformat the git repo cache for source-respositories - -This reverses the process of creating the /opt/git cache on worker -nodes. It turns that cache into the format used by the -source-repositories diskimage builder element. - -**Role Variables** - -.. zuul:rolevar:: cached_repos_root - :default: /opt/git - - The root of the cached repos. diff --git a/roles/make-source-repositories-cache/defaults/main.yaml b/roles/make-source-repositories-cache/defaults/main.yaml deleted file mode 100644 index 4c8a06d..0000000 --- a/roles/make-source-repositories-cache/defaults/main.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# Matches prepare_workspace_root -cached_repos_root: /opt/git diff --git a/roles/make-source-repositories-cache/tasks/main.yaml b/roles/make-source-repositories-cache/tasks/main.yaml deleted file mode 100644 index 4bf265f..0000000 --- a/roles/make-source-repositories-cache/tasks/main.yaml +++ /dev/null @@ -1,27 +0,0 @@ -- name: Reformat the git repo cache for source-repositories - environment: "{{ build_diskimage_environment }}" - args: - executable: /bin/bash - shell: | - GIT_CACHE_ROOT="{{ cached_repos_root }}" - - REPOTYPE=git - CACHE_BASE=${DIB_IMAGE_CACHE}/source-repositories - mkdir -p ${CACHE_BASE} - - # Note: maxdepth 4 is based on the current opendev practice of - # not having repos deeper than two directory levels. This is - # fragile and should probably be replaced with something better, - # but as written it saves considerable time. - for NODE_CACHE_PATH in $(find $GIT_CACHE_ROOT -maxdepth 4 -type d -name .git); do - REPOPATH=$(dirname $NODE_CACHE_PATH) - REPONAME=$(basename $REPOPATH) - REPOLOCATION="$(echo $REPOPATH | sed "s,$GIT_CACHE_ROOT/,https://,").git" - - # From diskimage-builder element source-repositories - CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' ) - CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME} - CACHE_PATH=${CACHE_BASE}/${CACHE_NAME} - - mv ${REPOPATH} ${CACHE_PATH} - done