Make the LXC cache prep use the host package source config

Currently the container cache preparation process uses a pre-prepared LXC base image which includes its own package repository configuration. This presents a few problems: - The first packages installed will make use of the base image's package repo configuration, resulting in a bypass of local mirrors to install the first set of packages. - A set of vars need to be set in order to have the containers use a local mirror, otherwise it'll use the mirrors set in the role's vars files. This is counterintuitive. Another problem introduced by I95c210c83ca968d11ba6f6a36b634bb798fa291f as a result of the package repository vars moving from the role defaults to the vars files is that the precedence has changed. The change in precedence means that a task which sets a fact can't be used to override the defaults set in the vars file. This method is used in all the role tests to ensure that the OpenStack-CI repositories can be discovered from the host and then used. This patch changes the image cache preparation process to ensure that the container package repository configuration matches the host configuration. This is simpler and more intuitive. Additionally the copy task from the deployment host into the container cache is set to assume the same destination in the container as the source (to reduce configuration verbosity), appropriately sets the leading '0' for the mode (to prevent unexpected surprises), and appropriately quotes the variable (to ensure forward compatibility with Ansible 2.0). Finally, the use of lxc_container_caches in the test configuration has been removed as it is no longer used. Change-Id: I420382fd3bbbb5fcae90ae0c6160233202a1a51a
2016-05-11 15:43:45 +01:00 · 2016-05-11 15:43:45 +01:00 · aca3d5e01d
commit aca3d5e01d
parent b93b84c59c
7 changed files with 108 additions and 167 deletions
--- a/releasenotes/notes/container-repo-host-match-2be99b14642e0591.yaml
+++ b/releasenotes/notes/container-repo-host-match-2be99b14642e0591.yaml
@ -0,0 +1,12 @@
+---
+upgrade:
+  - |
+    The LXC container cache preparation process now copies package
+    repository configuration from the host instead of implementing
+    its own configuration. The following variables are therefore
+    unnecessary and have been removed:
+
+    * ``lxc_container_template_main_apt_repo``
+    * ``lxc_container_template_security_apt_repo``
+    * ``lxc_container_template_apt_components``
+
--- a/tasks/lxc_cache_preparation.yml
+++ b/tasks/lxc_cache_preparation.yml
@ -13,14 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+- name: Copy files from the LXC host to the container cache
+  local_action:
+    module: copy
+    src: "{{ item }}"
+    dest: "/var/lib/lxc/LXC_NAME/rootfs{{ item }}"
+  with_items: "{{ lxc_cache_map.copy_from_host }}"
+  tags:
+    - lxc-cache
+
 - name: Copy files from deployment host to the container cache
  copy:
    src: "{{ item.src }}"
-    dest: "/var/lib/lxc/LXC_NAME/rootfs{{ item.dest }}"
+    dest: "/var/lib/lxc/LXC_NAME/rootfs{{ item.dest | default(item.src) }}"
    owner: "{{ item.owner | default('root') }}"
    group: "{{ item.group | default('root') }}"
-    mode: "{{ item.mode | default('644') }}"
-  with_items: lxc_container_cache_files
+    mode: "{{ item.mode | default('0644') }}"
+  with_items: "{{ lxc_container_cache_files }}"
  tags:
    - lxc-cache
    - lxc-cache-copy-files
@ -30,7 +39,7 @@
    content: |
      #!/usr/bin/env bash
      set -x
-      {{ lxc_cache_map.cache_base_commands }}
+      {{ lxc_cache_map.cache_prep_commands }}
    dest: "/var/lib/lxc/LXC_NAME/rootfs/usr/local/bin/cache-prep-commands.sh"
    mode: "0755"
  tags:
@ -45,51 +54,6 @@
    - lxc-cache
    - lxc-cache-update

- name: Create repos in the cached container
-  copy:
-    content: "{{ item.value }}"
-    dest: "/var/lib/lxc/LXC_NAME/rootfs{{ item.key }}"
-  with_dict: lxc_cache_map.repos
-  tags:
-    - lxc-cache
-    - lxc-cache-update
-
- name: Copy cached image preparation package script into the target
-  copy:
-    content: |
-      #!/usr/bin/env bash
-      set -x
-      {{ lxc_cache_install_command }} {{ lxc_cache_map.cache_packages | join(' ') }}
-    dest: "/var/lib/lxc/LXC_NAME/rootfs/usr/local/bin/cache-package-prep-commands.sh"
-    mode: "0755"
-  tags:
-    - lxc-cache
-    - lxc-cache-update
-
- name: Prepare cached image with packages
-  command: "chroot /var/lib/lxc/LXC_NAME/rootfs /usr/local/bin/cache-package-prep-commands.sh"
-  tags:
-    - lxc-cache
-    - lxc-cache-update
-
- name: Cached image post-preparation package script
-  copy:
-    content: |
-      #!/usr/bin/env bash
-      set -x
-      {{ lxc_cache_map.cache_post_commands }}
-    dest: "/var/lib/lxc/LXC_NAME/rootfs/usr/local/bin/cache-post-prep-commands.sh"
-    mode: "0755"
-  tags:
-    - lxc-cache
-    - lxc-cache-update
-
- name: Post-prepare cached image setup commands
-  command: "chroot /var/lib/lxc/LXC_NAME/rootfs /usr/local/bin/cache-post-prep-commands.sh"
-  tags:
-    - lxc-cache
-    - lxc-cache-update
-
 - name: Adjust sshd configuration in container
  lineinfile:
    dest: "/var/lib/lxc/LXC_NAME/rootfs/etc/ssh/sshd_config"
--- a/tests/test.yml
+++ b/tests/test.yml
@ -42,11 +42,6 @@
      lxc_net_bridge: lxcbr0
      lxc_kernel_options:
        - { key: 'fs.inotify.max_user_instances', value: 1024 }
-      lxc_container_caches:
-        - url: "https://rpc-repo.rackspace.com/container_images/rpc-trusty-container.tgz"
-          name: "trusty.tgz"
-          sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c"
-          chroot_path: trusty/rootfs-amd64
      lxc_container_cache_files:
        - src: files/container-file-copy-test.txt
          dest: /tmp/file-copied-from-deployment-host.txt
--- a/tox.ini
+++ b/tox.ini
@ -125,6 +125,7 @@ commands =
                   --force
    ansible-playbook -i {toxinidir}/tests/inventory \
                     -e "rolename={toxinidir}" \
+                     -vv \
                     {toxinidir}/tests/test.yml


--- a/vars/redhat-7.yml
+++ b/vars/redhat-7.yml
@ -41,25 +41,29 @@ lxc_cache_map:
  distro: centos
  arch: amd64
  release: 7
-  cache_packages:
-    - openssh-server
-    - sudo
-  repos: {}
-  cache_base_commands: |
+  copy_from_host:
+    - /etc/yum.repos.d/
+    - /etc/pki/rpm-gpg/
+  cache_prep_commands: |
    rm /etc/resolv.conf
    {% for resolver in lxc_cache_resolvers %}
    echo "{{ resolver }}" | tee -a /etc/resolv.conf
    {% endfor %}
-    {{ lxc_cache_install_command }} wget python2
+    {{ lxc_cache_install_command }} {{ lxc_cache_packages | join(' ') }}
    rm -f /usr/bin/python
    ln -s /usr/bin/python2.7 /usr/bin/python
    /usr/bin/wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -O /tmp/epel-release-latest-7.noarch.rpm
    /usr/bin/rpm -ivh /tmp/epel-release-latest-7.noarch.rpm || true
-  cache_post_commands: |
    yum clean all

 lxc_cache_install_command: "yum install -y"

+lxc_cache_packages:
+  - openssh-server
+  - python2
+  - sudo
+  - wget
+
 pip_install_options: >
  --global-option=build_ext
  --global-option="-L/opt/lxc_embedded/x86_64-linux-gnu/"
--- a/vars/ubuntu-14.04.yml
+++ b/vars/ubuntu-14.04.yml
@ -16,23 +16,6 @@
 ## APT Cache Options
 cache_timeout: 600

-# Repo mappings for each architecture
-lxc_main_apt_repo_mapping:
-  x86_64: "https://mirror.rackspace.com/ubuntu"
-  ppc64le: "http://ports.ubuntu.com/ubuntu-ports/"
-
-lxc_security_apt_repo_mapping:
-  x86_64: "{{ lxc_container_template_main_apt_repo }}"
-  ppc64le: "{{ lxc_container_template_main_apt_repo }}"
-
-
-# Container repos
-lxc_container_template_main_apt_repo: "{{ lxc_main_apt_repo_mapping.get( ansible_architecture ) }}"
-lxc_container_template_security_apt_repo: "{{ lxc_security_apt_repo_mapping.get( ansible_architecture ) }}"
-lxc_container_template_apt_components:
-  - main
-  - universe
-
 # Required apt packages.
 lxc_packages:
  - apparmor
@ -57,63 +40,11 @@ lxc_cache_map:
  distro: ubuntu
  arch: "{{ lxc_architecture_mapping.get( ansible_architecture ) }}"
  release: trusty
-  cache_packages:
-    - bridge-utils
-    - bsdmainutils
-    - build-essential
-    - cgmanager
-    - cgmanager-utils
-    - cgroup-lite
-    - comerr-dev
-    - curl
-    - debconf-utils
-    - debhelper
-    - dh-apparmor
-    - gettext
-    - gir1.2-glib-2.0
-    - git
-    - groff-base
-    - intltool-debian
-    - iptables
-    - iso-codes
-    - krb5-multidev
-    - libcgmanager-dev
-    - libdbus-1-dev
-    - libdbus-glib-1-2
-    - libdevmapper-event1.02.1
-    - libexpat1-dev
-    - libffi-dev
-    - libpq-dev
-    - libpq5
-    - libpython-dev
-    - libxml2-dev
-    - libxslt1-dev
-    - libxslt1.1
-    - lvm2
-    - openssh-server
-    - python-apt
-    - python-apt-common
-    - python-dev
-    - python-pycurl
-    - python-software-properties
-    - python3-apt
-    - python3-dbus
-    - python3-gi
-    - python3-minimal
-    - python3-pycurl
-    - python3-software-properties
-    - software-properties-common
-    - sqlite3
-    - ssh
-    - sshpass
-    - time
-  repos:
-    "/etc/apt/sources.list": |
-      deb {{ lxc_container_template_main_apt_repo }} trusty {{ lxc_container_template_apt_components | join(' ') }}
-      deb {{ lxc_container_template_main_apt_repo }} trusty-updates {{ lxc_container_template_apt_components | join(' ') }}
-      deb {{ lxc_container_template_main_apt_repo }} trusty-backports {{ lxc_container_template_apt_components | join(' ') }}
-      deb {{ lxc_container_template_security_apt_repo }} trusty-security {{ lxc_container_template_apt_components | join(' ') }}
-  cache_base_commands: |
+  copy_from_host:
+    - /etc/apt/sources.list
+    - /etc/apt/sources.list.d/
+    - /etc/apt/apt.conf.d/
+  cache_prep_commands: |
    rm /run/resolvconf/resolv.conf
    # This enforces the resolvers from within the image
    {% for resolver in lxc_cache_resolvers %}
@ -123,20 +54,70 @@ lxc_cache_map:
    echo "# NONE" > /etc/resolvconf/resolv.conf.d/tail
    echo "# NONE" > /etc/resolvconf/resolv.conf.d/original
    rm -rf /var/lib/apt/lists/*
-    # This is done because its possible the base repos being installed will use https
-    #  and ansible needs access to python2.7 which the system may not have by default
-    {{ lxc_cache_install_command }} apt-transport-https python2.7
+    {{ lxc_cache_install_command }} {{ lxc_cache_packages | join(' ') }}
    apt-get -y upgrade
    rm -f /usr/bin/python
    ln -s /usr/bin/python2.7 /usr/bin/python
    mkdir -p /root/.ssh
    chmod 700 /root/.ssh
    userdel --force --remove ubuntu || true
-  cache_post_commands: |
    apt-get clean

 lxc_cache_install_command: "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y"

+lxc_cache_packages:
+  - apt-transport-https
+  - bridge-utils
+  - bsdmainutils
+  - build-essential
+  - cgmanager
+  - cgmanager-utils
+  - cgroup-lite
+  - comerr-dev
+  - curl
+  - debconf-utils
+  - debhelper
+  - dh-apparmor
+  - gettext
+  - gir1.2-glib-2.0
+  - git
+  - groff-base
+  - intltool-debian
+  - iptables
+  - iso-codes
+  - krb5-multidev
+  - libcgmanager-dev
+  - libdbus-1-dev
+  - libdbus-glib-1-2
+  - libdevmapper-event1.02.1
+  - libexpat1-dev
+  - libffi-dev
+  - libpq-dev
+  - libpq5
+  - libpython-dev
+  - libxml2-dev
+  - libxslt1-dev
+  - libxslt1.1
+  - lvm2
+  - openssh-server
+  - python2.7
+  - python-apt
+  - python-apt-common
+  - python-dev
+  - python-pycurl
+  - python-software-properties
+  - python3-apt
+  - python3-dbus
+  - python3-gi
+  - python3-minimal
+  - python3-pycurl
+  - python3-software-properties
+  - software-properties-common
+  - sqlite3
+  - ssh
+  - sshpass
+  - time
+
 lxc_cached_network_interfaces:
  - src: "lxc-net-bridge.cfg.j2"
    dest: "/etc/network/interfaces.d/lxc-net-bridge.cfg"
--- a/vars/ubuntu-16.04.yml
+++ b/vars/ubuntu-16.04.yml
@ -16,23 +16,6 @@
 ## APT Cache Options
 cache_timeout: 600

-# Repo mappings for each architecture
-lxc_main_apt_repo_mapping:
-  x86_64: "https://mirror.rackspace.com/ubuntu"
-  ppc64le: "http://ports.ubuntu.com/ubuntu-ports/"
-
-lxc_security_apt_repo_mapping:
-  x86_64: "{{ lxc_container_template_main_apt_repo }}"
-  ppc64le: "{{ lxc_container_template_main_apt_repo }}"
-
-
-# Container repos
-lxc_container_template_main_apt_repo: "{{ lxc_main_apt_repo_mapping.get( ansible_architecture ) }}"
-lxc_container_template_security_apt_repo: "{{ lxc_security_apt_repo_mapping.get( ansible_architecture ) }}"
-lxc_container_template_apt_components:
-  - main
-  - universe
-
 # Required apt packages.
 lxc_packages:
  - apparmor
@ -58,14 +41,13 @@ lxc_cache_map:
  arch: "{{ lxc_architecture_mapping.get( ansible_architecture ) }}"
  release: xenial
  cache_packages:
+    - apt-transport-https
    - openssh-server
-  repos:
-    "/etc/apt/sources.list": |
-      deb {{ lxc_container_template_main_apt_repo }} xenial {{ lxc_container_template_apt_components | join(' ') }}
-      deb {{ lxc_container_template_main_apt_repo }} xenial-updates {{ lxc_container_template_apt_components | join(' ') }}
-      deb {{ lxc_container_template_main_apt_repo }} xenial-backports {{ lxc_container_template_apt_components | join(' ') }}
-      deb {{ lxc_container_template_security_apt_repo }} xenial-security {{ lxc_container_template_apt_components | join(' ') }}
-  cache_base_commands: |
+  copy_from_host:
+    - /etc/apt/sources.list
+    - /etc/apt/sources.list.d/
+    - /etc/apt/apt.conf.d/
+  cache_prep_commands: |
    rm /run/resolvconf/resolv.conf
    # This enforces the resolvers from within the image
    {% for resolver in lxc_cache_resolvers %}
@ -75,20 +57,22 @@ lxc_cache_map:
    echo "# NONE" > /etc/resolvconf/resolv.conf.d/tail
    echo "# NONE" > /etc/resolvconf/resolv.conf.d/original
    rm -rf /var/lib/apt/lists/*
-    # This is done because its possible the base repos being installed will use https
-    #  and ansible needs access to python2.7 which the system may not have by default
-    {{ lxc_cache_install_command }} apt-transport-https python2.7
+    {{ lxc_cache_install_command }} {{ lxc_cache_packages | join(' ') }}
    apt-get -y upgrade
    rm -f /usr/bin/python
    ln -s /usr/bin/python2.7 /usr/bin/python
    mkdir -p /root/.ssh
    chmod 700 /root/.ssh
    userdel --force --remove ubuntu || true
-  cache_post_commands: |
    apt-get clean

 lxc_cache_install_command: "export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y"

+lxc_cache_packages:
+  - apt-transport-https
+  - openssh-server
+  - python2.7
+
 lxc_cached_network_interfaces:
  - src: "lxc-net-bridge.cfg.j2"
    dest: "/etc/network/interfaces.d/lxc-net-bridge.cfg"