Change: filelists are now rsync include statements

Update rq/default.yaml accordingly
Fix scp not working when key is not in known_hosts

Change-Id: Iddf645380bfc8c251021e22c0f4630dee72cb0df

rq/default.yaml

@@ -1,31 +1,47 @@
-files:
+filelists:
-  __default: ['/etc/resolv.conf', '/etc/mcollective', '/etc/astute.yaml', '/root/anaconda*', '/root/*.log', '/root/*.ks', '/var/lib/puppet/state/last_run_summary.yaml', '/var/run/pcap_dir', '/var/lib/cloud', '/etc/default']
+  __default:
+    - 'etc-astute.yaml'
+    - 'etc-default'
+    - 'etc-mcollective'
+    - 'etc-resolv.conf'
+    - 'root-anaconda'
+    - 'root-.ks'
+    - 'root-.log'
+    - 'var-lib-cloud'
+    - 'var-lib-puppet-state-last_run_summary.yaml'
+    - 'var-run-pcap_dir'
   by_roles:
-    controller: ['/etc/apache2', '/etc/keystone', '/etc/swift', '/etc/ceph', '/etc/cinder', '/etc/glance', '/etc/haproxy', '/etc/heat', '/etc/mysql', '/etc/nova']
+    fuel:
+      - 'etc-astute'
+      - 'etc-centos-release'
+      - 'etc-cobbler'
+      - 'etc-dnsmasq.conf'
+      - 'etc-fuel'
+      - 'etc-fuel_build_'
+      - 'etc-nailgun'
+    controller:
+      - 'etc-apache2'
+      - 'etc-ceph'
+      - 'etc-cinder'
+      - 'etc-glance'
+      - 'etc-haproxy'
+      - 'etc-heat'
+      - 'etc-keystone'
+      - 'etc-mysql'
+      - 'etc-nova'
+      - 'etc-swift'
+      - 'root-ceph'
     cinder: ['/etc/cinder']
-    fuel: ['/etc/astute', '/etc/dnsmasq.conf', '/etc/centos-release', '/etc/fuel_build_number', '/etc/fuel_build_id', '/etc/cobbler', '/etc/cobbler.dnsmasq.conf', '/root/*.log', '/etc/fuel', '/etc/nailgun']
+    ceph-osd: ['root-ceph', 'etc-ceph']
-    ceph: ['/root/.ceph*', '/etc/ceph']
+    compute: ['etc-libvirt', 'etc-nova']
-    compute: ['/etc/libvirt', '/etc/nova']
+    no_fuel: ['etc-hiera', 'etc-neutron']
-    no_fuel: ['/etc/hiera', '/etc/neutron']
+    virt: ['etc-libvirt']
-    virt: ['/etc/libvirt']
   by_os_platform:
-    ubuntu: ['/etc/apt', '/etc/lsb-release', '/etc/network']
+    ubuntu: ['etc-apt', 'etc-lsb-release', 'etc-network']
-    centos: ['/etc/redhat-release', '/etc/sysconfig', '/etc/yum', '/etc/yum.d']
+    centos: ['etc-redhat-release', 'etc-sysconfig', 'etc-yum']
   by_release:
     '9.0': ['/etc/fuel']
     '9.1': ['/etc/fuel']
-# filelists:
-#   by_roles:
-#     fuel: [etc-nailgun, etc-fuel]
-#     ceph-osd: [etc-ceph]
-#     cinder: [etc-cinder]
-#     compute: [etc-nova, etc-libvirt]
-#     controller: [etc-glance, etc-haproxy, etc-nova,
-#       etc-keystone, etc-neutron, etc-mysql, etc-ceph, etc-heat]
-#     # f3flight: need to make a better way to decide which files to collect
-#   by_os_platform:
-#     centos: [etc-yum]
-#     ubuntu: [etc-apt]
 scripts:
   by_release:
     '4.1':

rq/filelists/etc-apache2

@@ -0,0 +1,7 @@
++ /etc
++ /etc/apache2
+- /etc/apache2/ssl
++ /etc/apache2/**
++ /etc/httpd
+- /etc/httpd/ssl
++ /etc/httpd/**

rq/filelists/etc-apt

@@ -1 +1,3 @@
-/etc/apt
++ /etc
++ /etc/apt
++ /etc/apt/**

rq/filelists/etc-astute

@@ -0,0 +1,3 @@
++ /etc
++ /etc/astute
++ /etc/astute/**

rq/filelists/etc-astute.yaml

@@ -0,0 +1,2 @@
++ /etc
++ /etc/astute.yaml

rq/filelists/etc-centos-release

@@ -0,0 +1,2 @@
++ /etc
++ /etc/centos-release

rq/filelists/etc-ceph

@@ -1,2 +1,3 @@
-#ceph configs
++ /etc
-/etc/ceph/
++ /etc/ceph
++ /etc/ceph/**

rq/filelists/etc-cinder

@@ -1 +1,3 @@
-/etc/cinder/
++ /etc
++ /etc/cinder
++ /etc/cinder/**

rq/filelists/etc-cobbler

@@ -0,0 +1,4 @@
++ /etc
++ /etc/cobbler
++ /etc/cobbler*
++ /etc/cobbler/**

rq/filelists/etc-default

@@ -0,0 +1,3 @@
++ /etc
++ /etc/default
++ /etc/default/**

rq/filelists/etc-dnsmasq.conf

@@ -0,0 +1,2 @@
++ /etc
++ /etc/dnsmasq.conf

rq/filelists/etc-fuel

@@ -1 +1,3 @@
-/etc/fuel/
++ /etc
++ /etc/fuel
++ /etc/fuel/**

rq/filelists/etc-fuel_build_

@@ -0,0 +1,3 @@
++ /etc
++ /etc/fuel_build_number
++ /etc/fuel_build_id

rq/filelists/etc-glance

@@ -1 +1,3 @@
-/etc/glance/
++ /etc
++ /etc/glance
++ /etc/glance/**

rq/filelists/etc-haproxy

@@ -1 +1,3 @@
-/etc/haproxy/
++ /etc
++ /etc/haproxy
++ /etc/haproxy/**

rq/filelists/etc-heat

@@ -1 +1,3 @@
-/etc/heat/
++ /etc
++ /etc/heat
++ /etc/heat/**

rq/filelists/etc-hiera

@@ -0,0 +1,3 @@
++ /etc
++ /etc/hiera
++ /etc/hiera/**

rq/filelists/etc-keystone

@@ -1 +1,5 @@
-/etc/keystone/
++ /etc
++ /etc/keystone
+- /etc/keystone/ssl
+- /etc/keystone/fernet-keys
++ /etc/keystone/**

rq/filelists/etc-libvirt

@@ -1 +1,3 @@
-/etc/libvirt/
++ /etc
++ /etc/libvirt
++ /etc/libvirt/**

rq/filelists/etc-lsb-release

@@ -0,0 +1,2 @@
++ /etc
++ /etc/lsb-release

rq/filelists/etc-mcollective

@@ -0,0 +1,3 @@
++ /etc
++ /etc/mcollective
++ /etc/mcollective/**

rq/filelists/etc-mysql

@@ -1 +1,3 @@
-/etc/mysql/
++ /etc
++ /etc/mysql
++ /etc/mysql/**

rq/filelists/etc-nailgun

@@ -1 +1,3 @@
-/etc/nailgun/
++ /etc
++ /etc/nailgun
++ /etc/nailgun/**

rq/filelists/etc-network

@@ -0,0 +1,3 @@
++ /etc
++ /etc/network
++ /etc/network/**

rq/filelists/etc-neutron

@@ -1 +1,3 @@
-/etc/neutron/
++ /etc
++ /etc/neutron
++ /etc/neutron/**

rq/filelists/etc-nova

@@ -1 +1,3 @@
-/etc/nova/
++ /etc
++ /etc/nova
++ /etc/nova/**

rq/filelists/etc-redhat-release

@@ -0,0 +1,2 @@
++ /etc
++ /etc/redhat-release

rq/filelists/etc-resolv.conf

@@ -0,0 +1,2 @@
++ /etc
++ /etc/resolv.conf

rq/filelists/etc-swift

@@ -0,0 +1,3 @@
++ /etc
++ /etc/swift
++ /etc/swift/**

rq/filelists/etc-sysconfig

@@ -0,0 +1,3 @@
++ /etc
++ /etc/sysconfig
++ /etc/sysconfig/**

rq/filelists/etc-yum

@@ -1 +1,5 @@
-/etc/yum
++ /etc
++ /etc/yum
++ /etc/yum/**
++ /etc/yum.d
++ /etc/yum.d/**

rq/filelists/root-.ks

@@ -0,0 +1,2 @@
++ /root
++ /root/*.ks

rq/filelists/root-.log

@@ -0,0 +1,2 @@
++ /root
++ /root/*.log

rq/filelists/root-anaconda

@@ -0,0 +1,2 @@
++ /root
++ /root/anaconda*

rq/filelists/root-ceph

@@ -0,0 +1,3 @@
++ /root
++ /root/ceph*
++ /root/.ceph*

rq/filelists/var-lib-cloud

@@ -0,0 +1,4 @@
++ /var
++ /var/lib
++ /var/lib/cloud
++ /var/lib/cloud/**

rq/filelists/var-lib-puppet-state-last_run_summary.yaml

@@ -0,0 +1,5 @@
++ /var
++ /var/lib
++ /var/lib/puppet
++ /var/lib/puppet/state
++ /var/lib/puppet/state/last_run_summary.yaml

rq/filelists/var-run-pcap_dir

@@ -0,0 +1,4 @@
++ /var
++ /var/run
++ /var/run/pcap_dir
++ /var/run/pcap_dir/**

timmy/env.py

@@ -16,7 +16,7 @@
 #    under the License.
 
 project_name = 'timmy'
-version = '1.19.6'
+version = '1.20.0'
 
 if __name__ == '__main__':
     import sys

timmy/nodes.py

@@ -372,36 +372,33 @@ class Node(object):
         if self.files or self.filelists:
             ddir = os.path.join(self.outdir, Node.fkey, cl, self.repr)
             tools.mdir(ddir)
-        if self.shell_mode:
+        data = ''
-            for f in self.files:
+        for f in self.filelists:
-                outs, errs, code = tools.get_file_scp(ip=self.ip,
+            if os.path.sep in f:
-                                                      file=f,
+                fname = f
-                                                      ddir=ddir,
+            else:
-                                                      recursive=True)
+                fname = os.path.join(self.rqdir, Node.flkey, f)
-                self.check_code(code, 'get_files', 'tools.get_file_scp', errs)
+            try:
-        else:
+                with open(fname, 'r') as df:
-            data = ''
+                    for line in df:
-            for f in self.filelists:
+                        if not line.isspace() and line[0] != '#':
-                if os.path.sep in f:
+                            data += line
-                    fname = f
+            except:
-                else:
+                self.logger.error('could not read file: %s' % fname)
-                    fname = os.path.join(self.rqdir, Node.flkey, f)
+        self.logger.debug('%s: data:\n%s' % (self.repr, data))
-                try:
+        if data:
-                    with open(fname, 'r') as df:
+            o, e, c = tools.get_files_rsync(ip=self.ip,
-                        for line in df:
+                                            data=data,
-                            if not line.isspace() and line[0] != '#':
+                                            ssh_opts=self.ssh_opts,
-                                data += line
+                                            dpath=ddir,
-                except:
+                                            timeout=self.timeout)
-                    self.logger.error('could not read file: %s' % fname)
+            self.check_code(c, 'get_files', 'tools.get_files_rsync', e)
-            data += '\n'.join(self.files)
+        for f in self.files:
-            self.logger.debug('%s: data:\n%s' % (self.repr, data))
+            outs, errs, code = tools.get_file_scp(ip=self.ip,
-            if data:
+                                                  file=f,
-                o, e, c = tools.get_files_rsync(ip=self.ip,
+                                                  ddir=ddir,
-                                                data=data,
+                                                  recursive=True)
-                                                ssh_opts=self.ssh_opts,
+            self.check_code(code, 'get_files', 'tools.get_file_scp', errs)
-                                                dpath=ddir,
-                                                timeout=self.timeout)
-                self.check_code(c, 'get_files', 'tools.get_files_rsync', e)
 
     def put_files(self):
         self.logger.info('%s: putting files' % self.repr)

timmy/tools.py

@@ -277,14 +277,12 @@ def get_files_rsync(ip, data, ssh_opts, dpath, timeout=15):
         ssh_opts = ' '.join(ssh_opts)
     if (ip in ['localhost', '127.0.0.1']) or ip.startswith('127.'):
         logger.info("skip ssh rsync")
-        cmd = ("timeout '%s' rsync -avzr --files-from=- / '%s'"
+        cmd = ("timeout '%s' rsync -avzr --include-from=- / '%s' --exclude='*'"
-               " --exclude '*/ssl/*' --exclude '*/fernet-keys/*'"
                " --progress --partial --delete-before" %
                (timeout, dpath))
     else:
         cmd = ("timeout '%s' rsync -avzr -e 'ssh %s"
-               " -oCompression=no' --files-from=- '%s':/ '%s'"
+               " -oCompression=no' --include-from=- '%s':/ '%s' --exclude='*'"
-               " --exclude '*/ssl/*' --exclude '*/fernet-keys/*'"
                " --progress --partial --delete-before"
                ) % (timeout, ssh_opts, ip, dpath)
     logger.debug("command:%s\ndata:\n%s" % (cmd, data))
@@ -298,13 +296,15 @@ def get_file_scp(ip, file, ddir, timeout=600, recursive=False):
     ddir = os.path.join(os.path.normpath(ddir), dest)
     mdir(ddir)
     r = '-r ' if recursive else ''
-    cmd = "timeout '%s' scp -q %s'%s':'%s' '%s'" % (timeout, r, ip, file, ddir)
+    cmd = ("timeout '%s' scp -oStrictHostKeyChecking=no -q %s'%s':'%s' '%s'" %
+           (timeout, r, ip, file, ddir))
     return launch_cmd(cmd, timeout)
 
 
 def put_file_scp(ip, file, dest, timeout=600, recursive=True):
     r = '-r ' if recursive else ''
-    cmd = "timeout '%s' scp -q %s'%s' '%s':'%s'" % (timeout, r, file, ip, dest)
+    cmd = ("timeout '%s' scp -oStrictHostKeyChecking=no -q %s'%s' '%s':'%s'" %
+           (timeout, r, file, ip, dest))
     return launch_cmd(cmd, timeout)