From 2eae427377270f794ef1906ed741de1301f02be4 Mon Sep 17 00:00:00 2001 From: Dmitry Sutyagin Date: Mon, 12 Sep 2016 14:58:00 +0300 Subject: [PATCH] Change: filelists are now rsync include statements Update rq/default.yaml accordingly Fix scp not working when key is not in known_hosts Change-Id: Iddf645380bfc8c251021e22c0f4630dee72cb0df --- rq/default.yaml | 60 ++++++++++++------- rq/filelists/etc-apache2 | 7 +++ rq/filelists/etc-apt | 4 +- rq/filelists/etc-astute | 3 + rq/filelists/etc-astute.yaml | 2 + rq/filelists/etc-centos-release | 2 + rq/filelists/etc-ceph | 5 +- rq/filelists/etc-cinder | 4 +- rq/filelists/etc-cobbler | 4 ++ rq/filelists/etc-default | 3 + rq/filelists/etc-dnsmasq.conf | 2 + rq/filelists/etc-fuel | 4 +- rq/filelists/etc-fuel_build_ | 3 + rq/filelists/etc-glance | 4 +- rq/filelists/etc-haproxy | 4 +- rq/filelists/etc-heat | 4 +- rq/filelists/etc-hiera | 3 + rq/filelists/etc-keystone | 6 +- rq/filelists/etc-libvirt | 4 +- rq/filelists/etc-lsb-release | 2 + rq/filelists/etc-mcollective | 3 + rq/filelists/etc-mysql | 4 +- rq/filelists/etc-nailgun | 4 +- rq/filelists/etc-network | 3 + rq/filelists/etc-neutron | 4 +- rq/filelists/etc-nova | 4 +- rq/filelists/etc-redhat-release | 2 + rq/filelists/etc-resolv.conf | 2 + rq/filelists/etc-swift | 3 + rq/filelists/etc-sysconfig | 3 + rq/filelists/etc-yum | 6 +- rq/filelists/root-.ks | 2 + rq/filelists/root-.log | 2 + rq/filelists/root-anaconda | 2 + rq/filelists/root-ceph | 3 + rq/filelists/var-lib-cloud | 4 ++ ...var-lib-puppet-state-last_run_summary.yaml | 5 ++ rq/filelists/var-run-pcap_dir | 4 ++ timmy/env.py | 2 +- timmy/nodes.py | 57 +++++++++--------- timmy/tools.py | 12 ++-- 41 files changed, 187 insertions(+), 74 deletions(-) create mode 100644 rq/filelists/etc-apache2 create mode 100644 rq/filelists/etc-astute create mode 100644 rq/filelists/etc-astute.yaml create mode 100644 rq/filelists/etc-centos-release create mode 100644 rq/filelists/etc-cobbler create mode 100644 rq/filelists/etc-default create mode 100644 rq/filelists/etc-dnsmasq.conf create mode 100644 rq/filelists/etc-fuel_build_ create mode 100644 rq/filelists/etc-hiera create mode 100644 rq/filelists/etc-lsb-release create mode 100644 rq/filelists/etc-mcollective create mode 100644 rq/filelists/etc-network create mode 100644 rq/filelists/etc-redhat-release create mode 100644 rq/filelists/etc-resolv.conf create mode 100644 rq/filelists/etc-swift create mode 100644 rq/filelists/etc-sysconfig create mode 100644 rq/filelists/root-.ks create mode 100644 rq/filelists/root-.log create mode 100644 rq/filelists/root-anaconda create mode 100644 rq/filelists/root-ceph create mode 100644 rq/filelists/var-lib-cloud create mode 100644 rq/filelists/var-lib-puppet-state-last_run_summary.yaml create mode 100644 rq/filelists/var-run-pcap_dir diff --git a/rq/default.yaml b/rq/default.yaml index 11f70c6..57b4488 100644 --- a/rq/default.yaml +++ b/rq/default.yaml @@ -1,31 +1,47 @@ -files: - __default: ['/etc/resolv.conf', '/etc/mcollective', '/etc/astute.yaml', '/root/anaconda*', '/root/*.log', '/root/*.ks', '/var/lib/puppet/state/last_run_summary.yaml', '/var/run/pcap_dir', '/var/lib/cloud', '/etc/default'] +filelists: + __default: + - 'etc-astute.yaml' + - 'etc-default' + - 'etc-mcollective' + - 'etc-resolv.conf' + - 'root-anaconda' + - 'root-.ks' + - 'root-.log' + - 'var-lib-cloud' + - 'var-lib-puppet-state-last_run_summary.yaml' + - 'var-run-pcap_dir' by_roles: - controller: ['/etc/apache2', '/etc/keystone', '/etc/swift', '/etc/ceph', '/etc/cinder', '/etc/glance', '/etc/haproxy', '/etc/heat', '/etc/mysql', '/etc/nova'] + fuel: + - 'etc-astute' + - 'etc-centos-release' + - 'etc-cobbler' + - 'etc-dnsmasq.conf' + - 'etc-fuel' + - 'etc-fuel_build_' + - 'etc-nailgun' + controller: + - 'etc-apache2' + - 'etc-ceph' + - 'etc-cinder' + - 'etc-glance' + - 'etc-haproxy' + - 'etc-heat' + - 'etc-keystone' + - 'etc-mysql' + - 'etc-nova' + - 'etc-swift' + - 'root-ceph' cinder: ['/etc/cinder'] - fuel: ['/etc/astute', '/etc/dnsmasq.conf', '/etc/centos-release', '/etc/fuel_build_number', '/etc/fuel_build_id', '/etc/cobbler', '/etc/cobbler.dnsmasq.conf', '/root/*.log', '/etc/fuel', '/etc/nailgun'] - ceph: ['/root/.ceph*', '/etc/ceph'] - compute: ['/etc/libvirt', '/etc/nova'] - no_fuel: ['/etc/hiera', '/etc/neutron'] - virt: ['/etc/libvirt'] + ceph-osd: ['root-ceph', 'etc-ceph'] + compute: ['etc-libvirt', 'etc-nova'] + no_fuel: ['etc-hiera', 'etc-neutron'] + virt: ['etc-libvirt'] by_os_platform: - ubuntu: ['/etc/apt', '/etc/lsb-release', '/etc/network'] - centos: ['/etc/redhat-release', '/etc/sysconfig', '/etc/yum', '/etc/yum.d'] + ubuntu: ['etc-apt', 'etc-lsb-release', 'etc-network'] + centos: ['etc-redhat-release', 'etc-sysconfig', 'etc-yum'] by_release: '9.0': ['/etc/fuel'] '9.1': ['/etc/fuel'] -# filelists: -# by_roles: -# fuel: [etc-nailgun, etc-fuel] -# ceph-osd: [etc-ceph] -# cinder: [etc-cinder] -# compute: [etc-nova, etc-libvirt] -# controller: [etc-glance, etc-haproxy, etc-nova, -# etc-keystone, etc-neutron, etc-mysql, etc-ceph, etc-heat] -# # f3flight: need to make a better way to decide which files to collect -# by_os_platform: -# centos: [etc-yum] -# ubuntu: [etc-apt] scripts: by_release: '4.1': diff --git a/rq/filelists/etc-apache2 b/rq/filelists/etc-apache2 new file mode 100644 index 0000000..258f8ae --- /dev/null +++ b/rq/filelists/etc-apache2 @@ -0,0 +1,7 @@ ++ /etc ++ /etc/apache2 +- /etc/apache2/ssl ++ /etc/apache2/** ++ /etc/httpd +- /etc/httpd/ssl ++ /etc/httpd/** diff --git a/rq/filelists/etc-apt b/rq/filelists/etc-apt index bc8b814..449d575 100644 --- a/rq/filelists/etc-apt +++ b/rq/filelists/etc-apt @@ -1 +1,3 @@ -/etc/apt ++ /etc ++ /etc/apt ++ /etc/apt/** diff --git a/rq/filelists/etc-astute b/rq/filelists/etc-astute new file mode 100644 index 0000000..f1d436a --- /dev/null +++ b/rq/filelists/etc-astute @@ -0,0 +1,3 @@ ++ /etc ++ /etc/astute ++ /etc/astute/** diff --git a/rq/filelists/etc-astute.yaml b/rq/filelists/etc-astute.yaml new file mode 100644 index 0000000..4b64fe9 --- /dev/null +++ b/rq/filelists/etc-astute.yaml @@ -0,0 +1,2 @@ ++ /etc ++ /etc/astute.yaml diff --git a/rq/filelists/etc-centos-release b/rq/filelists/etc-centos-release new file mode 100644 index 0000000..1fe8552 --- /dev/null +++ b/rq/filelists/etc-centos-release @@ -0,0 +1,2 @@ ++ /etc ++ /etc/centos-release diff --git a/rq/filelists/etc-ceph b/rq/filelists/etc-ceph index dd85968..54418d1 100644 --- a/rq/filelists/etc-ceph +++ b/rq/filelists/etc-ceph @@ -1,2 +1,3 @@ -#ceph configs -/etc/ceph/ ++ /etc ++ /etc/ceph ++ /etc/ceph/** diff --git a/rq/filelists/etc-cinder b/rq/filelists/etc-cinder index af79d07..2857f55 100644 --- a/rq/filelists/etc-cinder +++ b/rq/filelists/etc-cinder @@ -1 +1,3 @@ -/etc/cinder/ ++ /etc ++ /etc/cinder ++ /etc/cinder/** diff --git a/rq/filelists/etc-cobbler b/rq/filelists/etc-cobbler new file mode 100644 index 0000000..514c196 --- /dev/null +++ b/rq/filelists/etc-cobbler @@ -0,0 +1,4 @@ ++ /etc ++ /etc/cobbler ++ /etc/cobbler* ++ /etc/cobbler/** diff --git a/rq/filelists/etc-default b/rq/filelists/etc-default new file mode 100644 index 0000000..2a3c687 --- /dev/null +++ b/rq/filelists/etc-default @@ -0,0 +1,3 @@ ++ /etc ++ /etc/default ++ /etc/default/** diff --git a/rq/filelists/etc-dnsmasq.conf b/rq/filelists/etc-dnsmasq.conf new file mode 100644 index 0000000..5bed3f7 --- /dev/null +++ b/rq/filelists/etc-dnsmasq.conf @@ -0,0 +1,2 @@ ++ /etc ++ /etc/dnsmasq.conf diff --git a/rq/filelists/etc-fuel b/rq/filelists/etc-fuel index b9127f2..55b827e 100644 --- a/rq/filelists/etc-fuel +++ b/rq/filelists/etc-fuel @@ -1 +1,3 @@ -/etc/fuel/ ++ /etc ++ /etc/fuel ++ /etc/fuel/** diff --git a/rq/filelists/etc-fuel_build_ b/rq/filelists/etc-fuel_build_ new file mode 100644 index 0000000..adb641f --- /dev/null +++ b/rq/filelists/etc-fuel_build_ @@ -0,0 +1,3 @@ ++ /etc ++ /etc/fuel_build_number ++ /etc/fuel_build_id diff --git a/rq/filelists/etc-glance b/rq/filelists/etc-glance index 930a6de..0513dad 100644 --- a/rq/filelists/etc-glance +++ b/rq/filelists/etc-glance @@ -1 +1,3 @@ -/etc/glance/ ++ /etc ++ /etc/glance ++ /etc/glance/** diff --git a/rq/filelists/etc-haproxy b/rq/filelists/etc-haproxy index 9c72f3a..17f64b9 100644 --- a/rq/filelists/etc-haproxy +++ b/rq/filelists/etc-haproxy @@ -1 +1,3 @@ -/etc/haproxy/ ++ /etc ++ /etc/haproxy ++ /etc/haproxy/** diff --git a/rq/filelists/etc-heat b/rq/filelists/etc-heat index 07a9ad1..bbff0c3 100644 --- a/rq/filelists/etc-heat +++ b/rq/filelists/etc-heat @@ -1 +1,3 @@ -/etc/heat/ ++ /etc ++ /etc/heat ++ /etc/heat/** diff --git a/rq/filelists/etc-hiera b/rq/filelists/etc-hiera new file mode 100644 index 0000000..87d1e8d --- /dev/null +++ b/rq/filelists/etc-hiera @@ -0,0 +1,3 @@ ++ /etc ++ /etc/hiera ++ /etc/hiera/** diff --git a/rq/filelists/etc-keystone b/rq/filelists/etc-keystone index 53e67dd..76563ca 100644 --- a/rq/filelists/etc-keystone +++ b/rq/filelists/etc-keystone @@ -1 +1,5 @@ -/etc/keystone/ ++ /etc ++ /etc/keystone +- /etc/keystone/ssl +- /etc/keystone/fernet-keys ++ /etc/keystone/** diff --git a/rq/filelists/etc-libvirt b/rq/filelists/etc-libvirt index 32cfc45..ef4a12e 100644 --- a/rq/filelists/etc-libvirt +++ b/rq/filelists/etc-libvirt @@ -1 +1,3 @@ -/etc/libvirt/ ++ /etc ++ /etc/libvirt ++ /etc/libvirt/** diff --git a/rq/filelists/etc-lsb-release b/rq/filelists/etc-lsb-release new file mode 100644 index 0000000..1446aa9 --- /dev/null +++ b/rq/filelists/etc-lsb-release @@ -0,0 +1,2 @@ ++ /etc ++ /etc/lsb-release diff --git a/rq/filelists/etc-mcollective b/rq/filelists/etc-mcollective new file mode 100644 index 0000000..58e835d --- /dev/null +++ b/rq/filelists/etc-mcollective @@ -0,0 +1,3 @@ ++ /etc ++ /etc/mcollective ++ /etc/mcollective/** diff --git a/rq/filelists/etc-mysql b/rq/filelists/etc-mysql index 5007c5a..0c29025 100644 --- a/rq/filelists/etc-mysql +++ b/rq/filelists/etc-mysql @@ -1 +1,3 @@ -/etc/mysql/ ++ /etc ++ /etc/mysql ++ /etc/mysql/** diff --git a/rq/filelists/etc-nailgun b/rq/filelists/etc-nailgun index 2bf074d..a5b6b59 100644 --- a/rq/filelists/etc-nailgun +++ b/rq/filelists/etc-nailgun @@ -1 +1,3 @@ -/etc/nailgun/ ++ /etc ++ /etc/nailgun ++ /etc/nailgun/** diff --git a/rq/filelists/etc-network b/rq/filelists/etc-network new file mode 100644 index 0000000..b263281 --- /dev/null +++ b/rq/filelists/etc-network @@ -0,0 +1,3 @@ ++ /etc ++ /etc/network ++ /etc/network/** diff --git a/rq/filelists/etc-neutron b/rq/filelists/etc-neutron index 80c7a75..556ed80 100644 --- a/rq/filelists/etc-neutron +++ b/rq/filelists/etc-neutron @@ -1 +1,3 @@ -/etc/neutron/ ++ /etc ++ /etc/neutron ++ /etc/neutron/** diff --git a/rq/filelists/etc-nova b/rq/filelists/etc-nova index f30d922..e450ae7 100644 --- a/rq/filelists/etc-nova +++ b/rq/filelists/etc-nova @@ -1 +1,3 @@ -/etc/nova/ ++ /etc ++ /etc/nova ++ /etc/nova/** diff --git a/rq/filelists/etc-redhat-release b/rq/filelists/etc-redhat-release new file mode 100644 index 0000000..c871daa --- /dev/null +++ b/rq/filelists/etc-redhat-release @@ -0,0 +1,2 @@ ++ /etc ++ /etc/redhat-release diff --git a/rq/filelists/etc-resolv.conf b/rq/filelists/etc-resolv.conf new file mode 100644 index 0000000..0f5f8f6 --- /dev/null +++ b/rq/filelists/etc-resolv.conf @@ -0,0 +1,2 @@ ++ /etc ++ /etc/resolv.conf diff --git a/rq/filelists/etc-swift b/rq/filelists/etc-swift new file mode 100644 index 0000000..e8f22bf --- /dev/null +++ b/rq/filelists/etc-swift @@ -0,0 +1,3 @@ ++ /etc ++ /etc/swift ++ /etc/swift/** diff --git a/rq/filelists/etc-sysconfig b/rq/filelists/etc-sysconfig new file mode 100644 index 0000000..017b396 --- /dev/null +++ b/rq/filelists/etc-sysconfig @@ -0,0 +1,3 @@ ++ /etc ++ /etc/sysconfig ++ /etc/sysconfig/** diff --git a/rq/filelists/etc-yum b/rq/filelists/etc-yum index 74769da..03e929e 100644 --- a/rq/filelists/etc-yum +++ b/rq/filelists/etc-yum @@ -1 +1,5 @@ -/etc/yum ++ /etc ++ /etc/yum ++ /etc/yum/** ++ /etc/yum.d ++ /etc/yum.d/** diff --git a/rq/filelists/root-.ks b/rq/filelists/root-.ks new file mode 100644 index 0000000..7bb0d35 --- /dev/null +++ b/rq/filelists/root-.ks @@ -0,0 +1,2 @@ ++ /root ++ /root/*.ks diff --git a/rq/filelists/root-.log b/rq/filelists/root-.log new file mode 100644 index 0000000..7a50635 --- /dev/null +++ b/rq/filelists/root-.log @@ -0,0 +1,2 @@ ++ /root ++ /root/*.log diff --git a/rq/filelists/root-anaconda b/rq/filelists/root-anaconda new file mode 100644 index 0000000..892e8f1 --- /dev/null +++ b/rq/filelists/root-anaconda @@ -0,0 +1,2 @@ ++ /root ++ /root/anaconda* diff --git a/rq/filelists/root-ceph b/rq/filelists/root-ceph new file mode 100644 index 0000000..73ad103 --- /dev/null +++ b/rq/filelists/root-ceph @@ -0,0 +1,3 @@ ++ /root ++ /root/ceph* ++ /root/.ceph* diff --git a/rq/filelists/var-lib-cloud b/rq/filelists/var-lib-cloud new file mode 100644 index 0000000..a1dc913 --- /dev/null +++ b/rq/filelists/var-lib-cloud @@ -0,0 +1,4 @@ ++ /var ++ /var/lib ++ /var/lib/cloud ++ /var/lib/cloud/** diff --git a/rq/filelists/var-lib-puppet-state-last_run_summary.yaml b/rq/filelists/var-lib-puppet-state-last_run_summary.yaml new file mode 100644 index 0000000..4c6e134 --- /dev/null +++ b/rq/filelists/var-lib-puppet-state-last_run_summary.yaml @@ -0,0 +1,5 @@ ++ /var ++ /var/lib ++ /var/lib/puppet ++ /var/lib/puppet/state ++ /var/lib/puppet/state/last_run_summary.yaml diff --git a/rq/filelists/var-run-pcap_dir b/rq/filelists/var-run-pcap_dir new file mode 100644 index 0000000..c17af2a --- /dev/null +++ b/rq/filelists/var-run-pcap_dir @@ -0,0 +1,4 @@ ++ /var ++ /var/run ++ /var/run/pcap_dir ++ /var/run/pcap_dir/** diff --git a/timmy/env.py b/timmy/env.py index 8642f0b..1583eaf 100644 --- a/timmy/env.py +++ b/timmy/env.py @@ -16,7 +16,7 @@ # under the License. project_name = 'timmy' -version = '1.19.6' +version = '1.20.0' if __name__ == '__main__': import sys diff --git a/timmy/nodes.py b/timmy/nodes.py index 1a068e4..4726eec 100644 --- a/timmy/nodes.py +++ b/timmy/nodes.py @@ -372,36 +372,33 @@ class Node(object): if self.files or self.filelists: ddir = os.path.join(self.outdir, Node.fkey, cl, self.repr) tools.mdir(ddir) - if self.shell_mode: - for f in self.files: - outs, errs, code = tools.get_file_scp(ip=self.ip, - file=f, - ddir=ddir, - recursive=True) - self.check_code(code, 'get_files', 'tools.get_file_scp', errs) - else: - data = '' - for f in self.filelists: - if os.path.sep in f: - fname = f - else: - fname = os.path.join(self.rqdir, Node.flkey, f) - try: - with open(fname, 'r') as df: - for line in df: - if not line.isspace() and line[0] != '#': - data += line - except: - self.logger.error('could not read file: %s' % fname) - data += '\n'.join(self.files) - self.logger.debug('%s: data:\n%s' % (self.repr, data)) - if data: - o, e, c = tools.get_files_rsync(ip=self.ip, - data=data, - ssh_opts=self.ssh_opts, - dpath=ddir, - timeout=self.timeout) - self.check_code(c, 'get_files', 'tools.get_files_rsync', e) + data = '' + for f in self.filelists: + if os.path.sep in f: + fname = f + else: + fname = os.path.join(self.rqdir, Node.flkey, f) + try: + with open(fname, 'r') as df: + for line in df: + if not line.isspace() and line[0] != '#': + data += line + except: + self.logger.error('could not read file: %s' % fname) + self.logger.debug('%s: data:\n%s' % (self.repr, data)) + if data: + o, e, c = tools.get_files_rsync(ip=self.ip, + data=data, + ssh_opts=self.ssh_opts, + dpath=ddir, + timeout=self.timeout) + self.check_code(c, 'get_files', 'tools.get_files_rsync', e) + for f in self.files: + outs, errs, code = tools.get_file_scp(ip=self.ip, + file=f, + ddir=ddir, + recursive=True) + self.check_code(code, 'get_files', 'tools.get_file_scp', errs) def put_files(self): self.logger.info('%s: putting files' % self.repr) diff --git a/timmy/tools.py b/timmy/tools.py index 9dadda7..d980d6c 100644 --- a/timmy/tools.py +++ b/timmy/tools.py @@ -277,14 +277,12 @@ def get_files_rsync(ip, data, ssh_opts, dpath, timeout=15): ssh_opts = ' '.join(ssh_opts) if (ip in ['localhost', '127.0.0.1']) or ip.startswith('127.'): logger.info("skip ssh rsync") - cmd = ("timeout '%s' rsync -avzr --files-from=- / '%s'" - " --exclude '*/ssl/*' --exclude '*/fernet-keys/*'" + cmd = ("timeout '%s' rsync -avzr --include-from=- / '%s' --exclude='*'" " --progress --partial --delete-before" % (timeout, dpath)) else: cmd = ("timeout '%s' rsync -avzr -e 'ssh %s" - " -oCompression=no' --files-from=- '%s':/ '%s'" - " --exclude '*/ssl/*' --exclude '*/fernet-keys/*'" + " -oCompression=no' --include-from=- '%s':/ '%s' --exclude='*'" " --progress --partial --delete-before" ) % (timeout, ssh_opts, ip, dpath) logger.debug("command:%s\ndata:\n%s" % (cmd, data)) @@ -298,13 +296,15 @@ def get_file_scp(ip, file, ddir, timeout=600, recursive=False): ddir = os.path.join(os.path.normpath(ddir), dest) mdir(ddir) r = '-r ' if recursive else '' - cmd = "timeout '%s' scp -q %s'%s':'%s' '%s'" % (timeout, r, ip, file, ddir) + cmd = ("timeout '%s' scp -oStrictHostKeyChecking=no -q %s'%s':'%s' '%s'" % + (timeout, r, ip, file, ddir)) return launch_cmd(cmd, timeout) def put_file_scp(ip, file, dest, timeout=600, recursive=True): r = '-r ' if recursive else '' - cmd = "timeout '%s' scp -q %s'%s' '%s':'%s'" % (timeout, r, file, ip, dest) + cmd = ("timeout '%s' scp -oStrictHostKeyChecking=no -q %s'%s' '%s':'%s'" % + (timeout, r, file, ip, dest)) return launch_cmd(cmd, timeout)