Merge remote-tracking branch 'origin/ng' into ng

cmds.py

@@ -51,14 +51,9 @@ def main(argv=None):
                         format='%(asctime)s %(levelname)s %(message)s')
     args.extended = args.extended == "1"
     conf = loadconf.load_conf(args.config)
-    n = nodes.Nodes(filesd=conf['rqdir'],
+    n = nodes.Nodes(conf=conf,
-                    logdir=conf['logdir'],
                     extended=args.extended,
-                    fuelip=conf['fuelip'],
                     cluster=args.cluster,
-                    sshopts=conf['ssh']['opts'],
-                    sshvars=conf['ssh']['vars'],
-                    timeout=conf['timeout'],
                     destdir=args.dest_file)
     # nodes.print_nodes()
     n.get_node_file_list()

config.yaml

@@ -8,3 +8,12 @@ out-dir: ../timmy-ng/info
 timeout: 15
 find:
     template: -name '*.log'
+log-files:
+     default: -name '*.log'
+     by-role:
+         compute: -name '*.log'
+         controller: -name '*.log' -name '*.log' -o -name '*.log.1' -o -name '*' ! -path '/var/log/atop*'
+         mongo: -name '*.log'
+         ceph: -name '*.log'
+     by-node-id:
+         0: -name '*.log' -o -name '*.log.1'

default.yaml

@@ -10,3 +10,7 @@ find:
     path: /var/log/
 logs-archive: /tmp/timmy-logs.tar
 compress-timeout: 3600
+log-files:
+     template: -name '*.gz' -o -name '*.log' -o -name '*-[0-9]4'
+     by-role: compute, controller, mongo, ceph
+     by-node-id: 0

getlogs.py

@@ -58,18 +58,14 @@ def main(argv=None):
                         format='%(asctime)s %(levelname)s %(message)s')
     conf = loadconf.load_conf('config.yaml')
     args.extended = args.extended == "1"
-    n = nodes.Nodes(filesd=conf['rqdir'],
+    n = nodes.Nodes(conf=conf,
-                    logdir=conf['logdir'],
                     extended=args.extended,
-                    fuelip=conf['fuelip'],
                     cluster=args.cluster,
-                    sshopts=conf['ssh']['opts'],
-                    sshvars=conf['ssh']['vars'],
-                    timeout=conf['timeout'],
                     destdir=args.dest_file)
     lock = flock.FLock('/tmp/timmy-logs.lock')
     if not lock.lock():
         logging.warning('Unable to obtain lock, skipping "logs"-part')
+        return 1
     n.get_node_file_list()
     n.calculate_log_size(conf['find']['template'])
     if n.is_enough_space():

nodes.py

@@ -32,12 +32,13 @@ from tools import *
 ckey = 'cmds'
 fkey = 'files'
 lkey = 'logs'
+varlogdir = '/var/log'
 
 
 class Node(object):
 
     def __init__(self, node_id, mac, cluster, roles, os_platform,
-                 online, status, ip):
+                 online, status, ip, flogs=False):
         self.node_id = node_id
         self.mac = mac
         self.cluster = cluster
@@ -49,6 +50,8 @@ class Node(object):
         self.files = {}
         self.data = {}
         self.logsize = 0
+        # include logs from the command 'find /var/log/ ...'
+        self.flogs = flogs
 
     def set_files(self, dirname, key, ds, version):
         files = []
@@ -185,9 +188,9 @@ class Node(object):
                           (self.node_id, key, self.data[key]))
 
     def log_size_from_find(self, template, sshopts, odir, timeout=5):
-        # template = "\( -name '*.gz' -o -name '*.log' -o -name '*-[0-9]4' \)"
+        logging.info('template find: %s' % template)
-        cmd = ("find "
+        cmd = ("find '%s' -type f \( %s \) -exec du -b {} +" %
-               "/var/log -type f \( %s \) -exec du -b {} +" % template)
+               (varlogdir, str(template)))
         logging.info('node: %s, logs du-cmd: %s' % (self.node_id, cmd))
         outs, errs, code = ssh_node(ip=self.ip,
                                     command=cmd,
@@ -229,21 +232,21 @@ class Node(object):
 class Nodes(object):
     """Class nodes """
 
-    def __init__(self, filesd, logdir, extended, timeout,
+    def __init__(self, cluster, extended, conf, destdir, filename=None):
-                 fuelip, cluster, sshopts, sshvars, destdir, filename=None):
+        self.dirname = conf['rqdir'].rstrip('/')
-        self.dirname = filesd.rstrip('/')
         if (not os.path.exists(self.dirname)):
             logging.error("directory %s doesn't exist" % (self.dirname))
             sys.exit(1)
-        self.files = get_dir_structure(filesd)[os.path.basename(self.dirname)]
+        self.files = get_dir_structure(conf['rqdir'])[os.path.basename(self.dirname)]
-        self.fuelip = fuelip
+        self.fuelip = conf['fuelip']
-        self.sshopts = sshopts
+        self.sshopts = conf['ssh']['opts']
-        self.sshvars = sshvars
+        self.sshvars = conf['ssh']['vars']
-        self.timeout = timeout
+        self.timeout = conf['timeout']
+        self.conf = conf
         self.destdir = destdir
         self.get_version()
         self.cluster = cluster
-        self.logdir = logdir
+        self.logdir = conf['logdir']
         self.extended = extended
         logging.info('extended: %s' % self.extended)
         if filename is not None:
@@ -415,13 +418,12 @@ class Nodes(object):
             logging.error("Can't create archive %s" % (errs))
 
     def create_archive_logs(self, template, outfile, timeout):
-        sdir = '/var/log'
         fuelnode = self.nodes[self.fuelip]
         tstr = '--transform \\"flags=r;s|^|logs/fuel/|\\"'
         cmd = ("find %s -type f \( %s \) -print0 "
                "| tar --create %s --file - "
                "--null --files-from -" %
-               (sdir, template, tstr))
+               (varlogdir, template, tstr))
         outs, errs, code = ssh_node(ip=fuelnode.ip,
                                     command=cmd,
                                     sshopts=self.sshopts,