x/timmy
Code Issues Proposed changes

Implement timestamping for outputs and dirs

Browse Source 
+ remove occasional debug from get_logs
This commit is contained in:
f3flight 2016-05-27 01:01:27 +00:00
parent 842fbbc507
commit 4a59582193
4 changed files with 76 additions and 47 deletions

2
setup.py

@ -9,7 +9,7 @@ rqfiles = [(os.path.join(dtm, root), [os.path.join(root, f) for f in files])
rqfiles.append((os.path.join(dtm, 'configs'), ['config.yaml', 'rq.yaml']))
setup(name='timmy',
version='1.1.2',
version='1.2.0',
author="Aleksandr Dobdin",
author_email='dobdin@gmail.com',
license='Apache2',

47
timmy/cli.py

@ -51,7 +51,9 @@ def main(argv=None):
parser.add_argument('-o', '--dest-file',
help=('Output filename for the archive in tar.gz'
' format for command outputs and collected'
' files. Overrides "archives" config option.'))
' files. Overrides "archive_" config options.'
' If logs are collected they will be placed'
' in the same folder (but separate archives).'))
parser.add_argument('--log-file', default=None,
help='Redirect Timmy log to a file.')
parser.add_argument('-e', '--env', type=int,
@ -114,6 +116,20 @@ def main(argv=None):
'execution.'))
parser.add_argument('-L', '--logs-maxthreads', type=int, default=100,
help='Maximum simultaneous nodes for log collection.')
parser.add_argument('-t', '--outputs-timestamp',
help='Add timestamp to outputs - allows accumulating'
' outputs of identical commands/scripts across'
' runs. Only makes sense with --no-clean for'
' subsequent runs.',
action='store_true')
parser.add_argument('-T', '--dir-timestamp',
help='Add timestamp to output folders (defined by'
' "outdir" and "archive_dir" config options).'
' Makes each run store results in new folders.'
' This way Timmy will always preserve previous'
' results. Do not forget to clean up the results'
' manually when using this option.',
action='store_true')
parser.add_argument('-w', '--warning',
help='Sets log level to warning (default).',
action='store_true')
@ -168,9 +184,13 @@ def main(argv=None):
filter['roles'] = args.role
if args.env is not None:
filter['cluster'] = [args.env]
main_arc = os.path.join(conf['archives'], 'general.tar.gz')
if args.outputs_timestamp:
conf['outputs_timestamp'] = True
if args.dir_timestamp:
conf['dir_timestamp'] = True
if args.dest_file:
main_arc = args.dest_file
conf['archive_dir'] = os.path.split(args.dest_file)[0]
conf['archive_name'] = os.path.split(args.dest_file)[1]
nm = pretty_run(args.quiet, 'Initializing node data',
NodeManager,
kwargs={'conf': conf, 'extended': args.extended,
@ -180,15 +200,13 @@ def main(argv=None):
pretty_run(args.quiet, 'Uploading files', nm.put_files)
if nm.has(Node.ckey, Node.skey):
pretty_run(args.quiet, 'Executing commands and scripts',
nm.run_commands, args=(conf['outdir'],
args.maxthreads))
nm.run_commands, args=(args.maxthreads,))
if nm.has(Node.fkey, Node.flkey):
pretty_run(args.quiet, 'Collecting files and filelists',
nm.get_files, args=(conf['outdir'], args.maxthreads))
nm.get_files, args=(args.maxthreads,))
if not args.no_archive and nm.has(*Node.conf_archive_general):
pretty_run(args.quiet, 'Creating outputs and files archive',
nm.create_archive_general, args=(conf['outdir'],
main_arc, 60))
nm.create_archive_general, args=(60,))
if args.only_logs or args.getlogs:
size = pretty_run(args.quiet, 'Calculating logs size',
nm.calculate_log_size, args=(args.maxthreads,))
@ -196,16 +214,15 @@ def main(argv=None):
logging.warning('Size zero - no logs to collect.')
return
enough = pretty_run(args.quiet, 'Checking free space',
nm.is_enough_space, args=(conf['archives'],))
nm.is_enough_space)
if enough:
pretty_run(args.quiet, 'Collecting and packing logs', nm.get_logs,
args=(conf['archives'], conf['compress_timeout']),
args=(conf['compress_timeout'],),
kwargs={'maxthreads': args.logs_maxthreads,
'fake': args.fake_logs})
else:
logging.warning(('Not enough space for logs in "%s", skipping'
'log collection.') %
conf['archives'])
'log collection.') % nm.conf['archive_dir'])
logging.info("Nodes:\n%s" % nm)
if not args.quiet:
print('Run complete. Node information:')
@ -217,11 +234,11 @@ def main(argv=None):
for node in nm.sorted_nodes():
node.print_results(node.mapcmds)
node.print_results(node.mapscr)
if nm.has(Node.fkey, Node.flkey) and not args.quiet:
print('Outputs and files available in "%s".' % conf['outdir'])
if nm.has(Node.ckey, Node.skey, Node.fkey, Node.flkey) and not args.quiet:
print('Outputs and/or files available in "%s".' % nm.conf['outdir'])
if all([not args.no_archive, nm.has(*Node.conf_archive_general),
not args.quiet]):
print('Archives available in "%s".' % conf['archives'])
print('Archives available in "%s".' % nm.conf['archive_dir'])
return 0
if __name__ == '__main__':

12
timmy/conf.py

@ -13,7 +13,6 @@ def load_conf(filename):
'-lroot', '-oBatchMode=yes']
conf['env_vars'] = ['OPENRC=/root/openrc', 'IPTABLES_STR="iptables -nvL"']
conf['fuelip'] = '127.0.0.1'
conf['outdir'] = os.path.join(gettempdir(), 'timmy', 'info')
conf['timeout'] = 15
conf['prefix'] = 'nice -n 19 ionice -c 3'
rqdir = 'rq'
@ -28,8 +27,11 @@ def load_conf(filename):
else:
conf['rqfile'] = rqfile
conf['compress_timeout'] = 3600
conf['archives'] = os.path.join(gettempdir(), 'timmy', 'archives')
conf['cmds_archive'] = ''
conf['outdir'] = os.path.join(gettempdir(), 'timmy', 'info')
conf['archive_dir'] = os.path.join(gettempdir(), 'timmy', 'archives')
conf['archive_name'] = 'general.tar.gz'
conf['outputs_timestamp'] = False
conf['dir_timestamp'] = False
conf['put'] = []
conf['cmds'] = []
conf['scripts'] = []
@ -45,9 +47,9 @@ def load_conf(filename):
Skip Fuel node;
Print command execution results. Files and outputs will also be in a
place specified by conf['outdir'], archive will also be created and put
in a place specified by conf['archives'].'''
in a place specified by conf['archive_dir'].'''
conf['shell_mode'] = False
'''Clean - erase previous results in outdir and archives dir, if any.'''
'''Clean - erase previous results in outdir and archive_dir dir, if any.'''
conf['clean'] = True
if filename:
conf_extra = load_yaml_file(filename)

62
timmy/nodes.py

@ -25,6 +25,7 @@ import shutil
import logging
import sys
import re
from datetime import datetime
import tools
from tools import w_list, run_with_lock
from copy import deepcopy
@ -70,6 +71,8 @@ class Node(object):
self.mapcmds = {}
self.mapscr = {}
self.filtered_out = False
self.outputs_timestamp = False
self.outputs_timestamp_dir = None
self.apply_conf(conf)
def __str__(self):
@ -162,11 +165,11 @@ class Node(object):
(self.id, self.release))
return self
def exec_cmd(self, odir='info', fake=False, ok_codes=None):
def exec_cmd(self, fake=False, ok_codes=None):
sn = 'node-%s' % self.id
cl = 'cluster-%s' % self.cluster
logging.debug('%s/%s/%s/%s' % (odir, Node.ckey, cl, sn))
ddir = os.path.join(odir, Node.ckey, cl, sn)
logging.debug('%s/%s/%s/%s' % (self.outdir, Node.ckey, cl, sn))
ddir = os.path.join(self.outdir, Node.ckey, cl, sn)
if self.cmds:
tools.mdir(ddir)
self.cmds = sorted(self.cmds)
@ -174,6 +177,8 @@ class Node(object):
for cmd in c:
dfile = os.path.join(ddir, 'node-%s-%s-%s' %
(self.id, self.ip, cmd))
if self.outputs_timestamp:
dfile += self.outputs_timestamp_str
logging.info('outfile: %s' % dfile)
self.mapcmds[cmd] = dfile
if not fake:
@ -190,7 +195,6 @@ class Node(object):
except:
logging.error("exec_cmd: can't write to file %s" %
dfile)
ddir = os.path.join(odir, Node.skey, cl, sn)
if self.scripts:
tools.mdir(ddir)
self.scripts = sorted(self.scripts)
@ -199,6 +203,8 @@ class Node(object):
logging.info('node:%s(%s), exec: %s' % (self.id, self.ip, f))
dfile = os.path.join(ddir, 'node-%s-%s-%s' %
(self.id, self.ip, os.path.basename(f)))
if self.outputs_timestamp:
dfile += self.outputs_timestamp_str
logging.info('outfile: %s' % dfile)
self.mapscr[scr] = dfile
if not fake:
@ -231,12 +237,12 @@ class Node(object):
prefix=self.prefix)
self.check_code(code, 'exec_simple_cmd', cmd, ok_codes)
def get_files(self, odir='info', timeout=15):
def get_files(self, timeout=15):
logging.info('get_files: node: %s, IP: %s' % (self.id, self.ip))
sn = 'node-%s' % self.id
cl = 'cluster-%s' % self.cluster
if self.files or self.filelists:
ddir = os.path.join(odir, Node.fkey, cl, sn)
ddir = os.path.join(self.outdir, Node.fkey, cl, sn)
tools.mdir(ddir)
if self.shell_mode:
for f in self.files:
@ -344,10 +350,16 @@ class NodeManager(object):
def __init__(self, conf, extended=False, nodes_json=None):
self.conf = conf
self.nodes = {}
if conf['outputs_timestamp'] or conf['dir_timestamp']:
timestamp_str = datetime.now().strftime('_%F_%H-%M-%S')
if conf['outputs_timestamp']:
conf['outputs_timestamp_str'] = timestamp_str
if conf['dir_timestamp']:
conf['outdir'] += timestamp_str
conf['archive_dir'] += timestamp_str
if conf['clean']:
shutil.rmtree(conf['outdir'], ignore_errors=True)
shutil.rmtree(conf['archives'], ignore_errors=True)
shutil.rmtree(conf['archive_dir'], ignore_errors=True)
if not conf['shell_mode']:
self.rqdir = conf['rqdir']
if (not os.path.exists(self.rqdir)):
@ -356,6 +368,7 @@ class NodeManager(object):
sys.exit(1)
if self.conf['rqfile']:
self.import_rq()
self.nodes = {}
self.fuel_init()
if nodes_json:
try:
@ -546,14 +559,12 @@ class NodeManager(object):
return all(checks)
@run_with_lock
def run_commands(self, odir='info', timeout=15, fake=False,
maxthreads=100):
def run_commands(self, timeout=15, fake=False, maxthreads=100):
run_items = []
for key, node in self.nodes.items():
if not node.filtered_out:
run_items.append(tools.RunItem(target=node.exec_cmd,
args={'odir': odir,
'fake': fake},
args={'fake': fake},
key=key))
result = tools.run_batch(run_items, maxthreads, dict_result=True)
for key in result:
@ -577,9 +588,9 @@ class NodeManager(object):
self.alogsize = total_size / 1024
return self.alogsize
def is_enough_space(self, directory, coefficient=1.2):
tools.mdir(directory)
outs, errs, code = tools.free_space(directory, timeout=1)
def is_enough_space(self, coefficient=1.2):
tools.mdir(self.conf['outdir'])
outs, errs, code = tools.free_space(self.conf['outdir'], timeout=1)
if code != 0:
logging.error("Can't get free space: %s" % errs)
return False
@ -597,9 +608,11 @@ class NodeManager(object):
return True
@run_with_lock
def create_archive_general(self, directory, outfile, timeout):
cmd = "tar zcf '%s' -C %s %s" % (outfile, directory, ".")
tools.mdir(self.conf['archives'])
def create_archive_general(self, timeout):
outfile = os.path.join(self.conf['archive_dir'],
self.conf['archive_name'])
cmd = "tar zcf '%s' -C %s %s" % (outfile, self.conf['outdir'], ".")
tools.mdir(self.conf['archive_dir'])
logging.debug("create_archive_general: cmd: %s" % cmd)
outs, errs, code = tools.launch_cmd(cmd, timeout)
if code != 0:
@ -622,7 +635,7 @@ class NodeManager(object):
return speed
@run_with_lock
def get_logs(self, outdir, timeout, fake=False, maxthreads=10, speed=100):
def get_logs(self, timeout, fake=False, maxthreads=10, speed=100):
if fake:
logging.info('get_logs: fake = True, skipping' % fake)
return
@ -636,15 +649,13 @@ class NodeManager(object):
logging.info(("get_logs: node %s - no logs "
"to collect") % node.id)
continue
node.archivelogsfile = os.path.join(outdir,
node.archivelogsfile = os.path.join(self.conf['archive_dir'],
'logs-node-%s.tar.gz' %
str(node.id))
tools.mdir(outdir)
tools.mdir(self.conf['outdir'])
input = ''
for fn in node.logs_dict():
input += '%s\0' % fn.lstrip(os.path.abspath(os.sep))
with open('test-%s' % node.id, 'w') as fi:
fi.write(input)
cmd = ("tar --gzip -C %s --create --warning=no-file-changed "
" --file - --null --files-from -" % os.path.abspath(os.sep))
if not (node.ip == 'localhost' or node.ip.startswith('127.')):
@ -665,11 +676,10 @@ class NodeManager(object):
logging.error("get_logs: can't delete file %s" % tfile)
@run_with_lock
def get_files(self, odir=Node.fkey, timeout=15):
def get_files(self, timeout=15):
run_items = []
for n in [n for n in self.nodes.values() if not n.filtered_out]:
run_items.append(tools.RunItem(target=n.get_files,
args={'odir': odir}))
run_items.append(tools.RunItem(target=n.get_files))
tools.run_batch(run_items, 10)
@run_with_lock